From cb0c1a4f161d2b428c588b91e351790069ddd880 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Thu, 28 Jun 2018 12:34:47 -0400 Subject: [PATCH 01/97] Fixed CONTRIBUTING links --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 34e777e8..0cc5edcf 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ This is the official Cloudant library for Python. * [API Reference](http://python-cloudant.readthedocs.io/en/latest/cloudant.html) * [Related Documentation](#related-documentation) * [Development](#development) - * [Contributing](CONTRIBUTING.rst) - * [Test Suite](CONTRIBUTING.rst#running-the-tests) + * [Contributing](CONTRIBUTING.md) + * [Test Suite](CONTRIBUTING.md#running-the-tests) * [Using in Other Projects](#using-in-other-projects) * [License](#license) * [Issues](#issues) @@ -44,7 +44,7 @@ See [API reference docs (readthedocs.io)](http://python-cloudant.readthedocs.io/ ## Development -See [CONTRIBUTING.rst](https://github.com/cloudant/python-cloudant/blob/master/CONTRIBUTING.rst) +See [CONTRIBUTING.md](https://github.com/cloudant/python-cloudant/blob/master/CONTRIBUTING.md) ## Using in other projects From f0d00100412d84452d69f5464054be0d1cf74557 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Tue, 3 Jul 2018 13:21:32 +0100 Subject: [PATCH 02/97] Corrected bad merge of CHANGES --- CHANGES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 6aa70c70..432ae241 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,11 +1,12 @@ # Unreleased +- [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. + # 2.9.0 (2018-06-13) - [NEW] Added functionality to test if a key is in a database as in `key in db`, overriding dict `__contains__` and checking in the remote database. - [NEW] Moved `create_query_index` and other query related methods to `CouchDatabase` as the `_index`/`_find` API is available in CouchDB 2.x. - [NEW] Support IAM authentication in replication documents. -- [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. - [FIXED] Case where `Document` context manager would throw instead of creating a new document if no `_id` was provided. - [IMPROVED] Added support for IAM API key in `cloudant_bluemix` method. - [IMPROVED] Shortened length of client URLs by removing username and password. From fa1cf06ef36fd4a8270c7b477d76601b0b399b91 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Mon, 2 Jul 2018 17:01:26 +0100 Subject: [PATCH 03/97] Stopped raising exception on conflict retry success Moved exception raise into else block. Added tests for conflict retry cases: i) max_tries exhuastion ii) success on retry --- CHANGES.md | 1 + src/cloudant/document.py | 3 +- tests/unit/document_tests.py | 56 ++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 432ae241..1ca11aa0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ # Unreleased - [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. +- [FIXED] Case where an exception was raised after successful retry when using `doc.update_field`. # 2.9.0 (2018-06-13) diff --git a/src/cloudant/document.py b/src/cloudant/document.py index e8412c3c..768ecca5 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -259,7 +259,8 @@ def _update_field(self, action, field, value, max_tries, tries=0): if tries < max_tries and ex.response.status_code == 409: self._update_field( action, field, value, max_tries, tries=tries+1) - raise + else: + raise def update_field(self, action, field, value, max_tries=10): """ diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index 3881326a..dd443ccf 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -474,6 +474,62 @@ def test_update_field(self): self.assertTrue(doc['_rev'].startswith('2-')) self.assertEqual(doc['pets'], ['cat', 'dog', 'fish']) + @mock.patch('cloudant.document.Document.save') + def test_update_field_maxretries(self, m_save): + """ + Test that conflict retries work for updating a single field. + """ + # Create a doc + doc = Document(self.db, 'julia006') + doc['name'] = 'julia' + doc['age'] = 6 + doc.create() + self.assertTrue(doc['_rev'].startswith('1-')) + self.assertEqual(doc['age'], 6) + # Mock conflicts when saving updates + m_save.side_effect = requests.HTTPError(response=mock.Mock(status_code=409, reason='conflict')) + # Tests that failing on retry eventually throws + with self.assertRaises(requests.HTTPError) as cm: + doc.update_field(doc.field_set, 'age', 7, max_tries=2) + + # There is an off-by-one error for "max_tries" + # It really means max_retries i.e. 1 attempt + # followed by a max of 2 retries + self.assertEqual(m_save.call_count, 3) + self.assertEqual(cm.exception.response.status_code, 409) + self.assertEqual(cm.exception.response.reason, 'conflict') + # Fetch again before asserting, otherwise we assert against + # the locally updated age field + doc.fetch() + self.assertFalse(doc['_rev'].startswith('2-')) + self.assertNotEqual(doc['age'], 7) + + def test_update_field_success_on_retry(self): + """ + Test that conflict retries work for updating a single field. + """ + # Create a doc + doc = Document(self.db, 'julia006') + doc['name'] = 'julia' + doc['age'] = 6 + doc.create() + self.assertTrue(doc['_rev'].startswith('1-')) + self.assertEqual(doc['age'], 6) + + # Mock when saving the document + # 1st call throw a 409 + # 2nd call delegate to the real doc.save() + with mock.patch('cloudant.document.Document.save', + side_effect=[requests.HTTPError(response=mock.Mock(status_code=409, reason='conflict')), + doc.save()]) as m_save: + # A list of side effects containing only 1 element + doc.update_field(doc.field_set, 'age', 7, max_tries=1) + # Two calls to save, one with a 409 and one that succeeds + self.assertEqual(m_save.call_count, 2) + # Check that the _rev and age field were updated + self.assertTrue(doc['_rev'].startswith('2-')) + self.assertEqual(doc['age'], 7) + def test_delete_document_failure(self): """ Test failure condition when attempting to remove a document From 82017411beb12717780d036f7308086d8fe37ef5 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Mon, 9 Jul 2018 16:00:46 +0100 Subject: [PATCH 04/97] Add custom JSON encoder/decoder option to `Document` constructor --- CHANGES.md | 1 + src/cloudant/document.py | 9 +++++--- tests/unit/document_tests.py | 44 ++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1ca11aa0..3c3e925a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ # Unreleased +- [NEW] Add custom JSON encoder/decoder option to `Document` constructor. - [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. - [FIXED] Case where an exception was raised after successful retry when using `doc.update_field`. diff --git a/src/cloudant/document.py b/src/cloudant/document.py index 768ecca5..7feda18a 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -53,8 +53,10 @@ class Document(dict): :param database: A database instance used by the Document. Can be either a ``CouchDatabase`` or ``CloudantDatabase`` instance. :param str document_id: Optional document id used to identify the document. + :param str encoder: Optional JSON encoder object. + :param str decoder: Optional JSON decoder object. """ - def __init__(self, database, document_id=None): + def __init__(self, database, document_id=None, **kwargs): super(Document, self).__init__() self._client = database.client self._database = database @@ -63,7 +65,8 @@ def __init__(self, database, document_id=None): self._document_id = document_id if self._document_id is not None: self['_id'] = self._document_id - self.encoder = self._client.encoder + self.encoder = kwargs.get('encoder') or self._client.encoder + self.decoder = kwargs.get('decoder') or json.JSONDecoder @property def r_session(self): @@ -165,7 +168,7 @@ def fetch(self): resp = self.r_session.get(self.document_url) resp.raise_for_status() self.clear() - self.update(resp.json()) + self.update(resp.json(cls=self.decoder)) def save(self): """ diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index dd443ccf..3c7b5afc 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -30,6 +30,8 @@ import uuid import inspect +from datetime import datetime + from cloudant.document import Document from cloudant.error import CloudantDocumentException @@ -859,5 +861,47 @@ def test_document_request_fails_after_client_disconnects(self): finally: self.client.connect() + def test_document_custom_json_encoder_and_decoder(self): + dt_format = '%Y-%m-%dT%H:%M:%S' + + class DTEncoder(json.JSONEncoder): + + def default(self, obj): + if isinstance(obj, datetime): + return { + '_type': 'datetime', + 'value': obj.strftime(dt_format) + } + return super(DTEncoder, self).default(obj) + + class DTDecoder(json.JSONDecoder): + + def __init__(self, *args, **kwargs): + json.JSONDecoder.__init__(self, object_hook=self.object_hook, + *args, **kwargs) + + def object_hook(self, obj): + if '_type' not in obj: + return obj + if obj['_type'] == 'datetime': + return datetime.strptime(obj['value'], dt_format) + return obj + + doc = Document(self.db, encoder=DTEncoder) + doc['name'] = 'julia' + doc['dt'] = datetime(2018, 7, 9, 15, 11, 10, 0) + doc.save() + + raw_doc = self.db.all_docs(include_docs=True)['rows'][0]['doc'] + + self.assertEquals(raw_doc['name'], 'julia') + self.assertEquals(raw_doc['dt']['_type'], 'datetime') + self.assertEquals(raw_doc['dt']['value'], '2018-07-09T15:11:10') + + doc2 = Document(self.db, doc['_id'], decoder=DTDecoder) + doc2.fetch() + + self.assertEquals(doc2['dt'], doc['dt']) + if __name__ == '__main__': unittest.main() From 9bd77189a47527ac8f7d157ba88c90c552c7be87 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Wed, 18 Jul 2018 13:33:00 -0400 Subject: [PATCH 05/97] Added connect parameter to CouchDB client example Issue #374 --- docs/getting_started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index caf0b1ab..a1cd43f7 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -38,7 +38,7 @@ Connecting with a client # Use CouchDB to create a CouchDB client # from cloudant.client import CouchDB - # client = CouchDB(USERNAME, PASSWORD, url='http://127.0.0.1:5984') + # client = CouchDB(USERNAME, PASSWORD, url='http://127.0.0.1:5984', connect=True) # Use Cloudant to create a Cloudant client using account from cloudant.client import Cloudant From a01e95ae7574fdbdf4e3c96ab4330e3ef668ab21 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Mon, 13 Aug 2018 18:41:19 -0400 Subject: [PATCH 06/97] Fixed pylint error messages - useless-object-inheritance warning message added to the disable list in pylintrc --- pylintrc | 3 ++- src/cloudant/_common_util.py | 4 ++-- src/cloudant/client.py | 2 +- src/cloudant/database.py | 4 ++-- src/cloudant/document.py | 14 ++++++-------- src/cloudant/feed.py | 3 +-- src/cloudant/index.py | 2 -- src/cloudant/result.py | 10 +++++----- src/cloudant/scheduler.py | 8 ++++---- 9 files changed, 23 insertions(+), 27 deletions(-) diff --git a/pylintrc b/pylintrc index 4cfbabaf..0b4f402d 100644 --- a/pylintrc +++ b/pylintrc @@ -66,7 +66,8 @@ confidence= # Disable "redefined-variable-type" refactor warning messages # Disable "too-many-..." and "too-few-..." refactor warning messages # Disable "locally-disabled" message -disable=R0204,R0901,R0902,R0903,R0904,R0913,R0914,R0915,locally-disabled,keyword-arg-before-vararg +# Disable Python 3 "useless-object-inheritance" message +disable=R0204,R0901,R0902,R0903,R0904,R0913,R0914,R0915,locally-disabled,keyword-arg-before-vararg,useless-object-inheritance [REPORTS] diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 6d9cca61..2af49b34 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -147,7 +147,7 @@ def feed_arg_types(feed_type): """ if feed_type == 'Cloudant': return _DB_UPDATES_ARG_TYPES - elif feed_type == 'CouchDB': + if feed_type == 'CouchDB': return _COUCH_DB_UPDATES_ARG_TYPES return _CHANGES_ARG_TYPES @@ -200,7 +200,7 @@ def _py_to_couch_translate(key, val): try: if key in ['keys', 'endkey_docid', 'startkey_docid', 'stale', 'update']: return {key: val} - elif val is None: + if val is None: return {key: None} arg_converter = TYPE_CONVERTERS.get(type(val)) return {key: arg_converter(val)} diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 6c9b5f33..06a6092c 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -387,9 +387,9 @@ def __getitem__(self, key): db = self._DATABASE_CLASS(self, key) if db.exists(): super(CouchDB, self).__setitem__(key, db) - return db else: raise KeyError(key) + return db def __delitem__(self, key, remote=False): """ diff --git a/src/cloudant/database.py b/src/cloudant/database.py index 85cb0313..ec6c8b7a 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -337,7 +337,7 @@ def get_view_result(self, ddoc_id, view_name, raw_result=False, **kwargs): view = View(DesignDocument(self, ddoc_id), view_name) if raw_result: return view(**kwargs) - elif kwargs: + if kwargs: return Result(view, **kwargs) return view.result @@ -613,9 +613,9 @@ def __getitem__(self, key): if doc.exists(): doc.fetch() super(CouchDatabase, self).__setitem__(key, doc) - return doc else: raise KeyError(key) + return doc def __contains__(self, key): """ diff --git a/src/cloudant/document.py b/src/cloudant/document.py index 7feda18a..ac08ee65 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -112,10 +112,10 @@ def exists(self): """ if self._document_id is None: return False - else: - resp = self.r_session.head(self.document_url) - if resp.status_code not in [200, 404]: - resp.raise_for_status() + + resp = self.r_session.head(self.document_url) + if resp.status_code not in [200, 404]: + resp.raise_for_status() return resp.status_code == 200 @@ -154,7 +154,6 @@ def create(self): self._document_id = data['id'] super(Document, self).__setitem__('_id', data['id']) super(Document, self).__setitem__('_rev', data['rev']) - return def fetch(self): """ @@ -320,7 +319,6 @@ def delete(self): del_resp.raise_for_status() self.clear() self.__setitem__('_id', self._document_id) - return def __enter__(self): """ @@ -413,13 +411,13 @@ def get_attachment( attachment_type = 'binary' if write_to is not None: - if attachment_type == 'text' or attachment_type == 'json': + if attachment_type in ('text', 'json'): write_to.write(resp.text) else: write_to.write(resp.content) if attachment_type == 'text': return resp.text - elif attachment_type == 'json': + if attachment_type == 'json': return resp.json() return resp.content diff --git a/src/cloudant/feed.py b/src/cloudant/feed.py index 44fa3525..4a93fc16 100644 --- a/src/cloudant/feed.py +++ b/src/cloudant/feed.py @@ -164,8 +164,7 @@ def _process_data(self, line): skip = False if self._raw_data: return skip, line - else: - line = unicode_(line) + line = unicode_(line) if not line: if (self._options.get('heartbeat', False) and self._options.get('feed') in ('continuous', 'longpoll') and diff --git a/src/cloudant/index.py b/src/cloudant/index.py index 5f34b76b..3d0d49c3 100644 --- a/src/cloudant/index.py +++ b/src/cloudant/index.py @@ -145,7 +145,6 @@ def create(self): resp.raise_for_status() self._ddoc_id = resp.json()['id'] self._name = resp.json()['name'] - return def _def_check(self): """ @@ -168,7 +167,6 @@ def delete(self): url = '/'.join((self.index_url, ddoc_id, self._type, self._name)) resp = self._r_session.delete(url) resp.raise_for_status() - return class TextIndex(Index): """ diff --git a/src/cloudant/result.py b/src/cloudant/result.py index 230811f1..53718c7e 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -268,13 +268,13 @@ def _handle_result_by_idx_slice(self, idx_slice): start = idx_slice.start stop = idx_slice.stop data = None - if (start is not None and stop is not None and - start >= 0 and stop >= 0 and start < stop): + # start and stop cannot be None and both must be greater than 0 + if all(i is not None and i >= 0 for i in [start, stop]) and start < stop: if limit is not None: if start >= limit: # Result is out of range return dict() - elif stop > limit: + if stop > limit: # Ensure that slice does not extend past original limit return self._ref(skip=skip+start, limit=limit-start, **opts) data = self._ref(skip=skip+start, limit=stop-start, **opts) @@ -498,8 +498,8 @@ def __getitem__(self, arg): type_or_none(int, arg.start) and type_or_none(int, arg.stop))): return super(QueryResult, self).__getitem__(arg) - else: - raise ResultException(101, arg) + + raise ResultException(101, arg) def _parse_data(self, data): """ diff --git a/src/cloudant/scheduler.py b/src/cloudant/scheduler.py index 76cdc101..723635b2 100644 --- a/src/cloudant/scheduler.py +++ b/src/cloudant/scheduler.py @@ -41,9 +41,9 @@ def list_docs(self, limit=None, skip=None): :param skip: How many result to skip starting at the beginning, if ordered by document ID. """ params = dict() - if limit != None: + if limit is not None: params["limit"] = limit - if skip != None: + if skip is not None: params["skip"] = skip resp = self._r_session.get('/'.join([self._scheduler, 'docs']), params=params) resp.raise_for_status() @@ -72,9 +72,9 @@ def list_jobs(self, limit=None, skip=None): :param skip: How many result to skip starting at the beginning, if ordered by document ID. """ params = dict() - if limit != None: + if limit is not None: params["limit"] = limit - if skip != None: + if skip is not None: params["skip"] = skip resp = self._r_session.get('/'.join([self._scheduler, 'jobs']), params=params) resp.raise_for_status() From e1c4846c8e1acd66a79ef42a4011f2863425b1aa Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Tue, 4 Sep 2018 17:06:37 -0400 Subject: [PATCH 07/97] Added if statement to break from while loop when the result set is less than page size - Converted page_size to int once and removed any additional conversions --- CHANGES.md | 1 + src/cloudant/result.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3c3e925a..64b5f9c0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,7 @@ - [NEW] Add custom JSON encoder/decoder option to `Document` constructor. - [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. - [FIXED] Case where an exception was raised after successful retry when using `doc.update_field`. +- [FIXED] Removed unnecessary request when retrieving a Result collection that is less than the 'page_size' value # 2.9.0 (2018-06-13) diff --git a/src/cloudant/result.py b/src/cloudant/result.py index 53718c7e..302b7128 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -341,7 +341,8 @@ def __iter__(self): raise ResultException(103, invalid_options, self.options) try: - if int(self._page_size) <= 0: + self._page_size = int(self._page_size) + if self._page_size <= 0: raise ResultException(104, self._page_size) except ValueError: raise ResultException(104, self._page_size) @@ -349,15 +350,17 @@ def __iter__(self): skip = 0 while True: response = self._ref( - limit=int(self._page_size), + limit=self._page_size, skip=skip, **self.options ) result = self._parse_data(response) - skip += int(self._page_size) + skip += self._page_size if result: for row in result: yield row + if len(result) < self._page_size: + break del result else: break From 03385dc195d554a86dc2e143514ab59fd0c4b96e Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 11 Sep 2018 16:06:57 +0100 Subject: [PATCH 08/97] Allow query parameters to be passed to custom changes filters --- CHANGES.md | 1 + src/cloudant/_common_util.py | 4 ++++ src/cloudant/feed.py | 21 ++++++++++++++------- tests/unit/changes_tests.py | 18 ++++++++++++------ 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 64b5f9c0..5efcf9b6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ - [NEW] Add custom JSON encoder/decoder option to `Document` constructor. - [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. +- [NEW] Allow arbitrary query parameters to be passed to custom changes filters. - [FIXED] Case where an exception was raised after successful retry when using `doc.update_field`. - [FIXED] Removed unnecessary request when retrieving a Result collection that is less than the 'page_size' value diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 2af49b34..1904a0a4 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -47,6 +47,9 @@ # Argument Types +ANY_ARG = object() +ANY_TYPE = object() + RESULT_ARG_TYPES = { 'descending': (bool,), 'endkey': (int, LONGTYPE, STRTYPE, Sequence,), @@ -101,6 +104,7 @@ 'filter': (STRTYPE,), 'include_docs': (bool,), 'style': (STRTYPE,), + ANY_ARG: ANY_TYPE # pass arbitrary query parameters to a custom filter } _CHANGES_ARG_TYPES.update(_DB_UPDATES_ARG_TYPES) diff --git a/src/cloudant/feed.py b/src/cloudant/feed.py index 4a93fc16..b02b79d6 100644 --- a/src/cloudant/feed.py +++ b/src/cloudant/feed.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2016 IBM. All rights reserved. +# Copyright (c) 2015, 2018 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ from ._2to3 import iteritems_, next_, unicode_, STRTYPE, NONETYPE from .error import CloudantArgumentError, CloudantFeedException -from ._common_util import feed_arg_types, TYPE_CONVERTERS +from ._common_util import ANY_ARG, ANY_TYPE, feed_arg_types, TYPE_CONVERTERS class Feed(object): """ @@ -111,11 +111,18 @@ def _validate(self, key, val, arg_types): Ensures that the key and the value are valid arguments to be used with the feed. """ - if key not in arg_types: - raise CloudantArgumentError(116, key) - if (not isinstance(val, arg_types[key]) or - (isinstance(val, bool) and int in arg_types[key])): - raise CloudantArgumentError(117, key, arg_types[key]) + if key in arg_types: + arg_type = arg_types[key] + else: + if ANY_ARG not in arg_types: + raise CloudantArgumentError(116, key) + arg_type = arg_types[ANY_ARG] + + if arg_type == ANY_TYPE: + return + if (not isinstance(val, arg_type) or + (isinstance(val, bool) and int in arg_type)): + raise CloudantArgumentError(117, key, arg_type) if isinstance(val, int) and val < 0 and not isinstance(val, bool): raise CloudantArgumentError(118, key, val) if key == 'feed': diff --git a/tests/unit/changes_tests.py b/tests/unit/changes_tests.py index ee3dce13..007e07a6 100644 --- a/tests/unit/changes_tests.py +++ b/tests/unit/changes_tests.py @@ -465,14 +465,20 @@ def test_get_feed_using_doc_ids(self): self.assertSetEqual(set([x['id'] for x in changes]), expected) self.assertTrue(str(feed.last_seq).startswith('100')) - def test_invalid_argument(self): + def test_get_feed_with_custom_filter_query_params(self): """ - Test that an invalid argument is caught and an exception is raised + Test using feed with custom filter query parameters. """ - feed = Feed(self.db, foo='bar') - with self.assertRaises(CloudantArgumentError) as cm: - invalid_feed = [x for x in feed] - self.assertEqual(str(cm.exception), 'Invalid argument foo') + feed = Feed( + self.db, + filter='mailbox/new_mail', + foo='bar', # query parameters to a custom filter + include_docs=False + ) + params = feed._translate(feed._options) + self.assertEquals(params['filter'], 'mailbox/new_mail') + self.assertEquals(params['foo'], 'bar') + self.assertEquals(params['include_docs'], 'false') def test_invalid_argument_type(self): """ From c74b7d8e0e37ba6b64fd4f9657157b220c550455 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 13 Sep 2018 11:10:22 +0100 Subject: [PATCH 09/97] Use json.dumps as default type converter --- src/cloudant/feed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cloudant/feed.py b/src/cloudant/feed.py index b02b79d6..ef2c90fc 100644 --- a/src/cloudant/feed.py +++ b/src/cloudant/feed.py @@ -100,7 +100,7 @@ def _translate(self, options): if isinstance(val, STRTYPE): translation[key] = val elif not isinstance(val, NONETYPE): - arg_converter = TYPE_CONVERTERS.get(type(val)) + arg_converter = TYPE_CONVERTERS.get(type(val), json.dumps) translation[key] = arg_converter(val) except Exception as ex: raise CloudantArgumentError(115, key, ex) From c28a4b5fe1fe806c2ce60d06b4d2b048c15cf579 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 19 Sep 2018 09:47:37 +0100 Subject: [PATCH 10/97] Prepare for version 2.10.0 release --- CHANGES.md | 4 ++-- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5efcf9b6..e1ddefd2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,10 +1,10 @@ -# Unreleased +# 2.10.0 (2018-09-19) - [NEW] Add custom JSON encoder/decoder option to `Document` constructor. - [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. - [NEW] Allow arbitrary query parameters to be passed to custom changes filters. - [FIXED] Case where an exception was raised after successful retry when using `doc.update_field`. -- [FIXED] Removed unnecessary request when retrieving a Result collection that is less than the 'page_size' value +- [FIXED] Removed unnecessary request when retrieving a Result collection that is less than the `page_size` value. # 2.9.0 (2018-06-13) diff --git a/VERSION b/VERSION index 43870570..10c2c0c3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.9.1-SNAPSHOT +2.10.0 diff --git a/docs/conf.py b/docs/conf.py index 21304f8e..688375cc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.9.1-SNAPSHOT' +version = '2.10.0' # The full version, including alpha/beta/rc tags. -release = '2.9.1-SNAPSHOT' +release = '2.10.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index e34873a0..04dd3a06 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.9.1-SNAPSHOT' +__version__ = '2.10.0' # pylint: disable=wrong-import-position import contextlib From a54349d712d12cd2c8cafdb1937736f5533afada Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 20 Sep 2018 16:52:52 +0100 Subject: [PATCH 11/97] Update version to 2.11.0-SNAPSHOT --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 10c2c0c3..d33489b3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.10.0 +2.11.0-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index 688375cc..72e556ae 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.10.0' +version = '2.11.0-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.10.0' +release = '2.11.0-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 04dd3a06..ffb150c2 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.10.0' +__version__ = '2.11.0-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From 48b10c720f28d7a8eda4cbb2239c7c0da214b4a3 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 4 Oct 2018 15:08:07 +0100 Subject: [PATCH 12/97] Added test attributes Tagged tests that require a real database: * database name `cloudant` or `couch` * couchapi for tests that only run on a specific api level Added test filtering by attributes. Removed test skips now controlled by attributes. Added test attribute information to contributing. --- .travis.yml | 2 +- CONTRIBUTING.md | 9 +++++++ Jenkinsfile | 2 +- tests/unit/auth_renewal_tests.py | 9 ++++--- tests/unit/changes_tests.py | 16 +++++------ tests/unit/client_tests.py | 34 +++++++++++------------ tests/unit/database_tests.py | 39 +++++++++++---------------- tests/unit/db_updates_tests.py | 16 +++++------ tests/unit/design_document_tests.py | 36 +++++++++---------------- tests/unit/document_tests.py | 14 +++++----- tests/unit/index_tests.py | 27 +++++++++---------- tests/unit/infinite_feed_tests.py | 24 ++++++++--------- tests/unit/query_result_tests.py | 7 ++--- tests/unit/query_tests.py | 14 +++++----- tests/unit/replicator_tests.py | 16 ++++++----- tests/unit/result_tests.py | 8 +++--- tests/unit/security_document_tests.py | 8 +++--- tests/unit/view_execution_tests.py | 4 +++ tests/unit/view_tests.py | 12 +++++---- 19 files changed, 149 insertions(+), 148 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5fabac04..8a88780e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ before_script: # command to run tests script: - pylint ./src/cloudant - - nosetests -w ./tests/unit + - nosetests -A 'not db or ((db is "couch" or "couch" in db) and (not couchapi or couchapi <='${COUCHDB_VERSION:0:1}'))' -w ./tests/unit notifications: email: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 21890316..7f9e5300 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,3 +82,12 @@ test behaviour: This user is deleted at the end of the test run, but beware it'll break other applications using the CouchDB instance that rely on admin party mode being in effect while the tests are running. + +### Test attributes + +Database tests also have node attributes. Currently there are these attributes: +`db` - `cloudant` and/or `couch` +`couchapi` - Apache CouchDB major version number (i.e. API level) e.g. `2` + +Example to run database tests that require CouchDB version 1 API and no Cloudant features: +`nosetests -A 'db and ((db is "couch" or "couch" in db) and (not couchapi or couchapi <=1))' -w ./tests/unit` diff --git a/Jenkinsfile b/Jenkinsfile index f1c24d9c..a806771f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,7 +37,7 @@ def setupPythonAndTest(pythonVersion, testSuite) { pip install -r requirements.txt pip install -r test-requirements.txt pylint ./src/cloudant - nosetests -w ./tests/unit --with-xunit + nosetests -A 'not db or (db is "cloudant" or "cloudant" in db)' -w ./tests/unit --with-xunit """ } finally { // Load the test results diff --git a/tests/unit/auth_renewal_tests.py b/tests/unit/auth_renewal_tests.py index b5fb48b6..e32f7459 100644 --- a/tests/unit/auth_renewal_tests.py +++ b/tests/unit/auth_renewal_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016, 2017 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,15 +18,18 @@ See configuration options for environment variables in unit_t_db_base module docstring. """ -import unittest import os -import requests import time +import unittest +import requests from cloudant._client_session import CookieSession +from nose.plugins.attrib import attr from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase + +@attr(db=['cloudant','couch']) @unittest.skipIf(os.environ.get('ADMIN_PARTY') == 'true', 'Skipping - Admin Party mode') class AuthRenewalTests(UnitTestDbBase): """ diff --git a/tests/unit/changes_tests.py b/tests/unit/changes_tests.py index 007e07a6..d39ab159 100644 --- a/tests/unit/changes_tests.py +++ b/tests/unit/changes_tests.py @@ -16,20 +16,23 @@ Unit tests for _changes feed """ -import unittest -from requests import Session import json import os +import unittest -from cloudant.feed import Feed -from cloudant.document import Document +from cloudant._2to3 import unicode_ from cloudant.design_document import DesignDocument +from cloudant.document import Document from cloudant.error import CloudantArgumentError -from cloudant._2to3 import unicode_ +from cloudant.feed import Feed +from nose.plugins.attrib import attr +from requests import Session from .unit_t_db_base import UnitTestDbBase from .. import BYTETYPE + +@attr(db=['cloudant','couch']) class ChangesTests(UnitTestDbBase): """ _changes feed unit tests @@ -41,7 +44,6 @@ def setUp(self): """ super(ChangesTests, self).setUp() self.db_set_up() - self.cloudant_test = os.environ.get('RUN_CLOUDANT_TESTS') is not None def tearDown(self): """ @@ -448,8 +450,6 @@ def test_get_feed_using_conflicts_false(self): self.assertSetEqual(set([x['id'] for x in changes]), expected) self.assertTrue(str(feed.last_seq).startswith('3')) - @unittest.skipIf(os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping since _doc_ids filter is not supported on all Cloudant clusters') def test_get_feed_using_doc_ids(self): """ Test getting content back for a feed using doc_ids diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index a65e729f..e110e5fa 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2017 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,28 +20,29 @@ """ -import unittest -import requests -import json import base64 -import sys -import os import datetime -import mock - -from requests import ConnectTimeout, HTTPError +import json +import os +import sys +import unittest from time import sleep +import mock +import requests from cloudant import cloudant, cloudant_bluemix, couchdb, couchdb_admin_party -from cloudant.client import Cloudant, CouchDB from cloudant._client_session import BasicSession, CookieSession +from cloudant.client import Cloudant, CouchDB from cloudant.database import CloudantDatabase from cloudant.error import CloudantArgumentError, CloudantClientException from cloudant.feed import Feed, InfiniteFeed +from nose.plugins.attrib import attr +from requests import ConnectTimeout, HTTPError from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase from .. import bytes_, str_ + class CloudantClientExceptionTests(unittest.TestCase): """ Ensure CloudantClientException functions as expected. @@ -86,10 +87,10 @@ class ClientTests(UnitTestDbBase): """ @unittest.skipIf( - (os.environ.get('RUN_CLOUDANT_TESTS') is not None or - (os.environ.get('ADMIN_PARTY') and os.environ.get('ADMIN_PARTY') == 'true')), + ((os.environ.get('ADMIN_PARTY') and os.environ.get('ADMIN_PARTY') == 'true')), 'Skipping couchdb context manager test' ) + @attr(db='couch') def test_couchdb_context_helper(self): """ Test that the couchdb context helper works as expected. @@ -102,10 +103,10 @@ def test_couchdb_context_helper(self): self.fail('Exception {0} was raised.'.format(str(err))) @unittest.skipUnless( - (os.environ.get('RUN_CLOUDANT_TESTS') is None and - (os.environ.get('ADMIN_PARTY') and os.environ.get('ADMIN_PARTY') == 'true')), + ((os.environ.get('ADMIN_PARTY') and os.environ.get('ADMIN_PARTY') == 'true')), 'Skipping couchdb_admin_party context manager test' ) + @attr(db='couch') def test_couchdb_admin_party_context_helper(self): """ Test that the couchdb_admin_party context helper works as expected. @@ -599,10 +600,7 @@ def test_db_updates_feed_call(self): finally: self.client.disconnect() -@unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant client specific tests' -) +@attr(db='cloudant') class CloudantClientTests(UnitTestDbBase): """ Cloudant specific client unit tests diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index 517736bc..5881a92d 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -22,25 +22,27 @@ """ -import unittest -import mock -import requests import os +import unittest import uuid +import mock +import requests from cloudant._2to3 import UNICHR -from cloudant.result import Result, QueryResult -from cloudant.error import CloudantArgumentError, CloudantDatabaseException -from cloudant.document import Document from cloudant.design_document import DesignDocument -from cloudant.security_document import SecurityDocument -from cloudant.index import Index, TextIndex, SpecialIndex +from cloudant.document import Document +from cloudant.error import CloudantArgumentError, CloudantDatabaseException from cloudant.feed import Feed, InfiniteFeed -from tests.unit._test_util import LONG_NUMBER +from cloudant.index import Index, TextIndex, SpecialIndex +from cloudant.result import Result, QueryResult +from cloudant.security_document import SecurityDocument +from nose.plugins.attrib import attr +from tests.unit._test_util import LONG_NUMBER from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase, skip_if_iam from .. import unicode_ + class CloudantDatabaseExceptionTests(unittest.TestCase): """ Ensure CloudantDatabaseException functions as expected. @@ -79,6 +81,7 @@ def test_raise_with_proper_code_and_args(self): raise CloudantDatabaseException(400, 'foo') self.assertEqual(cm.exception.status_code, 400) +@attr(db=['cloudant','couch']) class DatabaseTests(UnitTestDbBase): """ CouchDatabase/CloudantDatabase unit tests @@ -780,8 +783,7 @@ def test_get_set_revision_limit(self): self.assertNotEqual(new_limit, limit) self.assertEqual(new_limit, 1234) - @unittest.skipIf(os.environ.get('RUN_CLOUDANT_TESTS'), - 'Skipping since view cleanup is automatic in Cloudant.') + @attr(db='couch') def test_view_clean_up(self): """ Test cleaning up old view files @@ -968,10 +970,7 @@ def test_database_request_fails_after_client_disconnects(self): finally: self.client.connect() - @unittest.skipIf(not os.environ.get('RUN_CLOUDANT_TESTS') or - (os.environ.get('COUCHDB_VERSION') and - os.environ.get('COUCHDB_VERSION').startswith('1')), - 'Skipping test_create_json_index test') + @attr(couchapi=2) def test_create_json_index(self): """ Ensure that a JSON index is created as expected. @@ -995,10 +994,7 @@ def test_create_json_index(self): self.assertEquals(index['options']['def']['fields'], ['name', 'age']) self.assertEquals(index['reduce'], '_count') - @unittest.skipIf(not os.environ.get('RUN_CLOUDANT_TESTS') or - (os.environ.get('COUCHDB_VERSION') and - os.environ.get('COUCHDB_VERSION').startswith('1')), - 'Skipping test_create_json_index test') + @attr(couchapi=2) def test_delete_json_index(self): """ Ensure that a JSON index is deleted as expected. @@ -1013,10 +1009,7 @@ def test_delete_json_index(self): self.db.delete_query_index('ddoc001', 'json', 'index001') self.assertFalse(ddoc.exists()) -@unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant specific Database tests' -) +@attr(db='cloudant') class CloudantDatabaseTests(UnitTestDbBase): """ Cloudant specific Database unit tests diff --git a/tests/unit/db_updates_tests.py b/tests/unit/db_updates_tests.py index 7cb8c956..0c83e75f 100644 --- a/tests/unit/db_updates_tests.py +++ b/tests/unit/db_updates_tests.py @@ -16,14 +16,15 @@ Unit tests for _db_updates feed """ -import unittest -from requests import Session import json import os +import unittest -from cloudant.feed import Feed -from cloudant.error import CloudantArgumentError from cloudant._2to3 import unicode_ +from cloudant.error import CloudantArgumentError +from cloudant.feed import Feed +from nose.plugins.attrib import attr +from requests import Session from .unit_t_db_base import UnitTestDbBase from .. import BYTETYPE @@ -104,8 +105,7 @@ def assert_changes_in_db_updates_feed(self, changes): self.assertDictEqual( changes[2], {'db_name': self.new_dbs[2].database_name, 'type': 'created'}) -@unittest.skipIf(os.environ.get('RUN_CLOUDANT_TESTS'), - 'Skipping CouchDB _db_updates feed tests') +@attr(db='couch') class CouchDbUpdatesTests(DbUpdatesTestsBase): """ CouchDB _db_updates feed unit tests @@ -276,8 +276,8 @@ def test_invalid_feed_value(self): self.assertTrue(str(cm.exception).startswith( 'Invalid value (normal) for feed option.')) -@unittest.skipIf(not os.environ.get('RUN_CLOUDANT_TESTS') or - os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') +@attr(db='cloudant') +@unittest.skipIf(os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') class CloudantDbUpdatesTests(DbUpdatesTestsBase): """ Cloudant _db_updates feed unit tests diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index 6cb14281..43714727 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,16 +24,18 @@ import json import os import unittest + import mock import requests - -from cloudant.document import Document from cloudant.design_document import DesignDocument -from cloudant.view import View, QueryIndexView +from cloudant.document import Document from cloudant.error import CloudantArgumentError, CloudantDesignDocumentException +from cloudant.view import View, QueryIndexView +from nose.plugins.attrib import attr from .unit_t_db_base import UnitTestDbBase, skip_if_iam + class CloudantDesignDocumentExceptionTests(unittest.TestCase): """ Ensure CloudantDesignDocumentException functions as expected. @@ -72,6 +74,7 @@ def test_raise_with_proper_code_and_args(self): raise CloudantDesignDocumentException(104, 'foo') self.assertEqual(cm.exception.status_code, 104) +@attr(db=['cloudant','couch']) class DesignDocumentTests(UnitTestDbBase): """ DesignDocument unit tests @@ -360,10 +363,7 @@ def test_fetch_map_reduce(self): self.assertIsInstance(ddoc_remote['views']['view001'], View) self.assertIsInstance(ddoc_remote['views']['view003'], View) - @unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant fetch dbcopy test' - ) + @attr(db='cloudant') def test_fetch_dbcopy(self): """ Ensure that the document fetch from the database returns the @@ -810,10 +810,7 @@ def test_get_info_raises_httperror(self): self.client.r_session.get.assert_called_with( '/'.join([ddoc.document_url, '_info'])) - @unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant _search_info endpoint test' - ) + @attr(db='cloudant') def test_get_search_info(self): """ Test retrieval of search_info endpoint from the DesignDocument. @@ -841,10 +838,7 @@ def test_get_search_info(self): self.assertTrue(search_index_metadata['pending_seq'] <= 101, 'The pending_seq should be 101 or fewer.') self.assertTrue(search_index_metadata['disk_size'] >0, 'The disk_size should be greater than 0.') - @unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant _search_disk_size endpoint test' - ) + @attr(db='cloudant') def test_get_search_disk_size(self): """ Test retrieval of search_disk_size endpoint from the DesignDocument. @@ -881,10 +875,7 @@ def test_get_search_disk_size(self): search_disk_size['search_index']['disk_size'] > 0, 'The "disk_size" should be greater than 0.') - @unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant _search_info raises HTTPError test' - ) + @attr(db='cloudant') def test_get_search_info_raises_httperror(self): """ Test get_search_info raises an HTTPError. @@ -1525,10 +1516,7 @@ def test_get_list_function(self): 'html += \'\'; return html; }); }' ) - @unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant specific Cloudant Geo tests' - ) + @attr(db='cloudant') def test_geospatial_index(self): """ Test retrieval and query of Cloudant Geo indexes from the DesignDocument. diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index 3c7b5afc..3676c2b1 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -22,21 +22,22 @@ """ -import unittest -import mock +import inspect import json -import requests import os +import unittest import uuid -import inspect - from datetime import datetime +import mock +import requests from cloudant.document import Document from cloudant.error import CloudantDocumentException +from nose.plugins.attrib import attr -from .. import StringIO, unicode_ from .unit_t_db_base import UnitTestDbBase +from .. import StringIO, unicode_ + def find_fixture(name): import tests.unit.fixtures as fixtures @@ -82,6 +83,7 @@ def test_raise_with_proper_code_and_args(self): raise CloudantDocumentException(102, 'foo') self.assertEqual(cm.exception.status_code, 102) +@attr(db=['cloudant','couch']) class DocumentTests(UnitTestDbBase): """ Document unit tests diff --git a/tests/unit/index_tests.py b/tests/unit/index_tests.py index 98dbf096..59a952d6 100644 --- a/tests/unit/index_tests.py +++ b/tests/unit/index_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,20 +22,22 @@ """ from __future__ import absolute_import +import os import unittest + import mock -import os import requests - -from cloudant.index import Index, TextIndex, SpecialIndex -from cloudant.query import Query -from cloudant.view import QueryIndexView from cloudant.design_document import DesignDocument from cloudant.document import Document from cloudant.error import CloudantArgumentError, CloudantIndexException +from cloudant.index import Index, TextIndex, SpecialIndex +from cloudant.query import Query +from cloudant.view import QueryIndexView +from nose.plugins.attrib import attr -from .. import PY2 from .unit_t_db_base import UnitTestDbBase +from .. import PY2 + class CloudantIndexExceptionTests(unittest.TestCase): """ @@ -66,10 +68,8 @@ def test_raise_with_proper_code_and_args(self): raise CloudantIndexException(101) self.assertEqual(cm.exception.status_code, 101) -@unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant Index tests' - ) +@attr(db=['cloudant','couch']) +@attr(couchapi=2) class IndexTests(UnitTestDbBase): """ Index unit tests @@ -392,10 +392,7 @@ def test_index_usage_via_query(self): selector={'age': {'$eq': 6}}, raw_result=True) self.assertTrue(str(result['warning']).startswith("no matching index found")) -@unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant Text Index tests' - ) +@attr(db='cloudant') class TextIndexTests(UnitTestDbBase): """ Search Index unit tests diff --git a/tests/unit/infinite_feed_tests.py b/tests/unit/infinite_feed_tests.py index 544aca60..7891cb91 100644 --- a/tests/unit/infinite_feed_tests.py +++ b/tests/unit/infinite_feed_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,18 +16,18 @@ feed module - Unit tests for Feed class """ -import unittest -from requests import Session -import json import os +import unittest from time import sleep -from cloudant.feed import InfiniteFeed, Feed -from cloudant.client import CouchDB from cloudant.error import CloudantArgumentError, CloudantFeedException +from cloudant.feed import InfiniteFeed, Feed +from nose.plugins.attrib import attr +from requests import Session from .unit_t_db_base import UnitTestDbBase + class MethodCallCount(object): """ This callable class is used as a proxy by the infinite feed tests to wrap @@ -71,6 +71,7 @@ def test_raise_with_proper_code_and_args(self): raise CloudantFeedException(101) self.assertEqual(cm.exception.status_code, 101) +@attr(db=['cloudant','couch']) class InfiniteFeedTests(UnitTestDbBase): """ Infinite Feed unit tests @@ -126,8 +127,7 @@ def test_constructor_with_invalid_feed_option(self): 'Invalid infinite feed option: longpoll. Must be set to continuous.' ) - @unittest.skipIf(os.environ.get('RUN_CLOUDANT_TESTS'), - 'Skipping since test is possible only when testing against CouchDB.') + @attr(db='couch') def test_invalid_source_couchdb(self): """ Ensure that a CouchDB client cannot be used with an infinite feed. @@ -137,8 +137,8 @@ def test_invalid_source_couchdb(self): self.assertEqual(str(cm.exception), 'Infinite _db_updates feed not supported for CouchDB.') - @unittest.skipIf(not os.environ.get('RUN_CLOUDANT_TESTS') or - os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') + @unittest.skipIf(os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') + @attr(db='cloudant') def test_constructor_db_updates(self): """ Test constructing an infinite _db_updates feed. @@ -185,8 +185,8 @@ def test_infinite_feed(self): # the continuous feed was started/restarted 3 separate times. self.assertEqual(feed._start.called_count, 3) - @unittest.skipIf(not os.environ.get('RUN_CLOUDANT_TESTS') or - os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') + @unittest.skipIf(os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') + @attr(db='cloudant') def test_infinite_db_updates_feed(self): """ Test that an _db_updates infinite feed will continue to issue multiple diff --git a/tests/unit/query_result_tests.py b/tests/unit/query_result_tests.py index d290f930..9f4fd170 100644 --- a/tests/unit/query_result_tests.py +++ b/tests/unit/query_result_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,11 +26,12 @@ from cloudant.query import Query from cloudant.result import QueryResult from cloudant.error import ResultException +from nose.plugins.attrib import attr from .unit_t_db_base import UnitTestDbBase -@unittest.skipUnless(os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant QueryResult tests') +@attr(db=['cloudant','couch']) +@attr(couchapi=2) class QueryResultTests(UnitTestDbBase): """ QueryResult unit tests diff --git a/tests/unit/query_tests.py b/tests/unit/query_tests.py index 72fd35f1..30f78bcd 100644 --- a/tests/unit/query_tests.py +++ b/tests/unit/query_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,19 +20,19 @@ """ -import unittest import os +import unittest +from cloudant.error import CloudantArgumentError from cloudant.query import Query from cloudant.result import QueryResult -from cloudant.error import CloudantArgumentError +from nose.plugins.attrib import attr from .unit_t_db_base import UnitTestDbBase -@unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant Query tests' - ) + +@attr(db=['cloudant','couch']) +@attr(couchapi=2) class QueryTests(UnitTestDbBase): """ Query unit tests diff --git a/tests/unit/replicator_tests.py b/tests/unit/replicator_tests.py index 538adbd2..610d3588 100644 --- a/tests/unit/replicator_tests.py +++ b/tests/unit/replicator_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,21 +22,22 @@ """ +import time import unittest import uuid -import time -from flaky import flaky import requests -from requests import ConnectionError - -from cloudant.replicator import Replicator from cloudant.document import Document from cloudant.error import CloudantReplicatorException, CloudantClientException +from cloudant.replicator import Replicator +from flaky import flaky +from nose.plugins.attrib import attr +from requests import ConnectionError -from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase +from .unit_t_db_base import UnitTestDbBase from .. import unicode_ + class CloudantReplicatorExceptionTests(unittest.TestCase): """ Ensure CloudantReplicatorException functions as expected. @@ -75,6 +76,7 @@ def test_raise_with_proper_code_and_args(self): raise CloudantReplicatorException(404, 'foo') self.assertEqual(cm.exception.status_code, 404) +@attr(db=['cloudant','couch']) class ReplicatorTests(UnitTestDbBase): """ Replicator unit tests diff --git a/tests/unit/result_tests.py b/tests/unit/result_tests.py index a7ede996..f56387ad 100644 --- a/tests/unit/result_tests.py +++ b/tests/unit/result_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,14 +16,15 @@ result module - Unit tests for Result class """ import unittest -import os -from requests.exceptions import HTTPError from cloudant.error import ResultException from cloudant.result import Result, ResultByKey +from nose.plugins.attrib import attr +from requests.exceptions import HTTPError from .unit_t_db_base import UnitTestDbBase + class ResultExceptionTests(unittest.TestCase): """ Ensure ResultException functions as expected. @@ -72,6 +73,7 @@ def test_raise_with_proper_code_and_args(self): raise ResultException(102, 'foo', 'bar') self.assertEqual(cm.exception.status_code, 102) +@attr(db=['cloudant','couch']) class ResultTests(UnitTestDbBase): """ Result unit tests diff --git a/tests/unit/security_document_tests.py b/tests/unit/security_document_tests.py index 4e8d1ed6..0e2ee024 100644 --- a/tests/unit/security_document_tests.py +++ b/tests/unit/security_document_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,16 +19,16 @@ module docstring. """ -import unittest -import requests import json -import os +import unittest from cloudant.security_document import SecurityDocument +from nose.plugins.attrib import attr from .unit_t_db_base import UnitTestDbBase +@attr(db=['cloudant','couch']) class SecurityDocumentTests(UnitTestDbBase): """ SecurityDocument unit tests diff --git a/tests/unit/view_execution_tests.py b/tests/unit/view_execution_tests.py index 4ac97dc5..45a082c6 100644 --- a/tests/unit/view_execution_tests.py +++ b/tests/unit/view_execution_tests.py @@ -17,8 +17,12 @@ """ import unittest +from nose.plugins.attrib import attr + from .unit_t_db_base import UnitTestDbBase + +@attr(db=['cloudant','couch']) class QueryParmExecutionTests(UnitTestDbBase): """ Test cases for the execution of views queries using translated parameters. diff --git a/tests/unit/view_tests.py b/tests/unit/view_tests.py index 28be9cd4..a362fa77 100644 --- a/tests/unit/view_tests.py +++ b/tests/unit/view_tests.py @@ -23,18 +23,19 @@ """ import unittest + import mock import requests -import os - -from cloudant.design_document import DesignDocument -from cloudant.view import View, QueryIndexView from cloudant._common_util import _Code -from cloudant.result import Result +from cloudant.design_document import DesignDocument from cloudant.error import CloudantArgumentError, CloudantViewException +from cloudant.result import Result +from cloudant.view import View, QueryIndexView +from nose.plugins.attrib import attr from .unit_t_db_base import UnitTestDbBase + class CodeTests(unittest.TestCase): """ _Code class unit test @@ -78,6 +79,7 @@ def test_raise_with_proper_code(self): raise CloudantViewException(101) self.assertEqual(cm.exception.status_code, 101) +@attr(db=['cloudant','couch']) class ViewTests(UnitTestDbBase): """ View class unit tests From cfe4ae174b3913ecb3391a8151b8b01b05118bc6 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Mon, 12 Nov 2018 14:12:14 +0000 Subject: [PATCH 13/97] Replaced usage of requests `response.json()` method Replaced `response.json()` calls with `_common_util.response_to_json_dict()`. Added CHANGES entry. Clarified types of JSON encoder and decoder in doc strings. Updated tests mocking `response.json()` with `response.text`. Added simplejson test axis. --- CHANGES.md | 6 + Jenkinsfile | 13 +- src/cloudant/_client_session.py | 11 +- src/cloudant/_common_util.py | 13 +- src/cloudant/client.py | 15 +- src/cloudant/database.py | 36 ++-- src/cloudant/design_document.py | 10 +- src/cloudant/document.py | 19 +- src/cloudant/index.py | 7 +- src/cloudant/query.py | 5 +- src/cloudant/scheduler.py | 8 +- src/cloudant/security_document.py | 5 +- src/cloudant/view.py | 6 +- tests/unit/client_tests.py | 5 +- tests/unit/database_tests.py | 3 +- tests/unit/design_document_tests.py | 13 +- tests/unit/iam_auth_tests.py | 53 +++--- tests/unit/scheduler_tests.py | 270 ++++++++++++++-------------- 18 files changed, 263 insertions(+), 235 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e1ddefd2..47e621f0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +# Unreleased + +- [FIXED] Unexpected keyword argument errors when using the library with the + `simplejson` module present in the environment caused by `requests` preferentially + loading it over the system `json` module. + # 2.10.0 (2018-09-19) - [NEW] Add custom JSON encoder/decoder option to `Document` constructor. diff --git a/Jenkinsfile b/Jenkinsfile index a806771f..1ab03a60 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,12 +10,13 @@ def getEnvForSuite(suiteName) { case 'basic': envVars.add("RUN_BASIC_AUTH_TESTS=1") break - case 'cookie': - break case 'iam': // Setting IAM_API_KEY forces tests to run using an IAM enabled client. envVars.add("IAM_API_KEY=$DB_IAM_API_KEY") break + case 'cookie': + case 'simplejson': + break default: error("Unknown test suite environment ${suiteName}") } @@ -36,6 +37,7 @@ def setupPythonAndTest(pythonVersion, testSuite) { . ./tmp/bin/activate pip install -r requirements.txt pip install -r test-requirements.txt + ${'simplejson'.equals(testSuite) ? 'pip install simplejson' : ''} pylint ./src/cloudant nosetests -A 'not db or (db is "cloudant" or "cloudant" in db)' -w ./tests/unit --with-xunit """ @@ -58,12 +60,15 @@ stage('Checkout'){ } stage('Test'){ - axes = [:] - ['2.7.12','3.5.2'].each { version -> + def py2 = '2.7.12' + def py3 = '3.5.2' + def axes = [:] + [py2, py3].each { version -> ['basic','cookie','iam'].each { auth -> axes.put("Python${version}-${auth}", {setupPythonAndTest(version, auth)}) } } + axes.put("Python${py3}-simplejson", {setupPythonAndTest(py3, 'simplejson')}) parallel(axes) } diff --git a/src/cloudant/_client_session.py b/src/cloudant/_client_session.py index 0e8fd386..820bc532 100644 --- a/src/cloudant/_client_session.py +++ b/src/cloudant/_client_session.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ from requests import RequestException, Session from ._2to3 import bytes_, unicode_, url_join +from ._common_util import response_to_json_dict from .error import CloudantException @@ -75,7 +76,7 @@ def info(self): resp = self.get(self._session_url) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def set_credentials(self, username, password): """ @@ -170,7 +171,7 @@ def request(self, method, url, **kwargs): is_expired = any(( resp.status_code == 403 and - resp.json().get('error') == 'credentials_expired', + response_to_json_dict(resp).get('error') == 'credentials_expired', resp.status_code == 401 )) @@ -282,10 +283,10 @@ def _get_access_token(self): 'apikey': self._api_key } ) - err = resp.json().get('errorMessage', err) + err = response_to_json_dict(resp).get('errorMessage', err) resp.raise_for_status() - return resp.json()['access_token'] + return response_to_json_dict(resp)['access_token'] except KeyError: raise CloudantException('Invalid response from IAM token service') diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 1904a0a4..7a373815 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -270,7 +270,7 @@ def append_response_error_content(response, **kwargs): """ if response.status_code >= 400: try: - resp_dict = response.json() + resp_dict = response_to_json_dict(response) error = resp_dict.get('error', '') reason = resp_dict.get('reason', '') # Append to the existing response's reason @@ -279,6 +279,17 @@ def append_response_error_content(response, **kwargs): pass return response +def response_to_json_dict(response, **kwargs): + """ + Standard place to convert responses to JSON. + + :param response: requests response object + :param **kwargs: arguments accepted by json.loads + + :returns: dict of JSON response + """ + return json.loads(response.text, **kwargs) + # Classes diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 06a6092c..9b22d0d7 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -35,6 +35,7 @@ USER_AGENT, append_response_error_content, CloudFoundryService, + response_to_json_dict, ) @@ -256,7 +257,7 @@ def all_dbs(self): url = '/'.join((self.server_url, '_all_dbs')) resp = self.r_session.get(url) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def create_database(self, dbname, **kwargs): """ @@ -345,7 +346,7 @@ def metadata(self): """ resp = self.r_session.get(self.server_url) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def keys(self, remote=False): """ @@ -622,7 +623,7 @@ def _usage_endpoint(self, endpoint, year=None, month=None): raise CloudantArgumentError(101, year, month) else: resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def bill(self, year=None, month=None): """ @@ -687,7 +688,7 @@ def shared_databases(self): self.server_url, '_api', 'v2', 'user', 'shared_databases')) resp = self.r_session.get(endpoint) resp.raise_for_status() - data = resp.json() + data = response_to_json_dict(resp) return data.get('shared_databases', []) def generate_api_key(self): @@ -699,7 +700,7 @@ def generate_api_key(self): endpoint = '/'.join((self.server_url, '_api', 'v2', 'api_keys')) resp = self.r_session.post(endpoint) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def cors_configuration(self): """ @@ -712,7 +713,7 @@ def cors_configuration(self): resp = self.r_session.get(endpoint) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def disable_cors(self): """ @@ -807,7 +808,7 @@ def _write_cors_configuration(self, config): ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) @classmethod def bluemix(cls, vcap_services, instance_name=None, service_name=None, **kwargs): diff --git a/src/cloudant/database.py b/src/cloudant/database.py index ec6c8b7a..51ff2430 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -26,7 +26,9 @@ SEARCH_INDEX_ARGS, SPECIAL_INDEX_TYPE, TEXT_INDEX_TYPE, - get_docs) + get_docs, + response_to_json_dict, + ) from .document import Document from .design_document import DesignDocument from .security_document import SecurityDocument @@ -123,7 +125,7 @@ def metadata(self): """ resp = self.r_session.get(self.database_url) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def doc_count(self): """ @@ -192,7 +194,7 @@ def design_documents(self): query = "startkey=\"_design\"&endkey=\"_design0\"&include_docs=true" resp = self.r_session.get(url, params=query) resp.raise_for_status() - data = resp.json() + data = response_to_json_dict(resp) return data['rows'] def list_design_documents(self): @@ -206,7 +208,7 @@ def list_design_documents(self): query = "startkey=\"_design\"&endkey=\"_design0\"" resp = self.r_session.get(url, params=query) resp.raise_for_status() - data = resp.json() + data = response_to_json_dict(resp) return [x.get('key') for x in data.get('rows', [])] def get_design_document(self, ddoc_id): @@ -403,7 +405,7 @@ def all_docs(self, **kwargs): '/'.join([self.database_url, '_all_docs']), self.client.encoder, **kwargs) - return resp.json() + return response_to_json_dict(resp) @contextlib.contextmanager def custom_result(self, **options): @@ -718,7 +720,7 @@ def bulk_docs(self, docs): headers=headers ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def missing_revisions(self, doc_id, *revisions): """ @@ -742,7 +744,7 @@ def missing_revisions(self, doc_id, *revisions): ) resp.raise_for_status() - resp_json = resp.json() + resp_json = response_to_json_dict(resp) missing_revs = resp_json['missing_revs'].get(doc_id) if missing_revs is None: missing_revs = [] @@ -771,7 +773,7 @@ def revisions_diff(self, doc_id, *revisions): ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def get_revision_limit(self): """ @@ -787,7 +789,7 @@ def get_revision_limit(self): try: ret = int(resp.text) except ValueError: - raise CloudantDatabaseException(400, resp.json()) + raise CloudantDatabaseException(400, response_to_json_dict(resp)) return ret @@ -806,7 +808,7 @@ def set_revision_limit(self, limit): resp = self.r_session.put(url, data=json.dumps(limit, cls=self.client.encoder)) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def view_cleanup(self): """ @@ -822,7 +824,7 @@ def view_cleanup(self): ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def get_list_function_result(self, ddoc_id, list_name, view_name, **kwargs): """ @@ -974,10 +976,10 @@ def get_query_indexes(self, raw_result=False): resp.raise_for_status() if raw_result: - return resp.json() + return response_to_json_dict(resp) indexes = [] - for data in resp.json().get('indexes', []): + for data in response_to_json_dict(resp).get('indexes', []): if data.get('type') == JSON_INDEX_TYPE: indexes.append(Index( self, @@ -1239,7 +1241,7 @@ def share_database(self, username, roles=None): headers={'Content-Type': 'application/json'} ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def unshare_database(self, username): """ @@ -1264,7 +1266,7 @@ def unshare_database(self, username): headers={'Content-Type': 'application/json'} ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def shards(self): """ @@ -1276,7 +1278,7 @@ def shards(self): resp = self.r_session.get(url) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def get_search_result(self, ddoc_id, index_name, **query_params): """ @@ -1399,4 +1401,4 @@ def get_search_result(self, ddoc_id, index_name, **query_params): data=json.dumps(query_params, cls=self.client.encoder) ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) diff --git a/src/cloudant/design_document.py b/src/cloudant/design_document.py index b32c5e33..66a9789d 100644 --- a/src/cloudant/design_document.py +++ b/src/cloudant/design_document.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ API module/class for interacting with a design document in a database. """ from ._2to3 import iteritems_, STRTYPE -from ._common_util import QUERY_LANGUAGE, codify +from ._common_util import QUERY_LANGUAGE, codify, response_to_json_dict from .document import Document from .view import View, QueryIndexView from .error import CloudantArgumentError, CloudantDesignDocumentException @@ -686,7 +686,7 @@ def info(self): ddoc_info = self.r_session.get( '/'.join([self.document_url, '_info'])) ddoc_info.raise_for_status() - return ddoc_info.json() + return response_to_json_dict(ddoc_info) def search_info(self, search_index): """ @@ -698,7 +698,7 @@ def search_info(self, search_index): ddoc_search_info = self.r_session.get( '/'.join([self.document_url, '_search_info', search_index])) ddoc_search_info.raise_for_status() - return ddoc_search_info.json() + return response_to_json_dict(ddoc_search_info) def search_disk_size(self, search_index): """ @@ -710,4 +710,4 @@ def search_disk_size(self, search_index): ddoc_search_disk_size = self.r_session.get( '/'.join([self.document_url, '_search_disk_size', search_index])) ddoc_search_disk_size.raise_for_status() - return ddoc_search_disk_size.json() + return response_to_json_dict(ddoc_search_disk_size) diff --git a/src/cloudant/document.py b/src/cloudant/document.py index ac08ee65..426b0656 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -20,6 +20,7 @@ from requests.exceptions import HTTPError from ._2to3 import url_quote, url_quote_plus +from ._common_util import response_to_json_dict from .error import CloudantDocumentException @@ -53,8 +54,8 @@ class Document(dict): :param database: A database instance used by the Document. Can be either a ``CouchDatabase`` or ``CloudantDatabase`` instance. :param str document_id: Optional document id used to identify the document. - :param str encoder: Optional JSON encoder object. - :param str decoder: Optional JSON decoder object. + :param str encoder: Optional JSON encoder object (extending json.JSONEncoder). + :param str decoder: Optional JSON decoder object (extending json.JSONDecoder). """ def __init__(self, database, document_id=None, **kwargs): super(Document, self).__init__() @@ -150,7 +151,7 @@ def create(self): data=json.dumps(doc, cls=self.encoder) ) resp.raise_for_status() - data = resp.json() + data = response_to_json_dict(resp) self._document_id = data['id'] super(Document, self).__setitem__('_id', data['id']) super(Document, self).__setitem__('_rev', data['rev']) @@ -167,7 +168,7 @@ def fetch(self): resp = self.r_session.get(self.document_url) resp.raise_for_status() self.clear() - self.update(resp.json(cls=self.decoder)) + self.update(response_to_json_dict(resp, cls=self.decoder)) def save(self): """ @@ -189,7 +190,7 @@ def save(self): headers=headers ) put_resp.raise_for_status() - data = put_resp.json() + data = response_to_json_dict(put_resp) super(Document, self).__setitem__('_rev', data['rev']) return @@ -418,7 +419,7 @@ def get_attachment( if attachment_type == 'text': return resp.text if attachment_type == 'json': - return resp.json() + return response_to_json_dict(resp) return resp.content @@ -447,7 +448,7 @@ def delete_attachment(self, attachment, headers=None): headers=headers ) resp.raise_for_status() - super(Document, self).__setitem__('_rev', resp.json()['rev']) + super(Document, self).__setitem__('_rev', response_to_json_dict(resp)['rev']) # Execute logic only if attachment metadata exists locally if self.get('_attachments'): # Remove the attachment metadata for the specified attachment @@ -457,7 +458,7 @@ def delete_attachment(self, attachment, headers=None): if not self['_attachments']: super(Document, self).__delitem__('_attachments') - return resp.json() + return response_to_json_dict(resp) def put_attachment(self, attachment, content_type, data, headers=None): """ @@ -494,4 +495,4 @@ def put_attachment(self, attachment, content_type, data, headers=None): ) resp.raise_for_status() self.fetch() - return resp.json() + return response_to_json_dict(resp) diff --git a/src/cloudant/index.py b/src/cloudant/index.py index 3d0d49c3..40274ce1 100644 --- a/src/cloudant/index.py +++ b/src/cloudant/index.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ from ._common_util import TEXT_INDEX_TYPE from ._common_util import SPECIAL_INDEX_TYPE from ._common_util import TEXT_INDEX_ARGS +from ._common_util import response_to_json_dict from .error import CloudantArgumentError, CloudantIndexException class Index(object): @@ -143,8 +144,8 @@ def create(self): headers=headers ) resp.raise_for_status() - self._ddoc_id = resp.json()['id'] - self._name = resp.json()['name'] + self._ddoc_id = response_to_json_dict(resp)['id'] + self._name = response_to_json_dict(resp)['name'] def _def_check(self): """ diff --git a/src/cloudant/query.py b/src/cloudant/query.py index b5bd2b58..ed9ec35e 100644 --- a/src/cloudant/query.py +++ b/src/cloudant/query.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ from .result import QueryResult from .error import CloudantArgumentError from ._common_util import QUERY_ARG_TYPES +from ._common_util import response_to_json_dict class Query(dict): """ @@ -172,7 +173,7 @@ def __call__(self, **kwargs): data=json.dumps(data, cls=self._encoder) ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) @contextlib.contextmanager def custom_result(self, **options): diff --git a/src/cloudant/scheduler.py b/src/cloudant/scheduler.py index 723635b2..012272fe 100644 --- a/src/cloudant/scheduler.py +++ b/src/cloudant/scheduler.py @@ -16,6 +16,8 @@ API module for interacting with scheduler endpoints """ +from ._common_util import response_to_json_dict + class Scheduler(object): """ API for retrieving scheduler jobs and documents. @@ -47,7 +49,7 @@ def list_docs(self, limit=None, skip=None): params["skip"] = skip resp = self._r_session.get('/'.join([self._scheduler, 'docs']), params=params) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def get_doc(self, doc_id): """ @@ -55,7 +57,7 @@ def get_doc(self, doc_id): """ resp = self._r_session.get('/'.join([self._scheduler, 'docs', '_replicator', doc_id])) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def list_jobs(self, limit=None, skip=None): @@ -78,4 +80,4 @@ def list_jobs(self, limit=None, skip=None): params["skip"] = skip resp = self._r_session.get('/'.join([self._scheduler, 'jobs']), params=params) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) diff --git a/src/cloudant/security_document.py b/src/cloudant/security_document.py index 22b0f79c..87b77ca3 100644 --- a/src/cloudant/security_document.py +++ b/src/cloudant/security_document.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import json from ._2to3 import url_quote_plus +from ._common_util import response_to_json_dict class SecurityDocument(dict): """ @@ -101,7 +102,7 @@ def fetch(self): resp = self.r_session.get(self.document_url) resp.raise_for_status() self.clear() - self.update(resp.json()) + self.update(response_to_json_dict(resp)) def save(self): """ diff --git a/src/cloudant/view.py b/src/cloudant/view.py index 685600e5..3a0e63fc 100644 --- a/src/cloudant/view.py +++ b/src/cloudant/view.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,7 +18,7 @@ import contextlib from ._2to3 import STRTYPE -from ._common_util import codify, get_docs +from ._common_util import codify, get_docs, response_to_json_dict from .result import Result from .error import CloudantArgumentError, CloudantViewException @@ -227,7 +227,7 @@ def __call__(self, **kwargs): self.url, self.design_doc.encoder, **kwargs) - return resp.json() + return response_to_json_dict(resp) @contextlib.contextmanager def custom_result(self, **options): diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index e110e5fa..0a544a15 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -247,7 +247,7 @@ def test_session_basic(self, m_req): """ m_response_ok = mock.MagicMock() type(m_response_ok).status_code = mock.PropertyMock(return_value=200) - m_response_ok.json.return_value = ['animaldb'] + type(m_response_ok).text = mock.PropertyMock(return_value='["animaldb"]') m_req.return_value = m_response_ok client = Cloudant('foo', 'bar', url=self.url, use_basic_auth=True) @@ -298,8 +298,7 @@ def test_change_credentials_basic(self, m_req): """ # mock 200 m_response_ok = mock.MagicMock() - m_response_ok.json.return_value = ['animaldb'] - + type(m_response_ok).text = mock.PropertyMock(return_value='["animaldb"]') # mock 401 m_response_bad = mock.MagicMock() m_response_bad.raise_for_status.side_effect = HTTPError('401 Unauthorized') diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index 5881a92d..f4c0e9aa 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -29,6 +29,7 @@ import mock import requests from cloudant._2to3 import UNICHR +from cloudant._common_util import response_to_json_dict from cloudant.design_document import DesignDocument from cloudant.document import Document from cloudant.error import CloudantArgumentError, CloudantDatabaseException @@ -237,7 +238,7 @@ def test_retrieve_db_metadata(self): """ resp = self.db.r_session.get( '/'.join((self.client.server_url, self.test_dbname))) - expected = resp.json() + expected = response_to_json_dict(resp) actual = self.db.metadata() self.assertListEqual(list(actual.keys()), list(expected.keys())) diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index 43714727..c986e769 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -27,6 +27,7 @@ import mock import requests +from cloudant._common_util import response_to_json_dict from cloudant.design_document import DesignDocument from cloudant.document import Document from cloudant.error import CloudantArgumentError, CloudantDesignDocumentException @@ -693,7 +694,7 @@ def test_save_with_no_views(self): # Ensure that remotely saved design document does not # include a views sub-document. resp = self.client.r_session.get(ddoc.document_url) - raw_ddoc = resp.json() + raw_ddoc = response_to_json_dict(resp) self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -1182,7 +1183,7 @@ def test_save_with_no_search_indexes(self): # Ensure that remotely saved design document does not # include a search indexes sub-document. resp = self.client.r_session.get(ddoc.document_url) - raw_ddoc = resp.json() + raw_ddoc = response_to_json_dict(resp) self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -1263,7 +1264,7 @@ def test_rewrite_rule(self): doc.save() resp = self.client.r_session.get('/'.join([ddoc.document_url, '_rewrite'])) self.assertEquals( - resp.json(), + response_to_json_dict(resp), { '_id': 'rewrite_doc', '_rev': doc['_rev'] @@ -1451,7 +1452,7 @@ def test_save_with_no_list_functions(self): # Ensure that remotely saved design document does not # include a lists sub-document. resp = self.client.r_session.get(ddoc.document_url) - raw_ddoc = resp.json() + raw_ddoc = response_to_json_dict(resp) self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -1752,7 +1753,7 @@ def test_save_with_no_show_functions(self): # Ensure that remotely saved design document does not # include a shows sub-document. resp = self.client.r_session.get(ddoc.document_url) - raw_ddoc = resp.json() + raw_ddoc = response_to_json_dict(resp) self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -1834,7 +1835,7 @@ def test_update_validator(self): data=json.dumps({'_id': 'test001'}) ) self.assertEqual( - resp.json(), + response_to_json_dict(resp), {'reason': 'Document must have an address.', 'error': 'forbidden'} ) diff --git a/tests/unit/iam_auth_tests.py b/tests/unit/iam_auth_tests.py index 6e0c7670..ec4fbd95 100644 --- a/tests/unit/iam_auth_tests.py +++ b/tests/unit/iam_auth_tests.py @@ -43,24 +43,22 @@ '2PTo4Exa17V-R_73Nq8VPCwpOvZcwKRA2sPTVgTMzU34max8b5kpTzVGJ' '6SXSItTVOUdAygZBng') -MOCK_IAM_TOKEN_RESPONSE = { - 'access_token': MOCK_ACCESS_TOKEN, - 'refresh_token': ('MO61FKNvVRWkSa4vmBZqYv_Jt1kkGMUc-XzTcNnR-GnIhVKXHUWxJVV3' - 'RddE8Kqh3X_TZRmyK8UySIWKxoJ2t6obUSUalPm90SBpTdoXtaljpNyo' - 'rmqCCYPROnk6JBym72ikSJqKHHEZVQkT0B5ggZCwPMnKagFj0ufs-VIh' - 'CF97xhDxDKcIPMWG02xxPuESaSTJJug7e_dUDoak_ZXm9xxBmOTRKwOx' - 'n5sTKthNyvVpEYPE7jIHeiRdVDOWhN5LomgCn3TqFCLpMErnqwgNYbyC' - 'Bd9rNm-alYKDb6Jle4njuIBpXxQPb4euDwLd1osApaSME3nEarFWqRBz' - 'hjoqCe1Kv564s_rY7qzD1nHGvKOdpSa0ZkMcfJ0LbXSQPs7gBTSVrBFZ' - 'qwlg-2F-U3Cto62-9qRR_cEu_K9ZyVwL4jWgOlngKmxV6Ku4L5mHp4Kg' - 'EJSnY_78_V2nm64E--i2ZA1FhiKwIVHDOivVNhggE9oabxg54vd63glp' - '4GfpNnmZsMOUYG9blJJpH4fDX4Ifjbw-iNBD7S2LRpP8b8vG9pb4WioG' - 'zN43lE5CysveKYWrQEZpThznxXlw1snDu_A48JiL3Lrvo1LobLhF3zFV' - '-kQ='), - 'token_type': 'Bearer', - 'expires_in': 3600, # 60mins - 'expiration': 1500470702 # Wed Jul 19 14:25:02 2017 -} +MOCK_IAM_TOKEN_RESPONSE = '{"access_token": "%s",\ + "refresh_token": "MO61FKNvVRWkSa4vmBZqYv_Jt1kkGMUc-XzTcNnR-GnIhVKXHUWxJVV3\ + RddE8Kqh3X_TZRmyK8UySIWKxoJ2t6obUSUalPm90SBpTdoXtaljpNyo\ + rmqCCYPROnk6JBym72ikSJqKHHEZVQkT0B5ggZCwPMnKagFj0ufs-VIh\ + CF97xhDxDKcIPMWG02xxPuESaSTJJug7e_dUDoak_ZXm9xxBmOTRKwOx\ + n5sTKthNyvVpEYPE7jIHeiRdVDOWhN5LomgCn3TqFCLpMErnqwgNYbyC\ + Bd9rNm-alYKDb6Jle4njuIBpXxQPb4euDwLd1osApaSME3nEarFWqRBz\ + hjoqCe1Kv564s_rY7qzD1nHGvKOdpSa0ZkMcfJ0LbXSQPs7gBTSVrBFZ\ + qwlg-2F-U3Cto62-9qRR_cEu_K9ZyVwL4jWgOlngKmxV6Ku4L5mHp4Kg\ + EJSnY_78_V2nm64E--i2ZA1FhiKwIVHDOivVNhggE9oabxg54vd63glp\ + 4GfpNnmZsMOUYG9blJJpH4fDX4Ifjbw-iNBD7S2LRpP8b8vG9pb4WioG\ + zN43lE5CysveKYWrQEZpThznxXlw1snDu_A48JiL3Lrvo1LobLhF3zFV\ + -kQ=",\ + "token_type": "Bearer",\ + "expires_in": 3600,\ + "expiration": 1500470702}'%(MOCK_ACCESS_TOKEN) class IAMAuthTests(unittest.TestCase): @@ -100,7 +98,8 @@ def test_iam_set_credentials(self): @mock.patch('cloudant._client_session.ClientSession.request') def test_iam_get_access_token(self, m_req): m_response = mock.MagicMock() - m_response.json.return_value = MOCK_IAM_TOKEN_RESPONSE + mock_token_response_text = mock.PropertyMock(return_value=MOCK_IAM_TOKEN_RESPONSE) + type(m_response).text = mock_token_response_text m_req.return_value = m_response iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984') @@ -120,7 +119,7 @@ def test_iam_get_access_token(self, m_req): self.assertEqual(access_token, MOCK_ACCESS_TOKEN) self.assertTrue(m_response.raise_for_status.called) - self.assertTrue(m_response.json.called) + mock_token_response_text.assert_called_with() @mock.patch('cloudant._client_session.ClientSession.request') @mock.patch('cloudant._client_session.IAMSession._get_access_token') @@ -152,10 +151,10 @@ def test_iam_logout(self): @mock.patch('cloudant._client_session.ClientSession.get') def test_iam_get_session_info(self, m_get): - m_info = {'ok': True, 'info': {'authentication_db': '_users'}} + m_info = '{"ok": true, "info": {"authentication_db": "_users"}}' m_response = mock.MagicMock() - m_response.json.return_value = m_info + type(m_response).text = mock.PropertyMock(return_value=m_info) m_get.return_value = m_response iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984') @@ -163,7 +162,7 @@ def test_iam_get_session_info(self, m_get): m_get.assert_called_once_with(iam._session_url) - self.assertEqual(info, m_info) + self.assertEqual(info, json.loads(m_info)) self.assertTrue(m_response.raise_for_status.called) @mock.patch('cloudant._client_session.IAMSession.login') @@ -172,7 +171,7 @@ def test_iam_first_request(self, m_req, m_login): # mock 200 m_response_ok = mock.MagicMock() type(m_response_ok).status_code = mock.PropertyMock(return_value=200) - m_response_ok.json.return_value = {'ok': True} + type(m_response_ok).text = mock.PropertyMock(return_value='{"ok": true}') m_req.return_value = m_response_ok @@ -197,7 +196,7 @@ def test_iam_renew_cookie_on_expiry(self, m_req, m_login): # mock 200 m_response_ok = mock.MagicMock() type(m_response_ok).status_code = mock.PropertyMock(return_value=200) - m_response_ok.json.return_value = {'ok': True} + type(m_response_ok).text = mock.PropertyMock(return_value='{"ok": true}') m_req.return_value = m_response_ok @@ -219,7 +218,7 @@ def test_iam_renew_cookie_on_401_success(self, m_req, m_login): # mock 200 m_response_ok = mock.MagicMock() type(m_response_ok).status_code = mock.PropertyMock(return_value=200) - m_response_ok.json.return_value = {'ok': True} + type(m_response_ok).text = mock.PropertyMock(return_value='{"ok": true}') # mock 401 m_response_bad = mock.MagicMock() type(m_response_bad).status_code = mock.PropertyMock(return_value=401) @@ -298,7 +297,7 @@ def test_iam_client_create(self, m_req, m_login): # mock 200 m_response_ok = mock.MagicMock() type(m_response_ok).status_code = mock.PropertyMock(return_value=200) - m_response_ok.json.return_value = ['animaldb'] + type(m_response_ok).text = mock.PropertyMock(return_value='["animaldb"]') m_req.return_value = m_response_ok diff --git a/tests/unit/scheduler_tests.py b/tests/unit/scheduler_tests.py index 97b5046f..5ef8cab9 100644 --- a/tests/unit/scheduler_tests.py +++ b/tests/unit/scheduler_tests.py @@ -46,107 +46,105 @@ def test_scheduler_docs(self): Test scheduler docs """ # set up mock response using a real captured response - m_response_ok = requests.Response() - m_response_ok.status_code = 200 - m_response_ok.json = mock.Mock() - m_response_ok.json.return_value = {"total_rows":6,"offset":0,"docs":[ - {"database":"tomblench/_replicator", - "doc_id":"296e48244e003eba8764b2156b3bf302", - "id":None, - "source":"https://tomblench.cloudant.com/animaldb/", - "target":"https://tomblench.cloudant.com/animaldb_copy/", - "state":"completed", - "error_count":0, - "info":{"revisions_checked":15, - "missing_revisions_found":2, - "docs_read":2, - "docs_written":2, - "changes_pending":None, - "doc_write_failures":0, - "checkpointed_source_seq":"19-g1AAAAGjeJyVz10KwjAMB_BoJ4KX8AZF2tWPJ3eVpqnO0XUg27PeTG9Wa_VhwmT6kkDIPz_iACArGcGS0DRnWxDmHE9HdJ3lxjUdad9yb1sXF6cacB9CqEqmZ3UczKUh2uGhHxeD8U9i_Z3AIla8vJVJUlBIZYTqX5A_KMM7SfFZrHCNLUK3p7RIkl5tSRD-K6kx6f6S0k8sScpYJTb5uFQ9AI9Ch9c"}, - "start_time":None, - "last_updated":"2017-04-13T14:53:50+00:00"}, - {"database":"tomblench/_replicator", - "doc_id":"3b749f320867d703550b0f758a4000ae", - "id":None, - "source":"https://examples.cloudant.com/animaldb/", - "target":"https://tomblench.cloudant.com/animaldb/", - "state":"completed", - "error_count":0, - "info":{"revisions_checked":15, - "missing_revisions_found":15, - "docs_read":15, - "docs_written":15, - "changes_pending":None, - "doc_write_failures":0, - "checkpointed_source_seq":"56-g1AAAAGveJzLYWBgYMlgTmFQSElKzi9KdUhJstDLTS3KLElMT9VLzskvTUnMK9HLSy3JAapkSmRIsv___39WBnMiby5QgN04JS3FLDUJWb8Jdv0gSxThigyN8diS5AAkk-qhFvFALEo2MTEwMSXGDDSbTPHYlMcCJBkagBTQsv0g28TBtpkbGCQapaF4C4cxJFt2AGIZ2GscYMuMDEzMUizMkC0zw25MFgBKoovi"}, - "start_time":None, - "last_updated":"2017-04-27T12:28:44+00:00"}, - {"database":"tomblench/_replicator", - "doc_id":"ad8f7896480b8081c8f0a2267ffd1859", - "id":None, - "source":"https://tortytherlediffecareette:*****@mikerhodestesty008.cloudant.com/moviesdb/", - "target":"https://tomblench.cloudant.com/moviesdb_rep/", - "state":"completed", - "error_count":0, - "info":{"revisions_checked":5997, - "missing_revisions_found":5997, - "docs_read":5997, - "docs_written":5997, - "changes_pending":None, - "doc_write_failures":0, - "checkpointed_source_seq":"5997-g1AAAANreJy10UEKwjAQAMBgBcVP2BeUpEm1PdmfaDYJSKkVtB486U_0J_oBTz5AHyAI3jxIjUml1x7ayy67LDssmyKE-nNHIleCWK5ULIF6uVrnW4xDT6TLjeRZ7mUqT_VkhyMYFkWRzB3Q1XOhez3iczKKghor6jvg6giTiroYiuNQYYqbpeIfNa2oh72KhQGosFlq9qN2FfUyFPgUCKONoneXR7TXSWuHkvsYjjEWjQVvgTta7lRyV_szKgmRbVx3ttzNcs7AcEoKCHAb3N1y_9-9DYeBYzEiNTYlX3EcE0s"}, - "start_time":None, - "last_updated":"2016-08-23T13:11:26+00:00"}, - {"database":"tomblench/_replicator", - "doc_id":"b63c053ecd95a4047b55ed8847b046f1", - "id":None, - "source":"https://tomblench.cloudant.com/atestdb2/", - "target":"https://tomblench.cloudant.com/atestdb1/", - "state":"completed", - "error_count":0, - "info":{"revisions_checked":1, - "missing_revisions_found":1, - "docs_read":1, - "docs_written":1, - "changes_pending":None, - "doc_write_failures":0, - "checkpointed_source_seq":"2-g1AAAAFHeJyNjkEOgjAQRSdAYjyFN2jSFCtdyVU6nSKQWhJC13ozvVktsoEF0c2fTPL_-98BQNHmBCdCM4y2JuQMuxu6YJlxQyDtJ-bt5JIx04DXGGOvYRsR-xGsk-JjTrW5hnv6Dg0XplRngmPwZJvOW9ry5D7PF0nhmU5CvmZm9mVKVVacLr8pfy9fmt5L02q9qEhJbtbr-w-AQmfD"}, - "start_time":None, - "last_updated":"2017-05-16T16:25:22+00:00"}, - {"database":"tomblench/_replicator", - "doc_id":"c71c9e69e30a182dc91d8938277bc85e", - "id":None, - "source":"https://tomblench.cloudant.com/animaldb/", - "target":"https://tomblench.cloudant.com/animaldb_copy/", - "state":"completed", - "error_count":0, - "info":{"revisions_checked":15, - "missing_revisions_found":15, - "docs_read":15, - "docs_written":15, - "changes_pending":None, - "doc_write_failures":0, - "checkpointed_source_seq":"14-g1AAAAEueJzLYWBgYMlgTmGQSUlKzi9KdUhJMtTLTU1M0UvOyS9NScwr0ctLLckBqmJKZEiy____f1YGUyJrLlCAPdHEPCktJZk43UkOQDKpHmoAI9gAw2STxCTzJOIMyGMBkgwNQApoxv6sDGaoK0yN04wsk80IGEGKHQcgdoAdygxxaIplklFaWhYAu2FdOA"}, - "start_time":None, - "last_updated":"2015-05-12T11:47:33+00:00"}, - {"database":"tomblench/_replicator", - "doc_id":"e6242d1e9ce059b0388fc75af3116a39", - "id":None, - "source":"https://tomblench.cloudant.com/atestdb1/", - "target":"https://tomblench.cloudant.com/atestdb2/", - "state":"completed", - "error_count":0, - "info":{"revisions_checked":1, - "missing_revisions_found":1, - "docs_read":1, - "docs_written":1, - "changes_pending":None, - "doc_write_failures":0, - "checkpointed_source_seq":"1-g1AAAAFheJyFzkEOgjAQBdBRSIyn8AZNgEJgJVeZ6bQCqSUhdK0305th1Q1dEDYzyWTy_rcAkHYJw4VJjZNumQpB_Y2s10LZ0TO6WTg92_B4RKDrsixDlyDcw-FUVUiFahjO3rE2vdMcY9k2Rm2Y9Ig8bWqspdz25Lbn0jDhGVYgX1_z8DMblnlp8n0lTir3kt7_pFV7NE2WYbluP3wATr5vQA"}, - "start_time":None, - "last_updated":"2017-05-16T16:24:02+00:00"} - ]} + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + type(m_response_ok).text = mock.PropertyMock(return_value='{"total_rows":6,"offset":0,"docs":[\ + {"database":"tomblench/_replicator",\ + "doc_id":"296e48244e003eba8764b2156b3bf302",\ + "id":null,\ + "source":"https://tomblench.cloudant.com/animaldb/",\ + "target":"https://tomblench.cloudant.com/animaldb_copy/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":15,\ + "missing_revisions_found":2,\ + "docs_read":2,\ + "docs_written":2,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"19-g1AAAAGjeJyVz10KwjAMB_BoJ4KX8AZF2tWPJ3eVpqnO0XUg27PeTG9Wa_VhwmT6kkDIPz_iACArGcGS0DRnWxDmHE9HdJ3lxjUdad9yb1sXF6cacB9CqEqmZ3UczKUh2uGhHxeD8U9i_Z3AIla8vJVJUlBIZYTqX5A_KMM7SfFZrHCNLUK3p7RIkl5tSRD-K6kx6f6S0k8sScpYJTb5uFQ9AI9Ch9c"},\ + "start_time":null,\ + "last_updated":"2017-04-13T14:53:50+00:00"},\ + {"database":"tomblench/_replicator",\ + "doc_id":"3b749f320867d703550b0f758a4000ae",\ + "id":null,\ + "source":"https://examples.cloudant.com/animaldb/",\ + "target":"https://tomblench.cloudant.com/animaldb/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":15,\ + "missing_revisions_found":15,\ + "docs_read":15,\ + "docs_written":15,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"56-g1AAAAGveJzLYWBgYMlgTmFQSElKzi9KdUhJstDLTS3KLElMT9VLzskvTUnMK9HLSy3JAapkSmRIsv___39WBnMiby5QgN04JS3FLDUJWb8Jdv0gSxThigyN8diS5AAkk-qhFvFALEo2MTEwMSXGDDSbTPHYlMcCJBkagBTQsv0g28TBtpkbGCQapaF4C4cxJFt2AGIZ2GscYMuMDEzMUizMkC0zw25MFgBKoovi"},\ + "start_time":null,\ + "last_updated":"2017-04-27T12:28:44+00:00"},\ + {"database":"tomblench/_replicator",\ + "doc_id":"ad8f7896480b8081c8f0a2267ffd1859",\ + "id":null,\ + "source":"https://tortytherlediffecareette:*****@mikerhodestesty008.cloudant.com/moviesdb/",\ + "target":"https://tomblench.cloudant.com/moviesdb_rep/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":5997,\ + "missing_revisions_found":5997,\ + "docs_read":5997,\ + "docs_written":5997,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"5997-g1AAAANreJy10UEKwjAQAMBgBcVP2BeUpEm1PdmfaDYJSKkVtB486U_0J_oBTz5AHyAI3jxIjUml1x7ayy67LDssmyKE-nNHIleCWK5ULIF6uVrnW4xDT6TLjeRZ7mUqT_VkhyMYFkWRzB3Q1XOhez3iczKKghor6jvg6giTiroYiuNQYYqbpeIfNa2oh72KhQGosFlq9qN2FfUyFPgUCKOnullXR7TXSWuHkvsYjjEWjQVvgTta7lRyV_szKgmRbVx3ttzNcs7AcEoKCHAb3N1y_9-9DYeBYzEiNTYlX3EcE0s"},\ + "start_time":null,\ + "last_updated":"2016-08-23T13:11:26+00:00"},\ + {"database":"tomblench/_replicator",\ + "doc_id":"b63c053ecd95a4047b55ed8847b046f1",\ + "id":null,\ + "source":"https://tomblench.cloudant.com/atestdb2/",\ + "target":"https://tomblench.cloudant.com/atestdb1/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":1,\ + "missing_revisions_found":1,\ + "docs_read":1,\ + "docs_written":1,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"2-g1AAAAFHeJyNjkEOgjAQRSdAYjyFN2jSFCtdyVU6nSKQWhJC13ozvVktsoEF0c2fTPL_-98BQNHmBCdCM4y2JuQMuxu6YJlxQyDtJ-bt5JIx04DXGGOvYRsR-xGsk-JjTrW5hnv6Dg0XplRngmPwZJvOW9ry5D7PF0nhmU5CvmZm9mVKVVacLr8pfy9fmt5L02q9qEhJbtbr-w-AQmfD"},\ + "start_time":null,\ + "last_updated":"2017-05-16T16:25:22+00:00"},\ + {"database":"tomblench/_replicator",\ + "doc_id":"c71c9e69e30a182dc91d8938277bc85e",\ + "id":null,\ + "source":"https://tomblench.cloudant.com/animaldb/",\ + "target":"https://tomblench.cloudant.com/animaldb_copy/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":15,\ + "missing_revisions_found":15,\ + "docs_read":15,\ + "docs_written":15,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"14-g1AAAAEueJzLYWBgYMlgTmGQSUlKzi9KdUhJMtTLTU1M0UvOyS9NScwr0ctLLckBqmJKZEiy____f1YGUyJrLlCAPdHEPCktJZk43UkOQDKpHmoAI9gAw2STxCTzJOIMyGMBkgwNQApoxv6sDGaoK0yN04wsk80IGEGKHQcgdoAdygxxaIplklFaWhYAu2FdOA"},\ + "start_time":null,\ + "last_updated":"2015-05-12T11:47:33+00:00"},\ + {"database":"tomblench/_replicator",\ + "doc_id":"e6242d1e9ce059b0388fc75af3116a39",\ + "id":null,\ + "source":"https://tomblench.cloudant.com/atestdb1/",\ + "target":"https://tomblench.cloudant.com/atestdb2/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":1,\ + "missing_revisions_found":1,\ + "docs_read":1,\ + "docs_written":1,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"1-g1AAAAFheJyFzkEOgjAQBdBRSIyn8AZNgEJgJVeZ6bQCqSUhdK0305th1Q1dEDYzyWTy_rcAkHYJw4VJjZNumQpB_Y2s10LZ0TO6WTg92_B4RKDrsixDlyDcw-FUVUiFahjO3rE2vdMcY9k2Rm2Y9Ig8bWqspdz25Lbn0jDhGVYgX1_z8DMblnlp8n0lTir3kt7_pFV7NE2WYbluP3wATr5vQA"},\ + "start_time":null,\ + "last_updated":"2017-05-16T16:24:02+00:00"}]}') self.client.r_session.get = mock.Mock(return_value=m_response_ok) scheduler = Scheduler(self.client) @@ -163,25 +161,24 @@ def test_scheduler_doc(self): Test scheduler doc """ # set up mock response using a real captured response - m_response_ok = requests.Response() - m_response_ok.status_code = 200 - m_response_ok.json = mock.Mock() - m_response_ok.json.return_value = {"database":"tomblench/_replicator", - "doc_id":"296e48244e003eba8764b2156b3bf302", - "id":None, - "source":"https://tomblench.cloudant.com/animaldb/", - "target":"https://tomblench.cloudant.com/animaldb_copy/", - "state":"completed", - "error_count":0, - "info":{"revisions_checked":15, - "missing_revisions_found":2, - "docs_read":2, - "docs_written":2, - "changes_pending":None, - "doc_write_failures":0, - "checkpointed_source_seq":"19-g1AAAAGjeJyVz10KwjAMB_BoJ4KX8AZF2tWPJ3eVpqnO0XUg27PeTG9Wa_VhwmT6kkDIPz_iACArGcGS0DRnWxDmHE9HdJ3lxjUdad9yb1sXF6cacB9CqEqmZ3UczKUh2uGhHxeD8U9i_Z3AIla8vJVJUlBIZYTqX5A_KMM7SfFZrHCNLUK3p7RIkl5tSRD-K6kx6f6S0k8sScpYJTb5uFQ9AI9Ch9c"}, - "start_time":None, - "last_updated":"2017-04-13T14:53:50+00:00"}; + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + type(m_response_ok).text = mock.PropertyMock(return_value='{"database":"tomblench/_replicator",\ + "doc_id":"296e48244e003eba8764b2156b3bf302",\ + "id":null,\ + "source":"https://tomblench.cloudant.com/animaldb/",\ + "target":"https://tomblench.cloudant.com/animaldb_copy/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":15,\ + "missing_revisions_found":2,\ + "docs_read":2,\ + "docs_written":2,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"19-g1AAAAGjeJyVz10KwjAMB_BoJ4KX8AZF2tWPJ3eVpqnO0XUg27PeTG9Wa_VhwmT6kkDIPz_iACArGcGS0DRnWxDmHE9HdJ3lxjUdad9yb1sXF6cacB9CqEqmZ3UczKUh2uGhHxeD8U9i_Z3AIla8vJVJUlBIZYTqX5A_KMM7SfFZrHCNLUK3p7RIkl5tSRD-K6kx6f6S0k8sScpYJTb5uFQ9AI9Ch9c"},\ + "start_time":null,\ + "last_updated":"2017-04-13T14:53:50+00:00"}') self.client.r_session.get = mock.Mock(return_value=m_response_ok) scheduler = Scheduler(self.client) response = scheduler.get_doc("296e48244e003eba8764b2156b3bf302") @@ -197,23 +194,22 @@ def test_scheduler_jobs(self): Test scheduler jobs """ # set up mock response using a real captured response - m_response_ok = requests.Response() - m_response_ok.status_code = 200 - m_response_ok.json = mock.Mock() - m_response_ok.json.return_value = {"total_rows":1,"offset":0, - "jobs":[{"database":None, - "id":"f11105eaaded4981d21ff8ebf846f48b+create_target", - "pid":"<0.5866.6800>", - "source":"https://clientlibs-test:*****@clientlibs-test.cloudant.com/largedb1g/", - "target":"https://tomblench:*****@tomblench.cloudant.com/largedb1g/", - "user":"tomblench", - "doc_id":None, - "history":[{"timestamp":"2018-04-12T13:06:20Z", - "type":"started"}, - {"timestamp":"2018-04-12T13:06:20Z", - "type":"added"}], - "node":"dbcore@db2.bigblue.cloudant.net", - "start_time":"2018-04-12T13:06:20Z"}]} + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + type(m_response_ok).text = mock.PropertyMock(return_value='{"total_rows":1,"offset":0,\ + "jobs":[{"database":null,\ + "id":"f11105eaaded4981d21ff8ebf846f48b+create_target",\ + "pid":"<0.5866.6800>",\ + "source":"https://clientlibs-test:*****@clientlibs-test.cloudant.com/largedb1g/",\ + "target":"https://tomblench:*****@tomblench.cloudant.com/largedb1g/",\ + "user":"tomblench",\ + "doc_id":null,\ + "history":[{"timestamp":"2018-04-12T13:06:20Z",\ + "type":"started"},\ + {"timestamp":"2018-04-12T13:06:20Z",\ + "type":"added"}],\ + "node":"dbcore@db2.bigblue.cloudant.net",\ + "start_time":"2018-04-12T13:06:20Z"}]}') self.client.r_session.get = mock.Mock(return_value=m_response_ok) scheduler = Scheduler(self.client) response = scheduler.list_jobs(skip=0, limit=10) From 38c047ecf8a2cf42f851f34c98d3af551bc66cf6 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 16 Nov 2018 10:15:26 +0000 Subject: [PATCH 14/97] Updated CHANGES for 2.10.1 release --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 47e621f0..bf15910c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,4 @@ -# Unreleased +# 2.10.1 (2018-11-16) - [FIXED] Unexpected keyword argument errors when using the library with the `simplejson` module present in the environment caused by `requests` preferentially From 30cd7857bd18c818093238641a85ca3177059b58 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 16 Nov 2018 10:17:48 +0000 Subject: [PATCH 15/97] Updated version to 2.10.1 --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index d33489b3..8bbb6e40 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.11.0-SNAPSHOT +2.10.1 diff --git a/docs/conf.py b/docs/conf.py index 72e556ae..4224bff9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.11.0-SNAPSHOT' +version = '2.10.1' # The full version, including alpha/beta/rc tags. -release = '2.11.0-SNAPSHOT' +release = '2.10.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index ffb150c2..32db1baf 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.11.0-SNAPSHOT' +__version__ = '2.10.1' # pylint: disable=wrong-import-position import contextlib From c296a5a5208d8e5850df939232c65713576e3855 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 16 Nov 2018 15:47:51 +0000 Subject: [PATCH 16/97] Updated version to 2.10.2-SNAPSHOT --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 8bbb6e40..4e9cca7c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.10.1 +2.10.2-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index 4224bff9..562082d6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.10.1' +version = '2.10.2-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.10.1' +release = '2.10.2-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 32db1baf..74f57225 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.10.1' +__version__ = '2.10.2-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From 9caf232a5eba0d558a2045bd7b2cdc21a1f83879 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 12 Dec 2018 10:34:28 +0000 Subject: [PATCH 17/97] Set repsonse encoding to UTF-8 to avoid chardet The performance of the default chardet used by requests can be very poor and since the JSON is UTF-8 we can avoid going throught that detection. --- CHANGES.md | 4 ++++ src/cloudant/_common_util.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index bf15910c..6d8e112f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +# Unreleased + +- [FIXED] A performance regression deserializing JSON in version 2.10.1. + # 2.10.1 (2018-11-16) - [FIXED] Unexpected keyword argument errors when using the library with the diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 7a373815..5460cd1c 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -288,6 +288,8 @@ def response_to_json_dict(response, **kwargs): :returns: dict of JSON response """ + if response.encoding is None: + response.encoding = 'utf-8' return json.loads(response.text, **kwargs) # Classes From 442526dc3eb0e6d4d69236e115b2a76109ebef9d Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 27 Nov 2018 13:08:43 +0000 Subject: [PATCH 18/97] Silence unnecessary-pass linter warnings --- src/cloudant/_client_session.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cloudant/_client_session.py b/src/cloudant/_client_session.py index 820bc532..232f821d 100644 --- a/src/cloudant/_client_session.py +++ b/src/cloudant/_client_session.py @@ -95,12 +95,14 @@ def login(self): """ No-op method - not implemented here. """ + # pylint: disable=unnecessary-pass pass def logout(self): """ No-op method - not implemented here. """ + # pylint: disable=unnecessary-pass pass From c13f42c61759f0c685338e8eb82d83116c292e82 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 18 Dec 2018 14:58:27 +0000 Subject: [PATCH 19/97] Updated version to 2.10.2 --- CHANGES.md | 2 +- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6d8e112f..2fc99a7f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,4 @@ -# Unreleased +# 2.10.2 (2018-12-19) - [FIXED] A performance regression deserializing JSON in version 2.10.1. diff --git a/VERSION b/VERSION index 4e9cca7c..c6436a85 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.10.2-SNAPSHOT +2.10.2 diff --git a/docs/conf.py b/docs/conf.py index 562082d6..7490396b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.10.2-SNAPSHOT' +version = '2.10.2' # The full version, including alpha/beta/rc tags. -release = '2.10.2-SNAPSHOT' +release = '2.10.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 74f57225..ef439a24 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.10.2-SNAPSHOT' +__version__ = '2.10.2' # pylint: disable=wrong-import-position import contextlib From 827f64684fb2d51f3770bce739e20623f38586fd Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 19 Dec 2018 11:07:18 +0000 Subject: [PATCH 20/97] Updated version to 2.10.3-SNAPSHOT --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index c6436a85..2814d561 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.10.2 +2.10.3-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index 7490396b..2ff46dea 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.10.2' +version = '2.10.3-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.10.2' +release = '2.10.3-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index ef439a24..3fac6b7e 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.10.2' +__version__ = '2.10.3-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From 2b3b14ae40256bb45577c1aa2d2c5707ad49c598 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 9 Jan 2019 10:41:36 +0000 Subject: [PATCH 21/97] Add option for client to authenticate with IAM token server. --- CHANGES.md | 4 ++++ src/cloudant/_client_session.py | 10 +++++++--- src/cloudant/client.py | 13 ++++++++++-- tests/unit/iam_auth_tests.py | 35 +++++++++++++++++++++++++++++++-- 4 files changed, 55 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2fc99a7f..ca875da9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +# UNRELEASED + +- [NEW] Added option for client to authenticate with IAM token server. + # 2.10.2 (2018-12-19) - [FIXED] A performance regression deserializing JSON in version 2.10.1. diff --git a/src/cloudant/_client_session.py b/src/cloudant/_client_session.py index 232f821d..82f3c7ab 100644 --- a/src/cloudant/_client_session.py +++ b/src/cloudant/_client_session.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (c) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -189,7 +189,8 @@ class IAMSession(ClientSession): This class extends ClientSession and provides IAM authentication. """ - def __init__(self, api_key, server_url, **kwargs): + def __init__(self, api_key, server_url, client_id=None, client_secret=None, + **kwargs): super(IAMSession, self).__init__( session_url=url_join(server_url, '_iam_session'), **kwargs) @@ -197,6 +198,9 @@ def __init__(self, api_key, server_url, **kwargs): self._api_key = api_key self._token_url = os.environ.get( 'IAM_TOKEN_URL', 'https://iam.bluemix.net/identity/token') + self._token_auth = None + if client_id and client_secret: + self._token_auth = (client_id, client_secret) @property def get_api_key(self): @@ -277,7 +281,7 @@ def _get_access_token(self): resp = super(IAMSession, self).request( 'POST', self._token_url, - auth=('bx', 'bx'), # required for user API keys + auth=self._token_auth, headers={'Accepts': 'application/json'}, data={ 'grant_type': 'urn:ibm:params:oauth:grant-type:apikey', diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 9b22d0d7..568f905e 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (c) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -78,6 +78,10 @@ class CouchDB(dict): IAM authentication with server. Default is False. Use :func:`~cloudant.client.CouchDB.iam` to construct an IAM authenticated client. + :param string iam_client_id: Keyword argument, client ID to use when + authenticating with the IAM token server. Default is ``None``. + :param string iam_client_secret: Keyword argument, client secret to use when + authenticating with the IAM token server. Default is ``None``. """ _DATABASE_CLASS = CouchDatabase @@ -95,6 +99,8 @@ def __init__(self, user, auth_token, admin_party=False, **kwargs): self._auto_renew = kwargs.get('auto_renew', False) self._use_basic_auth = kwargs.get('use_basic_auth', False) self._use_iam = kwargs.get('use_iam', False) + self._iam_client_id = kwargs.get('iam_client_id', None) + self._iam_client_secret = kwargs.get('iam_client_secret', None) # If user/pass exist in URL, remove and set variables if not self._use_basic_auth and self.server_url: parsed_url = url_parse(kwargs.get('url')) @@ -162,6 +168,8 @@ def connect(self): self._auth_token, self.server_url, auto_renew=self._auto_renew, + client_id=self._iam_client_id, + client_secret=self._iam_client_secret, timeout=self._timeout ) else: @@ -844,7 +852,8 @@ def bluemix(cls, vcap_services, instance_name=None, service_name=None, **kwargs) if hasattr(service, 'iam_api_key'): return Cloudant.iam(service.username, service.iam_api_key, - url=service.url) + url=service.url, + **kwargs) return Cloudant(service.username, service.password, url=service.url, diff --git a/tests/unit/iam_auth_tests.py b/tests/unit/iam_auth_tests.py index ec4fbd95..f36af4e0 100644 --- a/tests/unit/iam_auth_tests.py +++ b/tests/unit/iam_auth_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2017 IBM. All rights reserved. +# Copyright (c) 2017, 2019 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -108,7 +108,38 @@ def test_iam_get_access_token(self, m_req): m_req.assert_called_once_with( 'POST', iam._token_url, - auth=('bx', 'bx'), + auth=None, + headers={'Accepts': 'application/json'}, + data={ + 'grant_type': 'urn:ibm:params:oauth:grant-type:apikey', + 'response_type': 'cloud_iam', + 'apikey': MOCK_API_KEY + } + ) + + self.assertEqual(access_token, MOCK_ACCESS_TOKEN) + self.assertTrue(m_response.raise_for_status.called) + mock_token_response_text.assert_called_with() + + @mock.patch('cloudant._client_session.ClientSession.request') + def test_iam_get_access_token_with_iam_client_id_and_secret(self, m_req): + m_response = mock.MagicMock() + mock_token_response_text = mock.PropertyMock(return_value=MOCK_IAM_TOKEN_RESPONSE) + type(m_response).text = mock_token_response_text + m_req.return_value = m_response + + iam_client_id = 'foo' + iam_client_secret = 'bar' + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984', + client_id=iam_client_id, + client_secret=iam_client_secret) + access_token = iam._get_access_token() + + m_req.assert_called_once_with( + 'POST', + iam._token_url, + auth=(iam_client_id, iam_client_secret), headers={'Accepts': 'application/json'}, data={ 'grant_type': 'urn:ibm:params:oauth:grant-type:apikey', From 90473f459c6b75923811382c54fc9e70fc288709 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 8 Jan 2019 15:34:05 +0000 Subject: [PATCH 22/97] Update the default IAM token server URL --- CHANGES.md | 1 + docs/getting_started.rst | 2 +- src/cloudant/_client_session.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ca875da9..cd925be6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ # UNRELEASED - [NEW] Added option for client to authenticate with IAM token server. +- [FIXED] Updated the default IAM token server URL. # 2.10.2 (2018-12-19) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index a1cd43f7..04a46846 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -100,7 +100,7 @@ Cloud Platform. See `IBM Cloud Identity and Access Management `_ for more information. -The production IAM token service at *https://iam.bluemix.net/identity/token* is used +The production IAM token service at *https://iam.cloud.ibm.com/identity/token* is used by default. You can set an ``IAM_TOKEN_URL`` environment variable to override this. diff --git a/src/cloudant/_client_session.py b/src/cloudant/_client_session.py index 82f3c7ab..f6339ed2 100644 --- a/src/cloudant/_client_session.py +++ b/src/cloudant/_client_session.py @@ -197,7 +197,7 @@ def __init__(self, api_key, server_url, client_id=None, client_secret=None, self._api_key = api_key self._token_url = os.environ.get( - 'IAM_TOKEN_URL', 'https://iam.bluemix.net/identity/token') + 'IAM_TOKEN_URL', 'https://iam.cloud.ibm.com/identity/token') self._token_auth = None if client_id and client_secret: self._token_auth = (client_id, client_secret) From e3baf4746c485a0055ce2033e684e83dc5c3faec Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Mon, 21 Jan 2019 11:57:09 +0000 Subject: [PATCH 23/97] Prepare version 2.11.0 release --- CHANGES.md | 2 +- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cd925be6..3ed58be9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,4 @@ -# UNRELEASED +# 2.11.0 (2019-01-21) - [NEW] Added option for client to authenticate with IAM token server. - [FIXED] Updated the default IAM token server URL. diff --git a/VERSION b/VERSION index 2814d561..46b81d81 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.10.3-SNAPSHOT +2.11.0 diff --git a/docs/conf.py b/docs/conf.py index 2ff46dea..e0d01235 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.10.3-SNAPSHOT' +version = '2.11.0' # The full version, including alpha/beta/rc tags. -release = '2.10.3-SNAPSHOT' +release = '2.11.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 3fac6b7e..67a64efb 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.10.3-SNAPSHOT' +__version__ = '2.11.0' # pylint: disable=wrong-import-position import contextlib From 78ea0c476b2f1f3bf8844ea272f7691c47b9f806 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Mon, 21 Jan 2019 17:55:59 +0000 Subject: [PATCH 24/97] Update version to 2.11.1-SNAPSHOT --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 46b81d81..57f9a3c9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.11.0 +2.11.1-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index e0d01235..9a6ab4d4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.11.0' +version = '2.11.1-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.11.0' +release = '2.11.1-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 67a64efb..89d142f9 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.11.0' +__version__ = '2.11.1-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From e4a0224727b1b3ee5b9988801ddc2874484181f0 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Fri, 25 Jan 2019 11:32:25 -0500 Subject: [PATCH 25/97] Added get_query_result example to getting started (#423) Added get_query_result example to getting started --- CHANGES.md | 4 ++++ docs/getting_started.rst | 20 ++++++++++++++++++++ test-requirements.txt | 1 + 3 files changed, 25 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 3ed58be9..93f3b5e4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +# Unreleased + +- [IMPROVED] Updated `Getting started` section with a `get_query_result` example. + # 2.11.0 (2019-01-21) - [NEW] Added option for client to authenticate with IAM token server. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 04a46846..8ad13cd8 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -372,6 +372,26 @@ object already exists. for result in result_collection: print(result) +This example retrieves the query result from the specified database based on the query parameters provided, updates the +document, and saves the document in the remote database. +By default, the result is returned as a ``QueryResult`` which uses the skip and limit query parameters internally to +handle slicing and iteration through the query result collection. For more detail on slicing and iteration, refer +to the :class:`~cloudant.result.QueryResult` documentation. + +.. code-block:: python + + # Retrieve documents where the name field is 'foo' + selector = {'name': {'$eq': 'foo'}} + docs = my_database.get_query_result(selector) + for doc in docs: + # Create Document object from dict + updated_doc = Document(my_database, doc['_id']) + updated_doc.update(doc) + # Update document field + updated_doc['name'] = 'new_name' + # Save document + updated_doc.save() + **************** Context managers **************** diff --git a/test-requirements.txt b/test-requirements.txt index 098c70e5..88c13ffb 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,6 @@ mock==1.3.0 nose sphinx +sphinx_rtd_theme pylint flaky From 00b6cbb4e297477be214b7202d627bb99bc29223 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Fri, 1 Feb 2019 10:43:01 -0500 Subject: [PATCH 26/97] Fixed type in docstring for get_query_result (#425) Updated parameter `selector` from str to dict in docstring --- CHANGES.md | 1 + src/cloudant/database.py | 4 ++-- src/cloudant/query.py | 8 ++++---- src/cloudant/result.py | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 93f3b5e4..fed3ca54 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ # Unreleased - [IMPROVED] Updated `Getting started` section with a `get_query_result` example. +- [FIXED] Fixed parameter type of `selector` in docstring. # 2.11.0 (2019-01-21) diff --git a/src/cloudant/database.py b/src/cloudant/database.py index 51ff2430..da7b8a80 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -1112,7 +1112,7 @@ def get_query_result(self, selector, fields=None, raw_result=False, For more detail on slicing and iteration, refer to the :class:`~cloudant.result.QueryResult` documentation. - :param str selector: Dictionary object describing criteria used to + :param dict selector: Dictionary object describing criteria used to select documents. :param list fields: A list of fields to be returned by the query. :param bool raw_result: Dictates whether the query result is returned diff --git a/src/cloudant/query.py b/src/cloudant/query.py index ed9ec35e..791fd908 100644 --- a/src/cloudant/query.py +++ b/src/cloudant/query.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM. All rights reserved. +# Copyright (C) 2015, 2019 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -75,7 +75,7 @@ class Query(dict): :param int limit: Maximum number of results returned. :param int r: Read quorum needed for the result. Each document is read from at least 'r' number of replicas before it is returned in the results. - :param str selector: Dictionary object describing criteria used to select + :param dict selector: Dictionary object describing criteria used to select documents. :param int skip: Skip the first 'n' results, where 'n' is the value specified. @@ -137,7 +137,7 @@ def __call__(self, **kwargs): :param int r: Read quorum needed for the result. Each document is read from at least 'r' number of replicas before it is returned in the results. - :param str selector: Dictionary object describing criteria used to + :param dict selector: Dictionary object describing criteria used to select documents. :param int skip: Skip the first 'n' results, where 'n' is the value specified. @@ -201,7 +201,7 @@ def custom_result(self, **options): :param int r: Read quorum needed for the result. Each document is read from at least 'r' number of replicas before it is returned in the results. - :param str selector: Dictionary object describing criteria used to + :param dict selector: Dictionary object describing criteria used to select documents. :param list sort: A list of fields to sort by. Optionally the list can contain elements that are single member dictionary structures that diff --git a/src/cloudant/result.py b/src/cloudant/result.py index 302b7128..0fdcc472 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -454,7 +454,7 @@ class QueryResult(Result): :param int r: Read quorum needed for the result. Each document is read from at least 'r' number of replicas before it is returned in the results. - :param str selector: Dictionary object describing criteria used to + :param dict selector: Dictionary object describing criteria used to select documents. :param list sort: A list of fields to sort by. Optionally the list can contain elements that are single member dictionary structures that From 0bf212fe2f010b8dc47e66ee137f8b305e788a66 Mon Sep 17 00:00:00 2001 From: Alessandro Ogier Date: Sun, 10 Feb 2019 18:22:09 +0100 Subject: [PATCH 27/97] Fixing test_update_field_success_on_retry --- tests/unit/document_tests.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index 3676c2b1..4fa0e85c 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import cloudant """ _document_tests_ @@ -523,9 +524,18 @@ def test_update_field_success_on_retry(self): # Mock when saving the document # 1st call throw a 409 # 2nd call delegate to the real doc.save() - with mock.patch('cloudant.document.Document.save', - side_effect=[requests.HTTPError(response=mock.Mock(status_code=409, reason='conflict')), - doc.save()]) as m_save: + + class SaveMock(object): + calls = 0 + def save(self): + if self.calls == 0: + self.calls += 1 + raise requests.HTTPError(response=mock.Mock(status_code=409, reason='conflict')) + else: + return cloudant.document.Document.save(doc) + + with mock.patch.object(doc, 'save', + side_effect=SaveMock().save) as m_save: # A list of side effects containing only 1 element doc.update_field(doc.field_set, 'age', 7, max_tries=1) # Two calls to save, one with a 409 and one that succeeds From 0c67682adc6f27a12abcab4d401c0abff719d8c1 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 12 Feb 2019 12:00:39 +0000 Subject: [PATCH 28/97] Mock server responses for DatabaseTests.test_get_set_revision_limit. The `PUT /db/_revs_limit` endpoint was disabled in the Cloudant service. --- tests/unit/database_tests.py | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index f4c0e9aa..d57ba654 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -773,16 +773,36 @@ def test_revisions_diff(self): # Test no differences self.assertEqual(self.db.revisions_diff('julia006', doc['_rev']), {}) - def test_get_set_revision_limit(self): + @mock.patch('cloudant._client_session.ClientSession.request') + def test_get_set_revision_limit(self, m_req): """ Test setting and getting revision limits """ - limit = self.db.get_revision_limit() - self.assertIsInstance(limit, int) + # Setup mock responses. + mock_200_get_1 = mock.MagicMock() + type(mock_200_get_1).status_code = mock.PropertyMock(return_value=200) + type(mock_200_get_1).text = mock.PropertyMock(return_value='4321') + + mock_200_get_2 = mock.MagicMock() + type(mock_200_get_2).status_code = mock.PropertyMock(return_value=200) + type(mock_200_get_2).text = mock.PropertyMock(return_value='1234') + + mock_200_set = mock.MagicMock() + type(mock_200_set).status_code = mock.PropertyMock(return_value=200) + type(mock_200_set).text = mock.PropertyMock(return_value='{"ok":true}') + + m_req.side_effect = [mock_200_get_1, mock_200_set, mock_200_get_2] + + # Get current revisions limit. + self.assertEqual(self.db.get_revision_limit(), 4321) + + # Set new revisions limit. self.assertEqual(self.db.set_revision_limit(1234), {'ok': True}) - new_limit = self.db.get_revision_limit() - self.assertNotEqual(new_limit, limit) - self.assertEqual(new_limit, 1234) + + # Get new revisions limit. + self.assertEqual(self.db.get_revision_limit(), 1234) + + self.assertEquals(m_req.call_count, 3) @attr(db='couch') def test_view_clean_up(self): From 467dff23b6ad3400bee12800293e8ca586d65d80 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 12 Feb 2019 16:06:21 +0000 Subject: [PATCH 29/97] Stop document context manager from saving if an error is raised. --- CHANGES.md | 4 +++- src/cloudant/document.py | 9 ++++---- tests/unit/document_tests.py | 41 ++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fed3ca54..c7d48b60 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,9 @@ # Unreleased -- [IMPROVED] Updated `Getting started` section with a `get_query_result` example. +- [FIXED] Bug where document context manager performed remote save despite + uncaught exceptions being raised inside `with` block. - [FIXED] Fixed parameter type of `selector` in docstring. +- [IMPROVED] Updated `Getting started` section with a `get_query_result` example. # 2.11.0 (2019-01-21) diff --git a/src/cloudant/document.py b/src/cloudant/document.py index 426b0656..ad630ac0 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -341,12 +341,13 @@ def __enter__(self): return self - def __exit__(self, *args): + def __exit__(self, exc_type, exc_value, traceback): """ - Support context like editing of document fields. Handles context exit - logic. Executes a Document.save() upon exit. + Support context like editing of document fields. Handles context exit + logic. Executes a `Document.save()` upon exit if no exception occurred. """ - self.save() + if exc_type is None: + self.save() def __setitem__(self, key, value): """ diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index 4fa0e85c..b9d72a0f 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -622,6 +622,47 @@ def test_document_context_manager_no_doc_id(self): self.assertTrue(doc['_rev'].startswith('1-')) self.assertEqual(self.db['julia006'], doc) + def test_document_context_manager_creation_failure_on_error(self): + """ + Test that the document context manager skips document creation if there + is an error. + """ + with self.assertRaises(ZeroDivisionError), Document(self.db, 'julia006') as doc: + doc['name'] = 'julia' + doc['age'] = 6 + raise ZeroDivisionError() + + doc = Document(self.db, 'julia006') + try: + doc.fetch() + except requests.HTTPError as err: + self.assertEqual(err.response.status_code, 404) + else: + self.fail('Above statement should raise a HTTPError.') + + def test_document_context_manager_update_failure_on_error(self): + """ + Test that the document context manager skips document update if there + is an error. + """ + # Create the document. + doc = Document(self.db, 'julia006') + doc['name'] = 'julia' + doc['age'] = 6 + doc.save() + + # Make a document update and then raise an error. + with self.assertRaises(ZeroDivisionError), Document(self.db, 'julia006') as doc: + doc['age'] = 7 + raise ZeroDivisionError() + + # Assert the change persists locally. + self.assertEqual(doc['age'], 7) + + # Assert the document has not been saved to remote server. + self.assertTrue(doc['_rev'].startswith('1-')) + self.assertEqual(self.db['julia006']['age'], 6) + def test_document_context_manager_doc_create(self): """ Test that the document context manager will create a doc if it does From af07c04ae63754123a5a393061c0bde116c3f359 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 14 Feb 2019 14:54:44 +0000 Subject: [PATCH 30/97] Add warning to document CM documentation. --- docs/conf.py | 2 ++ docs/getting_started.rst | 54 +++++++++++++++++++++------------------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9a6ab4d4..3df55b2e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -21,6 +21,8 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath('../src')) + # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 8ad13cd8..072fe061 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -441,41 +441,45 @@ multiple updates to a single document. Note that we don't save to the server after each update. We only save once to the server upon exiting the ``Document`` context manager. - .. code-block:: python +.. warning:: Uncaught exceptions inside the ``with`` block will prevent your + document changes being saved to the remote server. However, changes + will still be applied to your local document object. - from cloudant import cloudant - from cloudant.document import Document +.. code-block:: python - with cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME) as client: + from cloudant import cloudant + from cloudant.document import Document - my_database = client.create_database('my_database') + with cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME) as client: - # Upon entry into the document context, fetches the document from the - # remote database, if it exists. Upon exit from the context, saves the - # document to the remote database with changes made within the context - # or creates a new document. - with Document(database, 'julia006') as document: - # If document exists, it's fetched from the remote database - # Changes are made locally - document['name'] = 'Julia' - document['age'] = 6 - # The document is saved to the remote database - - # Display a Document - print(my_database['julia30']) - - # Delete the database - client.delete_database('my_database') + my_database = client.create_database('my_database') - print('Databases: {0}'.format(client.all_dbs())) + # Upon entry into the document context, fetches the document from the + # remote database, if it exists. Upon exit from the context, saves the + # document to the remote database with changes made within the context + # or creates a new document. + with Document(database, 'julia006') as document: + # If document exists, it's fetched from the remote database + # Changes are made locally + document['name'] = 'Julia' + document['age'] = 6 + # The document is saved to the remote database + + # Display a Document + print(my_database['julia30']) + + # Delete the database + client.delete_database('my_database') + + print('Databases: {0}'.format(client.all_dbs())) Always use the ``_deleted`` document property to delete a document from within a ``Document`` context manager. For example: - .. code-block:: python +.. code-block:: python - with Document(my_database, 'julia30') as doc: - doc['_deleted'] = True + with Document(my_database, 'julia30') as doc: + doc['_deleted'] = True *You can also delete non underscore prefixed document keys to reduce the size of the request.* From e3e83d074528561cb296aeebdeaa0c190958156f Mon Sep 17 00:00:00 2001 From: Alessandro Ogier Date: Sun, 10 Feb 2019 14:01:40 +0100 Subject: [PATCH 31/97] removed _document_id logic (Fixes #430) --- CHANGES.md | 2 ++ src/cloudant/document.py | 38 +++++++++--------------------------- tests/unit/document_tests.py | 3 --- 3 files changed, 11 insertions(+), 32 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c7d48b60..0a3237d3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ uncaught exceptions being raised inside `with` block. - [FIXED] Fixed parameter type of `selector` in docstring. - [IMPROVED] Updated `Getting started` section with a `get_query_result` example. +- [FIXED] Removed internal `Document._document_id` property to allow a safe use of + dict's methods. # 2.11.0 (2019-01-21) diff --git a/src/cloudant/document.py b/src/cloudant/document.py index ad630ac0..38082054 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -63,9 +63,8 @@ def __init__(self, database, document_id=None, **kwargs): self._database = database self._database_host = self._client.server_url self._database_name = database.database_name - self._document_id = document_id - if self._document_id is not None: - self['_id'] = self._document_id + if document_id: + self['_id'] = document_id self.encoder = kwargs.get('encoder') or self._client.encoder self.decoder = kwargs.get('decoder') or json.JSONDecoder @@ -85,23 +84,23 @@ def document_url(self): :returns: Document URL """ - if self._document_id is None: + if '_id' not in self or self['_id'] is None: return None # handle design document url - if self._document_id.startswith('_design/'): + if self['_id'].startswith('_design/'): return '/'.join(( self._database_host, url_quote_plus(self._database_name), '_design', - url_quote(self._document_id[8:], safe='') + url_quote(self['_id'][8:], safe='') )) # handle document url return '/'.join(( self._database_host, url_quote_plus(self._database_name), - url_quote(self._document_id, safe='') + url_quote(self['_id'], safe='') )) def exists(self): @@ -111,7 +110,7 @@ def exists(self): :returns: True if the document exists in the remote database, otherwise False """ - if self._document_id is None: + if '_id' not in self or self['_id'] is None: return False resp = self.r_session.head(self.document_url) @@ -136,8 +135,6 @@ def create(self): updates the locally cached Document object with the ``_id`` and ``_rev`` returned as part of the successful response. """ - if self._document_id is not None: - self['_id'] = self._document_id # Ensure that an existing document will not be "updated" doc = dict(self) @@ -152,7 +149,6 @@ def create(self): ) resp.raise_for_status() data = response_to_json_dict(resp) - self._document_id = data['id'] super(Document, self).__setitem__('_id', data['id']) super(Document, self).__setitem__('_rev', data['rev']) @@ -318,8 +314,9 @@ def delete(self): params={"rev": self["_rev"]}, ) del_resp.raise_for_status() + _id = self['_id'] self.clear() - self.__setitem__('_id', self._document_id) + self['_id'] = _id def __enter__(self): """ @@ -349,23 +346,6 @@ def __exit__(self, exc_type, exc_value, traceback): if exc_type is None: self.save() - def __setitem__(self, key, value): - """ - Sets the _document_id when setting the '_id' field. - The _document_id is used to construct the document url. - """ - if key == '_id': - self._document_id = value - super(Document, self).__setitem__(key, value) - - def __delitem__(self, key): - """ - Sets the _document_id to None when deleting the '_id' field. - """ - if key == '_id': - self._document_id = None - super(Document, self).__delitem__(key) - def get_attachment( self, attachment, diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index b9d72a0f..61fa9fac 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -680,10 +680,8 @@ def test_setting_id(self): """ doc = Document(self.db) self.assertIsNone(doc.get('_id')) - self.assertEqual(doc._document_id, None) doc['_id'] = 'julia006' self.assertEqual(doc['_id'], 'julia006') - self.assertEqual(doc._document_id, 'julia006') def test_removing_id(self): """ @@ -693,7 +691,6 @@ def test_removing_id(self): doc['_id'] = 'julia006' del doc['_id'] self.assertIsNone(doc.get('_id')) - self.assertEqual(doc._document_id, None) def test_get_text_attachment(self): """ From cb44158d6b459c806d4760f03fc85e184567e6ba Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Mon, 11 Mar 2019 12:25:29 -0400 Subject: [PATCH 32/97] Fixed no-else-raise pylint warning (#438) --- src/cloudant/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 568f905e..79ce0993 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -629,9 +629,9 @@ def _usage_endpoint(self, endpoint, year=None, month=None): if err: raise CloudantArgumentError(101, year, month) - else: - resp.raise_for_status() - return response_to_json_dict(resp) + + resp.raise_for_status() + return response_to_json_dict(resp) def bill(self, year=None, month=None): """ From 1cb8abf9ccab6ff18c3d45dd8edcb32df1918f0a Mon Sep 17 00:00:00 2001 From: Alessandro Ogier Date: Mon, 18 Mar 2019 15:26:36 +0100 Subject: [PATCH 33/97] paginate by startkey (#437) * startkey/bookmark pagination --- CHANGES.md | 2 + src/cloudant/result.py | 95 +++++++++++++++++++++++++++----- tests/unit/query_result_tests.py | 14 +---- tests/unit/result_tests.py | 60 ++++++++++++++++---- tests/unit/unit_t_db_base.py | 5 ++ 5 files changed, 139 insertions(+), 37 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0a3237d3..3cbf6b60 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # Unreleased +- [IMPROVED] Updated `Result` iteration by paginating with views' `startkey` and + queries' `bookmark`. - [FIXED] Bug where document context manager performed remote save despite uncaught exceptions being raised inside `with` block. - [FIXED] Fixed parameter type of `selector` in docstring. diff --git a/src/cloudant/result.py b/src/cloudant/result.py index 0fdcc472..23fd3aac 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -15,6 +15,7 @@ """ API module for interacting with result collections. """ +from functools import partial from ._2to3 import STRTYPE from .error import ResultException from ._common_util import py_to_couch_validate, type_or_none @@ -327,16 +328,16 @@ def _handle_result_by_key_slice(self, key_slice): def __iter__(self): """ Provides iteration support, primarily for large data collections. - The iterator uses the ``skip`` and ``limit`` options to consume - data in chunks controlled by the ``page_size`` option. It retrieves - a batch of data from the result collection and then yields each - element. + The iterator uses the ``startkey``, ``startkey_docid``, and ``limit`` + options to consume data in chunks controlled by the ``page_size`` + option. It retrieves a batch of data from the result collection + and then yields each element. See :class:`~cloudant.result.Result` for Result iteration examples. :returns: Iterable data sequence """ - invalid_options = ('skip', 'limit') + invalid_options = ('limit', ) if any(x in invalid_options for x in self.options): raise ResultException(103, invalid_options, self.options) @@ -347,21 +348,60 @@ def __iter__(self): except ValueError: raise ResultException(104, self._page_size) - skip = 0 + init_opts = { + 'skip': self.options.pop('skip', None), + 'startkey': self.options.pop('startkey', None) + } + + self._call = partial(self._ref, #pylint: disable=attribute-defined-outside-init + limit=self._real_page_size, + **self.options) + + response = self._call(**{k: v + for k, v + in init_opts.items() + if v is not None}) + + return self._iterator(response) + + @property + def _real_page_size(self): + ''' + In views we paginate with N+1 items per page. + https://docs.couchdb.org/en/stable/ddocs/views/pagination.html#paging-alternate-method + ''' + return self._page_size + 1 + + def _iterator(self, response): + ''' + Iterate through view data. + ''' + while True: - response = self._ref( - limit=self._page_size, - skip=skip, - **self.options - ) result = self._parse_data(response) - skip += self._page_size if result: + doc_count = len(result) + last = result.pop() for row in result: yield row - if len(result) < self._page_size: + + # We expect doc_count = self._page_size + 1 results, if + # we have self._page_size or less it means we are on the + # last page and need to return the last result. + if doc_count < self._real_page_size: + yield last break del result + + # if we are in a view, keys could be duplicate so we + # need to start from the right docid + if last['id']: + response = self._call(startkey=last['key'], + startkey_docid=last['id']) + # reduce result keys are unique by definition + else: + response = self._call(startkey=last['key']) + else: break @@ -510,3 +550,32 @@ def _parse_data(self, data): query result JSON response content """ return data.get('docs', []) + + @property + def _real_page_size(self): + ''' + During queries iteration page size is user-specified + ''' + return self._page_size + + def _iterator(self, response): + ''' + Iterate through query data. + ''' + + while True: + result = self._parse_data(response) + bookmark = response.get('bookmark') + if result: + for row in result: + yield row + + del result + + if not bookmark: + break + + response = self._call(bookmark=bookmark) + + else: + break diff --git a/tests/unit/query_result_tests.py b/tests/unit/query_result_tests.py index 9f4fd170..98a16f3a 100644 --- a/tests/unit/query_result_tests.py +++ b/tests/unit/query_result_tests.py @@ -425,24 +425,14 @@ def test_get_item_index_slice_using_stop_only_limit_skip(self): def test_iteration_with_invalid_options(self): """ - Test that iteration raises an exception when "skip" and/or "limit" are - used as options for the result. + Test that iteration raises an exception when "limit" is + used as option for the result. """ - result = self.create_result(q_parms={'skip': 10}) - with self.assertRaises(ResultException) as cm: - invalid_result = [row for row in result] - self.assertEqual(cm.exception.status_code, 103) - result = self.create_result(q_parms={'limit': 10}) with self.assertRaises(ResultException) as cm: invalid_result = [row for row in result] self.assertEqual(cm.exception.status_code, 103) - result = self.create_result(q_parms={'limit': 10, 'skip': 10}) - with self.assertRaises(ResultException) as cm: - invalid_result = [row for row in result] - self.assertEqual(cm.exception.status_code, 103) - def test_iteration_invalid_page_size(self): """ Test that iteration raises an exception when and invalid "page_size" is diff --git a/tests/unit/result_tests.py b/tests/unit/result_tests.py index f56387ad..13b8eb7a 100644 --- a/tests/unit/result_tests.py +++ b/tests/unit/result_tests.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import mock """ result module - Unit tests for Result class """ @@ -19,6 +20,7 @@ from cloudant.error import ResultException from cloudant.result import Result, ResultByKey +from cloudant.view import View from nose.plugins.attrib import attr from requests.exceptions import HTTPError @@ -564,24 +566,14 @@ def test_get_item_key_slice_using_stop_only(self): def test_iteration_with_invalid_options(self): """ - Test that iteration raises an exception when "skip" and/or "limit" are - used as options for the result. + Test that iteration raises an exception when "limit" is + used as option for the result. """ - result = Result(self.view001, skip=10) - with self.assertRaises(ResultException) as cm: - invalid_result = [row for row in result] - self.assertEqual(cm.exception.status_code, 103) - result = Result(self.view001, limit=10) with self.assertRaises(ResultException) as cm: invalid_result = [row for row in result] self.assertEqual(cm.exception.status_code, 103) - result = Result(self.view001, skip=10, limit=10) - with self.assertRaises(ResultException) as cm: - invalid_result = [row for row in result] - self.assertEqual(cm.exception.status_code, 103) - def test_iteration_invalid_page_size(self): """ Test that iteration raises an exception when and invalid "page_size" is @@ -643,5 +635,49 @@ def test_iteration_no_data(self): result = Result(self.view001, startkey='ruby') self.assertEqual([x for x in result], []) + def test_iteration_integer_keys(self): + """ + Test that iteration works as expected when keys are integer. + """ + result = Result(self.view007, page_size=10) + self.assertEqual(len([x for x in result]), 100) + + def test_iteration_pagination(self): + """ + Test that iteration pagination works as expected. + """ + + class CallMock: + expected_calls = [ + {'limit': 28}, + {'limit': 28, 'startkey': 1, 'startkey_docid': 'julia027'}, + {'limit': 28, 'startkey': 1, 'startkey_docid': 'julia054'}, + {'limit': 28, 'startkey': 1, 'startkey_docid': 'julia081'}, + ] + + def __init__(self, outer): + self.outer = outer + self.expected_calls.reverse() + + def call(self, *args, **kwargs): + self.outer.assertEqual(dict(kwargs), + self.expected_calls.pop(), + 'pagination error') + return View.__call__(self.outer.view007, *args, **kwargs) + + with mock.patch.object(self, 'view007', + CallMock(self).call) as _: + + result = Result(self.view007, page_size=27) + + expected = [ + {'id': 'julia{0:03d}'.format(i), + 'key': 1, + 'value': 'julia'} + for i in range(100) + ] + self.assertEqual([x for x in result], expected) + + if __name__ == '__main__': unittest.main() diff --git a/tests/unit/unit_t_db_base.py b/tests/unit/unit_t_db_base.py index 1ca9220f..ad7eefd6 100644 --- a/tests/unit/unit_t_db_base.py +++ b/tests/unit/unit_t_db_base.py @@ -314,6 +314,10 @@ def create_views(self): 'function (doc) {\n emit([doc.name, doc.age], 1);\n}', '_count' ) + self.ddoc.add_view( + 'view007', + 'function (doc) {\n emit(1, doc.name);\n}' + ) self.ddoc.save() self.view001 = self.ddoc.get_view('view001') self.view002 = self.ddoc.get_view('view002') @@ -321,6 +325,7 @@ def create_views(self): self.view004 = self.ddoc.get_view('view004') self.view005 = self.ddoc.get_view('view005') self.view006 = self.ddoc.get_view('view006') + self.view007 = self.ddoc.get_view('view007') def create_search_index(self): """ From e1d4075f43627afae577506ab60d3a5a3eeb1720 Mon Sep 17 00:00:00 2001 From: Alessandro Ogier Date: Mon, 11 Mar 2019 17:16:05 +0100 Subject: [PATCH 34/97] slimming iterator --- src/cloudant/result.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cloudant/result.py b/src/cloudant/result.py index 23fd3aac..f51e1699 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -15,6 +15,7 @@ """ API module for interacting with result collections. """ +from collections import deque from functools import partial from ._2to3 import STRTYPE from .error import ResultException @@ -378,12 +379,12 @@ def _iterator(self, response): ''' while True: - result = self._parse_data(response) + result = deque(self._parse_data(response)) if result: doc_count = len(result) last = result.pop() - for row in result: - yield row + while result: + yield result.popleft() # We expect doc_count = self._page_size + 1 results, if # we have self._page_size or less it means we are on the From 55e7b472f75d7deb8dffd48c9250fcb31dc446ec Mon Sep 17 00:00:00 2001 From: Alessandro Ogier Date: Sat, 9 Mar 2019 00:02:42 +0100 Subject: [PATCH 35/97] halves memory usage iterator side --- src/cloudant/result.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cloudant/result.py b/src/cloudant/result.py index 23fd3aac..b934114d 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -379,6 +379,7 @@ def _iterator(self, response): while True: result = self._parse_data(response) + del response if result: doc_count = len(result) last = result.pop() From 5f926a0a122051f7fa73d31a5ade246cf1f28efb Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 6 Feb 2019 16:43:46 +0000 Subject: [PATCH 36/97] Add partitioned databases feature. --- CHANGES.md | 1 + src/cloudant/_common_util.py | 5 +- src/cloudant/client.py | 6 +- src/cloudant/database.py | 199 +++++++++++++++++++++++-- src/cloudant/design_document.py | 27 +++- src/cloudant/index.py | 24 ++- src/cloudant/query.py | 13 +- src/cloudant/view.py | 14 +- tests/unit/database_partition_tests.py | 130 ++++++++++++++++ tests/unit/unit_t_db_base.py | 21 ++- 10 files changed, 410 insertions(+), 30 deletions(-) create mode 100644 tests/unit/database_partition_tests.py diff --git a/CHANGES.md b/CHANGES.md index 3cbf6b60..63255bb2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ # Unreleased +- [NEW] Added partitioned database support. - [IMPROVED] Updated `Result` iteration by paginating with views' `startkey` and queries' `bookmark`. - [FIXED] Bug where document context manager performed remote save despite diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 5460cd1c..f7885f49 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -140,7 +140,8 @@ 'highlight_post_tag': STRTYPE, 'highlight_number': (int, LONGTYPE, NONETYPE), 'highlight_size': (int, LONGTYPE, NONETYPE), - 'include_fields': list + 'include_fields': list, + 'partition': STRTYPE } # Functions diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 79ce0993..b1993530 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -267,7 +267,7 @@ def all_dbs(self): resp.raise_for_status() return response_to_json_dict(resp) - def create_database(self, dbname, **kwargs): + def create_database(self, dbname, partitioned=False, **kwargs): """ Creates a new database on the remote server with the name provided and adds the new database object to the client's locally cached @@ -279,10 +279,12 @@ def create_database(self, dbname, **kwargs): :param bool throw_on_exists: Boolean flag dictating whether or not to throw a CloudantClientException when attempting to create a database that already exists. + :param bool partitioned: Create as a partitioned database. Defaults to + ``False``. :returns: The newly created database object """ - new_db = self._DATABASE_CLASS(self, dbname) + new_db = self._DATABASE_CLASS(self, dbname, partitioned=partitioned) try: new_db.create(kwargs.get('throw_on_exists', False)) except CloudantDatabaseException as ex: diff --git a/src/cloudant/database.py b/src/cloudant/database.py index da7b8a80..baf2df67 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -26,9 +26,9 @@ SEARCH_INDEX_ARGS, SPECIAL_INDEX_TYPE, TEXT_INDEX_TYPE, + TYPE_CONVERTERS, get_docs, - response_to_json_dict, - ) + response_to_json_dict) from .document import Document from .design_document import DesignDocument from .security_document import SecurityDocument @@ -50,13 +50,17 @@ class CouchDatabase(dict): :param str database_name: Database name used to reference the database. :param int fetch_limit: Optional fetch limit used to set the max number of documents to fetch per query during iteration cycles. Defaults to 100. + :param bool partitioned: Create as a partitioned database. Defaults to + ``False``. """ - def __init__(self, client, database_name, fetch_limit=100): + def __init__(self, client, database_name, fetch_limit=100, + partitioned=False): super(CouchDatabase, self).__init__() self.client = client self._database_host = client.server_url self.database_name = database_name self._fetch_limit = fetch_limit + self._partitioned = partitioned self.result = Result(self.all_docs) @property @@ -105,6 +109,18 @@ def creds(self): "user_ctx": session.get('userCtx') } + def database_partition_url(self, partition_key): + """ + Get the URL of the database partition. + + :param str partition_key: Partition key. + :return: URL of the database partition. + :rtype: str + """ + return '/'.join((self.database_url, + '_partition', + url_quote_plus(partition_key))) + def exists(self): """ Performs an existence check on the remote database. @@ -127,6 +143,18 @@ def metadata(self): resp.raise_for_status() return response_to_json_dict(resp) + def partition_metadata(self, partition_key): + """ + Retrieves the metadata dictionary for the remote database partition. + + :param str partition_key: Partition key. + :returns: Metadata dictionary for the database partition. + :rtype: dict + """ + resp = self.r_session.get(self.database_partition_url(partition_key)) + resp.raise_for_status() + return response_to_json_dict(resp) + def doc_count(self): """ Retrieves the number of documents in the remote database @@ -244,6 +272,33 @@ def get_security_document(self): sdoc.fetch() return sdoc + def get_partitioned_view_result(self, partition_key, ddoc_id, view_name, + raw_result=False, **kwargs): + """ + Retrieves the partitioned view result based on the design document and + view name. + + See :func:`~cloudant.database.CouchDatabase.get_view_result` method for + further details. + + :param str partition_key: Partition key. + :param str ddoc_id: Design document id used to get result. + :param str view_name: Name of the view used to get result. + :param bool raw_result: Dictates whether the view result is returned + as a default Result object or a raw JSON response. + Defaults to False. + :param kwargs: See + :func:`~cloudant.database.CouchDatabase.get_view_result` method for + available keyword arguments. + :returns: The result content either wrapped in a QueryResult or + as the raw response JSON content. + :rtype: QueryResult, dict + """ + ddoc = DesignDocument(self, ddoc_id) + view = View(ddoc, view_name, partition_key=partition_key) + + return self._get_view_result(view, raw_result, **kwargs) + def get_view_result(self, ddoc_id, view_name, raw_result=False, **kwargs): """ Retrieves the view result based on the design document and view name. @@ -336,7 +391,14 @@ def get_view_result(self, ddoc_id, view_name, raw_result=False, **kwargs): :returns: The result content either wrapped in a QueryResult or as the raw response JSON content """ - view = View(DesignDocument(self, ddoc_id), view_name) + ddoc = DesignDocument(self, ddoc_id) + view = View(ddoc, view_name) + + return self._get_view_result(view, raw_result, **kwargs) + + @staticmethod + def _get_view_result(view, raw_result, **kwargs): + """ Get view results helper. """ if raw_result: return view(**kwargs) if kwargs: @@ -359,7 +421,9 @@ def create(self, throw_on_exists=False): if not throw_on_exists and self.exists(): return self - resp = self.r_session.put(self.database_url) + resp = self.r_session.put(self.database_url, params={ + 'partitioned': TYPE_CONVERTERS.get(bool)(self._partitioned) + }) if resp.status_code == 201 or resp.status_code == 202: return self @@ -407,6 +471,29 @@ def all_docs(self, **kwargs): **kwargs) return response_to_json_dict(resp) + def partitioned_all_docs(self, partition_key, **kwargs): + """ + Wraps the _all_docs primary index on the database partition, and returns + the results by value. + + See :func:`~cloudant.database.CouchDatabase.all_docs` method for further + details. + + :param str partition_key: Partition key. + :param kwargs: See :func:`~cloudant.database.CouchDatabase.all_docs` + method for available keyword arguments. + :returns: Raw JSON response content from ``_all_docs`` endpoint. + :rtype: dict + """ + resp = get_docs(self.r_session, + '/'.join([ + self.database_partition_url(partition_key), + '_all_docs' + ]), + self.client.encoder, + **kwargs) + return response_to_json_dict(resp) + @contextlib.contextmanager def custom_result(self, **options): """ @@ -985,6 +1072,7 @@ def get_query_indexes(self, raw_result=False): self, data.get('ddoc'), data.get('name'), + partitioned=data.get('partitioned', False), **data.get('def', {}) )) elif data.get('type') == TEXT_INDEX_TYPE: @@ -992,6 +1080,7 @@ def get_query_indexes(self, raw_result=False): self, data.get('ddoc'), data.get('name'), + partitioned=data.get('partitioned', False), **data.get('def', {}) )) elif data.get('type') == SPECIAL_INDEX_TYPE: @@ -999,6 +1088,7 @@ def get_query_indexes(self, raw_result=False): self, data.get('ddoc'), data.get('name'), + partitioned=data.get('partitioned', False), **data.get('def', {}) )) else: @@ -1010,6 +1100,7 @@ def create_query_index( design_document_id=None, index_name=None, index_type='json', + partitioned=False, **kwargs ): """ @@ -1047,9 +1138,11 @@ def create_query_index( remote database """ if index_type == JSON_INDEX_TYPE: - index = Index(self, design_document_id, index_name, **kwargs) + index = Index(self, design_document_id, index_name, + partitioned=partitioned, **kwargs) elif index_type == TEXT_INDEX_TYPE: - index = TextIndex(self, design_document_id, index_name, **kwargs) + index = TextIndex(self, design_document_id, index_name, + partitioned=partitioned, **kwargs) else: raise CloudantArgumentError(103, index_type) index.create() @@ -1074,6 +1167,36 @@ def delete_query_index(self, design_document_id, index_type, index_name): raise CloudantArgumentError(103, index_type) index.delete() + def get_partitioned_query_result(self, partition_key, selector, fields=None, + raw_result=False, **kwargs): + """ + Retrieves the partitioned query result from the specified database based + on the query parameters provided. + + See :func:`~cloudant.database.CouchDatabase.get_query_result` method for + further details. + + :param str partition_key: Partition key. + :param str selector: Dictionary object describing criteria used to + select documents. + :param list fields: A list of fields to be returned by the query. + :param bool raw_result: Dictates whether the query result is returned + wrapped in a QueryResult or if the response JSON is returned. + Defaults to False. + :param kwargs: See + :func:`~cloudant.database.CouchDatabase.get_query_result` method for + available keyword arguments. + :returns: The result content either wrapped in a QueryResult or + as the raw response JSON content. + :rtype: QueryResult, dict + """ + query = Query(self, + selector=selector, + fields=fields, + partition_key=partition_key) + + return self._get_query_result(query, raw_result, **kwargs) + def get_query_result(self, selector, fields=None, raw_result=False, **kwargs): """ @@ -1143,10 +1266,15 @@ def get_query_result(self, selector, fields=None, raw_result=False, :returns: The result content either wrapped in a QueryResult or as the raw response JSON content """ - if fields: - query = Query(self, selector=selector, fields=fields) - else: - query = Query(self, selector=selector) + query = Query(self, + selector=selector, + fields=fields) + + return self._get_query_result(query, raw_result, **kwargs) + + @staticmethod + def _get_query_result(query, raw_result, **kwargs): + """ Get query results helper. """ if raw_result: return query(**kwargs) if kwargs: @@ -1166,12 +1294,16 @@ class CloudantDatabase(CouchDatabase): :param str database_name: Database name used to reference the database. :param int fetch_limit: Optional fetch limit used to set the max number of documents to fetch per query during iteration cycles. Defaults to 100. + :param bool partitioned: Create as a partitioned database. Defaults to + ``False``. """ - def __init__(self, client, database_name, fetch_limit=100): + def __init__(self, client, database_name, fetch_limit=100, + partitioned=False): super(CloudantDatabase, self).__init__( client, database_name, - fetch_limit=fetch_limit + fetch_limit=fetch_limit, + partitioned=partitioned ) def security_document(self): @@ -1280,6 +1412,36 @@ def shards(self): return response_to_json_dict(resp) + def get_partitioned_search_result(self, partition_key, ddoc_id, index_name, + **query_params): + """ + Retrieves the raw JSON content from the remote database based on the + partitioned search index on the server, using the query_params provided + as query parameters. + + See :func:`~cloudant.database.CouchDatabase.get_search_result` method + for further details. + + :param str partition_key: Partition key. + :param str ddoc_id: Design document id used to get the search result. + :param str index_name: Name used in part to identify the index. + :param query_params: See + :func:`~cloudant.database.CloudantDatabase.get_search_result` method + for available keyword arguments. + :returns: Search query result data in JSON format. + :rtype: dict + """ + ddoc = DesignDocument(self, ddoc_id) + + return self._get_search_result( + '/'.join(( + ddoc.document_partition_url(partition_key), + '_search', + index_name + )), + **query_params + ) + def get_search_result(self, ddoc_id, index_name, **query_params): """ Retrieves the raw JSON content from the remote database based on the @@ -1380,6 +1542,14 @@ def get_search_result(self, ddoc_id, index_name, **query_params): :returns: Search query result data in JSON format """ + ddoc = DesignDocument(self, ddoc_id) + return self._get_search_result( + '/'.join((ddoc.document_url, '_search', index_name)), + **query_params + ) + + def _get_search_result(self, query_url, **query_params): + """ Get search results helper. """ param_q = query_params.get('q') param_query = query_params.get('query') # Either q or query parameter is required @@ -1394,9 +1564,8 @@ def get_search_result(self, ddoc_id, index_name, **query_params): raise CloudantArgumentError(106, key, SEARCH_INDEX_ARGS[key]) # Execute query search headers = {'Content-Type': 'application/json'} - ddoc = DesignDocument(self, ddoc_id) resp = self.r_session.post( - '/'.join([ddoc.document_url, '_search', index_name]), + query_url, headers=headers, data=json.dumps(query_params, cls=self.client.encoder) ) diff --git a/src/cloudant/design_document.py b/src/cloudant/design_document.py index 66a9789d..34594b95 100644 --- a/src/cloudant/design_document.py +++ b/src/cloudant/design_document.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM. All rights reserved. +# Copyright (C) 2015, 2019 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ """ API module/class for interacting with a design document in a database. """ -from ._2to3 import iteritems_, STRTYPE +from ._2to3 import iteritems_, url_quote_plus, STRTYPE from ._common_util import QUERY_LANGUAGE, codify, response_to_json_dict from .document import Document from .view import View, QueryIndexView @@ -39,11 +39,18 @@ class DesignDocument(Document): either a ``CouchDatabase`` or ``CloudantDatabase`` instance. :param str document_id: Optional document id. If provided and does not start with ``_design/``, it will be prepended with ``_design/``. + :param bool partitioned: Optional. Create as a partitioned design document. + Defaults to ``False`` for both partitioned and non-partitioned + databases. """ - def __init__(self, database, document_id=None): + def __init__(self, database, document_id=None, partitioned=False): if document_id and not document_id.startswith('_design/'): document_id = '_design/{0}'.format(document_id) super(DesignDocument, self).__init__(database, document_id) + + if partitioned: + self.setdefault('options', {'partitioned': True}) + self._nested_object_names = frozenset(['views', 'indexes', 'lists', 'shows']) for prop in self._nested_object_names: self.setdefault(prop, dict()) @@ -269,6 +276,20 @@ def indexes(self): """ return self.get('indexes') + def document_partition_url(self, partition_key): + """ + Retrieve the design document partition URL. + + :param str partition_key: Partition key. + :return: Design document partition URL. + :rtype: str + """ + return '/'.join(( + self._database.database_partition_url(partition_key), + '_design', + url_quote_plus(self['_id'][8:], safe='') + )) + def add_view(self, view_name, map_func, reduce_func=None, **kwargs): """ Appends a MapReduce view to the locally cached DesignDocument View diff --git a/src/cloudant/index.py b/src/cloudant/index.py index 40274ce1..051c004b 100644 --- a/src/cloudant/index.py +++ b/src/cloudant/index.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM. All rights reserved. +# Copyright (C) 2015, 2019 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,18 +40,21 @@ class Index(object): Index. :param str design_document_id: Optional identifier of the design document. :param str name: Optional name of the index. + :param bool partitioned: Optional. Create as a partitioned index. Defaults + to ``False`` for both partitioned and non-partitioned databases. :param kwargs: Options used to construct the index definition for the purposes of index creation. For more details on valid options See :func:`~cloudant.database.CloudantDatabase.create_query_index`. """ - def __init__(self, database, design_document_id=None, name=None, **kwargs): + def __init__(self, database, design_document_id=None, name=None, partitioned=False, **kwargs): self._database = database self._r_session = self._database.r_session self._ddoc_id = design_document_id self._name = name self._type = JSON_INDEX_TYPE self._def = kwargs + self._partitioned = partitioned @property def index_url(self): @@ -100,6 +103,17 @@ def definition(self): """ return self._def + @property + def partitioned(self): + """ + Check if this index is partitioned. + + :return: ``True`` if index is partitioned, else ``False``. + :rtype: bool + """ + + return self._partitioned + def as_a_dict(self): """ Displays the index as a dictionary. This includes the design document @@ -114,6 +128,9 @@ def as_a_dict(self): 'def': self._def } + if self._partitioned: + index_dict['partitioned'] = True + return index_dict def create(self): @@ -137,6 +154,9 @@ def create(self): self._def_check() payload['index'] = self._def + if self._partitioned: + payload['partitioned'] = True + headers = {'Content-Type': 'application/json'} resp = self._r_session.post( self.index_url, diff --git a/src/cloudant/query.py b/src/cloudant/query.py index 791fd908..d6a0aebc 100644 --- a/src/cloudant/query.py +++ b/src/cloudant/query.py @@ -87,13 +87,18 @@ class Query(dict): :param str use_index: Identifies a specific index for the query to run against, rather than using the Cloudant Query algorithm which finds what it believes to be the best index. + :param str partition_key: Optional. Specify a query partition key. Defaults + to ``None`` resulting in global queries. """ def __init__(self, database, **kwargs): super(Query, self).__init__() self._database = database + self._partition_key = kwargs.pop('partition_key', None) self._r_session = self._database.r_session self._encoder = self._database.client.encoder + if kwargs.get('fields', True) is None: + del kwargs['fields'] # delete `None` fields kwarg if kwargs: super(Query, self).update(kwargs) self.result = QueryResult(self) @@ -105,7 +110,13 @@ def url(self): :returns: Query URL """ - return '/'.join((self._database.database_url, '_find')) + if self._partition_key: + base_url = self._database.database_partition_url( + self._partition_key) + else: + base_url = self._database.database_url + + return base_url + '/_find' def __call__(self, **kwargs): """ diff --git a/src/cloudant/view.py b/src/cloudant/view.py index 3a0e63fc..7b76f2a5 100644 --- a/src/cloudant/view.py +++ b/src/cloudant/view.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM. All rights reserved. +# Copyright (C) 2015, 2019 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -87,6 +87,8 @@ class View(dict): :param str view_name: Name used in part to identify the view. :param str map_func: Optional Javascript map function. :param str reduce_func: Optional Javascript reduce function. + :param str partition_key: Optional. Specify a view partition key. Defaults + to ``None`` resulting in global queries. """ def __init__( self, @@ -94,6 +96,7 @@ def __init__( view_name, map_func=None, reduce_func=None, + partition_key=None, **kwargs ): super(View, self).__init__() @@ -104,6 +107,7 @@ def __init__( self['map'] = codify(map_func) if reduce_func is not None: self['reduce'] = codify(reduce_func) + self._partition_key = partition_key self.update(kwargs) self.result = Result(self) @@ -167,8 +171,14 @@ def url(self): :returns: View URL """ + if self._partition_key: + base_url = self.design_doc.document_partition_url( + self._partition_key) + else: + base_url = self.design_doc.document_url + return '/'.join(( - self.design_doc.document_url, + base_url, '_view', self.view_name )) diff --git a/tests/unit/database_partition_tests.py b/tests/unit/database_partition_tests.py new file mode 100644 index 00000000..2c595f65 --- /dev/null +++ b/tests/unit/database_partition_tests.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# Copyright (C) 2019 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +_database_partition_tests_ +""" + +from cloudant.design_document import DesignDocument +from cloudant.index import Index, SpecialIndex + +from nose.plugins.attrib import attr + +from .unit_t_db_base import UnitTestDbBase + + +@attr(db=['cloudant']) +class DatabasePartitionTests(UnitTestDbBase): + + def setUp(self): + super(DatabasePartitionTests, self).setUp() + self.db_set_up(partitioned=True) + + def tearDown(self): + self.db_tear_down() + super(DatabasePartitionTests, self).tearDown() + + def test_is_partitioned_database(self): + self.assertTrue(self.db.metadata()['props']['partitioned']) + + def test_create_partitioned_design_document(self): + ddoc_id = 'empty_ddoc' + + ddoc = DesignDocument(self.db, ddoc_id, partitioned=True) + ddoc.save() + + r = self.db.r_session.get(ddoc.document_url) + r.raise_for_status() + + self.assertTrue(r.json()['options']['partitioned']) + + def test_partitioned_all_docs(self): + for partition_key in self.populate_db_with_partitioned_documents(5, 25): + docs = self.db.partitioned_all_docs(partition_key) + self.assertEquals(len(docs['rows']), 25) + + for doc in docs['rows']: + self.assertTrue(doc['id'].startswith(partition_key + ':')) + + def test_partition_metadata(self): + for partition_key in self.populate_db_with_partitioned_documents(5, 25): + meta = self.db.partition_metadata(partition_key) + self.assertEquals(meta['partition'], partition_key) + self.assertEquals(meta['doc_count'], 25) + + def test_partitioned_search(self): + ddoc = DesignDocument(self.db, 'partitioned_search', partitioned=True) + ddoc.add_search_index( + 'search1', + 'function(doc) { index("id", doc._id, {"store": true}); }' + ) + ddoc.save() + + for partition_key in self.populate_db_with_partitioned_documents(2, 10): + results = self.db.get_partitioned_search_result( + partition_key, ddoc['_id'], 'search1', query='*:*') + + i = 0 + for result in results['rows']: + print(result) + self.assertTrue(result['id'].startswith(partition_key + ':')) + i += 1 + self.assertEquals(i, 10) + + def test_get_partitioned_index(self): + index_name = 'test_partitioned_index' + + self.db.create_query_index(index_name=index_name, fields=['foo']) + + results = self.db.get_query_indexes() + self.assertEquals(len(results), 2) + + index_all_docs = results[0] + self.assertEquals(index_all_docs.name, '_all_docs') + self.assertEquals(type(index_all_docs), SpecialIndex) + self.assertFalse(index_all_docs.partitioned) + + index_partitioned = results[1] + self.assertEquals(index_partitioned.name, index_name) + self.assertEquals(type(index_partitioned), Index) + self.assertTrue(index_partitioned.partitioned) + + def test_partitioned_query(self): + self.db.create_query_index(fields=['foo']) + + for partition_key in self.populate_db_with_partitioned_documents(2, 10): + results = self.db.get_partitioned_query_result( + partition_key, selector={'foo': {'$eq': 'bar'}}) + + i = 0 + for result in results: + self.assertTrue(result['_id'].startswith(partition_key + ':')) + i += 1 + self.assertEquals(i, 10) + + def test_partitioned_view(self): + ddoc = DesignDocument(self.db, 'partitioned_view', partitioned=True) + ddoc.add_view('view1', 'function(doc) { emit(doc._id, 1); }') + ddoc.save() + + for partition_key in self.populate_db_with_partitioned_documents(2, 10): + results = self.db.get_partitioned_view_result( + partition_key, ddoc['_id'], 'view1') + + i = 0 + for result in results: + self.assertTrue( + result['id'].startswith(partition_key + ':')) + i += 1 + self.assertEquals(i, 10) diff --git a/tests/unit/unit_t_db_base.py b/tests/unit/unit_t_db_base.py index ad7eefd6..913d7b20 100644 --- a/tests/unit/unit_t_db_base.py +++ b/tests/unit/unit_t_db_base.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -253,13 +253,14 @@ def tearDown(self): """ del self.client - def db_set_up(self): + def db_set_up(self, partitioned=False): """ Set up test attributes for Database tests """ self.client.connect() self.test_dbname = self.dbname() - self.db = self.client._DATABASE_CLASS(self.client, self.test_dbname) + self.db = self.client._DATABASE_CLASS( + self.client, self.test_dbname, partitioned=partitioned) self.db.create() def db_tear_down(self): @@ -282,6 +283,20 @@ def populate_db_with_documents(self, doc_count=100, **kwargs): ] return self.db.bulk_docs(docs) + def populate_db_with_partitioned_documents(self, key_count, docs_per_partition): + partition_keys = [uuid.uuid4().hex.upper()[:8] for _ in range(key_count)] + for partition_key in partition_keys: + docs = [] + for i in range(docs_per_partition): + docs.append({ + '_id': '{0}:doc{1}'.format(partition_key, i), + 'foo': 'bar' + }) + + self.db.bulk_docs(docs) + + return partition_keys + def create_views(self): """ Create a design document with views for use with tests. From bb7cdb524925e11ffadb1f4dddfe66a59bff9775 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 7 Feb 2019 11:31:57 +0000 Subject: [PATCH 37/97] Add partitioned databases documentation. --- docs/getting_started.rst | 147 +++++++++++++++++++++++++++++++++++---- 1 file changed, 134 insertions(+), 13 deletions(-) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 072fe061..9cd4b65e 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -1,6 +1,6 @@ -=============== +############### Getting started -=============== +############### Now it's time to begin doing some work with Cloudant and Python. For working code samples of any of the API's please go to our test suite. @@ -32,7 +32,7 @@ a HTTP connection or a response on all requests. A timeout can be set using the ``timeout`` argument when constructing a client. Connecting with a client -^^^^^^^^^^^^^^^^^^^^^^^^ +======================== .. code-block:: python @@ -144,7 +144,7 @@ Note: Idle connections within the pool may be terminated by the server, so will indefinitely meaning that this will not completely remove the overhead of creating new connections. Using library in app server environment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +======================================= This library can be used in an app server, and the example below shows how to use ``client`` in a ``flask`` app server. @@ -190,7 +190,7 @@ existing database, or delete a database. The following examples assume a client connection has already been established. Creating a database -^^^^^^^^^^^^^^^^^^^ +=================== .. code-block:: python @@ -203,7 +203,7 @@ Creating a database print('SUCCESS!!') Opening a database -^^^^^^^^^^^^^^^^^^ +================== Opening an existing database is done by supplying the name of an existing database to the client. Since the ``Cloudant`` and ``CouchDB`` classes are @@ -216,13 +216,134 @@ sub-classes of ``dict``, this can be accomplished through standard Python my_database = client['my_database'] Deleting a database -^^^^^^^^^^^^^^^^^^^ +=================== .. code-block:: python # Delete a database using an initialized client client.delete_database('my_database') + +Partitioned Databases +===================== + +Partitioned databases introduce the ability for a user to create logical groups +of documents called partitions by providing a partition key with each document. + +.. warning:: Your Cloudant cluster must have the ``partitions`` feature enabled. + A full list of enabled features can be retrieved by calling the + client :func:`~cloudant.client.CouchDB.metadata` method. + +Creating a partitioned database +------------------------------- + +.. code-block:: python + + db = client.create_database('mydb', partitioned=True) + +Handling documents +------------------ + +The document ID contains both the partition key and document key in the form +``:`` where: + +- Partition Key *(string)*. Must be non-empty. Must not contain colons (as this + is the partition key delimiter) or begin with an underscore. +- Document Key *(string)*. Must be non-empty. Must not begin with an underscore. + +Be aware that ``_design`` documents and ``_local`` documents must not contain a +partition key as they are global definitions. + +**Create a document** + +.. code-block:: python + + partition_key = 'Year2' + document_key = 'julia30' + db.create_document({ + '_id': ':'.join((partition_key, document_key)), + 'name': 'Jules', + 'age': 6 + }) + +**Get a document** + +.. code-block:: python + + doc = db[':'.join((partition_key, document_key))] + +Creating design documents +------------------------- + +To define partitioned indexes you must set the ``partitioned=True`` optional +when constructing the new ``DesignDocument`` class. + +.. code-block:: python + + ddoc = DesignDocument(db, document_id='view', partitioned=True) + ddoc.add_view('myview','function(doc) { emit(doc.foo, doc.bar); }') + ddoc.save() + +Similarly, to define a partitioned Cloudant Query index you must set the +``partitioned=True`` optional. + +.. code-block:: python + + index = db.create_query_index( + design_document_id='query', + index_name='foo-index', + fields=['foo'], + partitioned=True + ) + index.create() + +Querying data +------------- + +A partition key can be specified when querying data so that results can be +constrained to a specific database partition. + +.. warning:: To run partitioned queries the database itself must be partitioned. + +**Query** + +.. code-block:: python + + results = self.db.get_partitioned_query_result( + partition_key, selector={'foo': {'$eq': 'bar'}}) + + for result in results: + ... + +See :func:`~cloudant.database.CouchDatabase.get_partitioned_query_result` for a +full list of supported parameters. + +**Search** + +.. code-block:: python + + results = self.db.get_partitioned_search_result( + partition_key, search_ddoc['_id'], 'search1', query='*:*') + + for result in results['rows']: + .... + +See :func:`~cloudant.database.CloudantDatabase.get_partitioned_search_result` +for a full list of supported parameters. + +**Views (MapReduce)** + +.. code-block:: python + + results = self.db.get_partitioned_view_result( + partition_key, view_ddoc['_id'], 'view1') + + for result in results: + .... + +See :func:`~cloudant.database.CouchDatabase.get_partitioned_view_result` for a +full list of supported parameters. + ********* Documents ********* @@ -235,7 +356,7 @@ create, read, update, and delete a document. These examples assume that either a CloudantDatabase or a CouchDatabase object already exists. Creating a document -^^^^^^^^^^^^^^^^^^^ +=================== .. code-block:: python @@ -255,7 +376,7 @@ Creating a document print('SUCCESS!!') Retrieving a document -^^^^^^^^^^^^^^^^^^^^^ +===================== Accessing a document from a database is done by supplying the document identifier of an existing document to either a ``CloudantDatabase`` or a @@ -271,7 +392,7 @@ classes are sub-classes of ``dict``, this is accomplished through standard print(my_document) Checking if a document exists -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +============================= You can check if a document exists in a database the same way you would check if a ``dict`` has a key-value pair by key. @@ -284,7 +405,7 @@ if a ``dict`` has a key-value pair by key. print('document with _id julia30 exists') Retrieve all documents -^^^^^^^^^^^^^^^^^^^^^^ +====================== You can also iterate over a ``CloudantDatabase`` or a ``CouchDatabase`` object to retrieve all documents in a database. @@ -296,7 +417,7 @@ to retrieve all documents in a database. print(document) Update a document -^^^^^^^^^^^^^^^^^ +================= .. code-block:: python @@ -312,7 +433,7 @@ Update a document my_document.save() Delete a document -^^^^^^^^^^^^^^^^^ +================= .. code-block:: python From 2135d6f3cf63218a907643bc42e247c313408d4e Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 19 Mar 2019 10:36:22 +0000 Subject: [PATCH 38/97] Use system Python installs in Jenkinsfile. --- Jenkinsfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1ab03a60..7501c07e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,8 +33,9 @@ def setupPythonAndTest(pythonVersion, testSuite) { withEnv(getEnvForSuite("${testSuite}")) { try { sh """ - virtualenv tmp -p /usr/local/lib/python${pythonVersion}/bin/${pythonVersion.startsWith('3') ? "python3" : "python"} + virtualenv tmp -p ${pythonVersion.startsWith('3') ? "python3" : "python"} . ./tmp/bin/activate + python --version pip install -r requirements.txt pip install -r test-requirements.txt ${'simplejson'.equals(testSuite) ? 'pip install simplejson' : ''} @@ -60,8 +61,8 @@ stage('Checkout'){ } stage('Test'){ - def py2 = '2.7.12' - def py3 = '3.5.2' + def py2 = '2' + def py3 = '3' def axes = [:] [py2, py3].each { version -> ['basic','cookie','iam'].each { auth -> From 3a535a87d2f85456e517c295cae1fb5b543534d3 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 27 Mar 2019 14:57:39 +0000 Subject: [PATCH 39/97] Set CHANGES.md to 100 char limit. --- CHANGES.md | 168 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 111 insertions(+), 57 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 63255bb2..e2c4dedf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,14 +1,13 @@ # Unreleased - [NEW] Added partitioned database support. -- [IMPROVED] Updated `Result` iteration by paginating with views' `startkey` and - queries' `bookmark`. -- [FIXED] Bug where document context manager performed remote save despite - uncaught exceptions being raised inside `with` block. +- [IMPROVED] Updated `Result` iteration by paginating with views' `startkey` and queries' + `bookmark`. +- [FIXED] Bug where document context manager performed remote save despite uncaught exceptions being + raised inside `with` block. - [FIXED] Fixed parameter type of `selector` in docstring. - [IMPROVED] Updated `Getting started` section with a `get_query_result` example. -- [FIXED] Removed internal `Document._document_id` property to allow a safe use of - dict's methods. +- [FIXED] Removed internal `Document._document_id` property to allow a safe use of dict's methods. # 2.11.0 (2019-01-21) @@ -21,9 +20,9 @@ # 2.10.1 (2018-11-16) -- [FIXED] Unexpected keyword argument errors when using the library with the - `simplejson` module present in the environment caused by `requests` preferentially - loading it over the system `json` module. +- [FIXED] Unexpected keyword argument errors when using the library with the `simplejson` module + present in the environment caused by `requests` preferentially loading it over the system `json` + module. # 2.10.0 (2018-09-19) @@ -31,14 +30,18 @@ - [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. - [NEW] Allow arbitrary query parameters to be passed to custom changes filters. - [FIXED] Case where an exception was raised after successful retry when using `doc.update_field`. -- [FIXED] Removed unnecessary request when retrieving a Result collection that is less than the `page_size` value. +- [FIXED] Removed unnecessary request when retrieving a Result collection that is less than the + `page_size` value. # 2.9.0 (2018-06-13) -- [NEW] Added functionality to test if a key is in a database as in `key in db`, overriding dict `__contains__` and checking in the remote database. -- [NEW] Moved `create_query_index` and other query related methods to `CouchDatabase` as the `_index`/`_find` API is available in CouchDB 2.x. +- [NEW] Added functionality to test if a key is in a database as in `key in db`, overriding dict + `__contains__` and checking in the remote database. +- [NEW] Moved `create_query_index` and other query related methods to `CouchDatabase` as the + `_index`/`_find` API is available in CouchDB 2.x. - [NEW] Support IAM authentication in replication documents. -- [FIXED] Case where `Document` context manager would throw instead of creating a new document if no `_id` was provided. +- [FIXED] Case where `Document` context manager would throw instead of creating a new document if no + `_id` was provided. - [IMPROVED] Added support for IAM API key in `cloudant_bluemix` method. - [IMPROVED] Shortened length of client URLs by removing username and password. - [IMPROVED] Verified library operation on Python 3.6.3. @@ -49,22 +52,28 @@ # 2.8.0 (2018-02-15) -- [NEW] Added support for `/_search_disk_size` endpoint which retrieves disk size information for a specific search index. +- [NEW] Added support for `/_search_disk_size` endpoint which retrieves disk size information for a + specific search index. - [FIXED] Updated default IBM Cloud Identity and Access Management token URL. -- [REMOVED] Removed broken source and target parameters that constantly threw `AttributeError` when creating a replication document. +- [REMOVED] Removed broken source and target parameters that constantly threw `AttributeError` when + creating a replication document. # 2.7.0 (2017-10-31) -- [NEW] Added API for upcoming Bluemix Identity and Access Management support for Cloudant on Bluemix. Note: IAM API key support is not yet enabled in the service. +- [NEW] Added API for upcoming Bluemix Identity and Access Management support for Cloudant on + Bluemix. Note: IAM API key support is not yet enabled in the service. - [NEW] Added HTTP basic authentication support. - [NEW] Added `Result.all()` convenience method. -- [NEW] Allow `service_name` to be specified when instantiating from a Bluemix VCAP_SERVICES environment variable. +- [NEW] Allow `service_name` to be specified when instantiating from a Bluemix VCAP_SERVICES + environment variable. - [IMPROVED] Updated `posixpath.join` references to use `'/'.join` when concatenating URL parts. -- [IMPROVED] Updated documentation by replacing deprecated Cloudant links with the latest Bluemix links. +- [IMPROVED] Updated documentation by replacing deprecated Cloudant links with the latest Bluemix + links. # 2.6.0 (2017-08-10) -- [NEW] Added `Cloudant.bluemix()` class method to the Cloudant client allowing service credentials to be passed using the CloudFoundry VCAP_SERVICES environment variable. +- [NEW] Added `Cloudant.bluemix()` class method to the Cloudant client allowing service credentials + to be passed using the CloudFoundry VCAP_SERVICES environment variable. - [FIXED] Fixed client construction in `cloudant_bluemix` context manager. - [FIXED] Fixed validation for feed options to accept zero as a valid value. @@ -76,26 +85,33 @@ - [FIXED] Fixed Cloudant exception code 409 with 412 when creating a database that already exists. - [FIXED] Catch error if `throw_on_exists` flag is `False` for creating a document. - [FIXED] Fixed /_all_docs call where `keys` is an empty list. -- [FIXED] Issue where docs with IDs that sorted lower than 0 were not returned when iterating through _all_docs. +- [FIXED] Issue where docs with IDs that sorted lower than 0 were not returned when iterating + through _all_docs. # 2.4.0 (2017-02-14) -- [NEW] Added `timeout` option to the client constructor for setting a timeout on a HTTP connection or a response. -- [NEW] Added `cloudant_bluemix` method to the Cloudant client allowing service credentials to be passed using the CloudFoundry VCAP_SERVICES environment variable. -- [IMPROVED] Updated non-response related errors with additional status code and improved error message for easier debugging. - All non-response error are handled using either CloudantException or CloudantArgumentError. +- [NEW] Added `timeout` option to the client constructor for setting a timeout on a HTTP connection + or a response. +- [NEW] Added `cloudant_bluemix` method to the Cloudant client allowing service credentials to be + passed using the CloudFoundry VCAP_SERVICES environment variable. +- [IMPROVED] Updated non-response related errors with additional status code and improved error + message for easier debugging. All non-response error are handled using either CloudantException + or CloudantArgumentError. - [FIXED] Support `long` type argument when executing in Python 2. # 2.3.1 (2016-11-30) -- [FIXED] Resolved issue where generated UUIDs for replication documents would not be converted to strings. +- [FIXED] Resolved issue where generated UUIDs for replication documents would not be converted to + strings. - [FIXED] Resolved issue where CouchDatabase.infinite_changes() method can cause a stack overflow. # 2.3.0 (2016-11-02) - [FIXED] Resolved issue where the custom JSON encoder was at times not used when transforming data. -- [NEW] Added support for managing the database security document through the SecurityDocument class and CouchDatabase convenience method `get_security_document`. -- [NEW] Added `auto_renewal` option to the client constructor to handle the automatic renewal of an expired session cookie auth. +- [NEW] Added support for managing the database security document through the SecurityDocument class + and CouchDatabase convenience method `get_security_document`. +- [NEW] Added `auto_renewal` option to the client constructor to handle the automatic renewal of an + expired session cookie auth. # 2.2.0 (2016-10-20) @@ -117,12 +133,14 @@ - [NEW] Added `st_indexes` accessor property for Cloudant Geospatial indexes. - [NEW] Added support for DesignDocument `_info` and `_search_info` endpoints. - [NEW] Added `validate_doc_update` accessor property for update validators. -- [NEW] Added support for a custom `requests.HTTPAdapter` to be configured using an optional `adapter` arg e.g. - `Cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME, adapter=Replay429Adapter())`. +- [NEW] Added support for a custom `requests.HTTPAdapter` to be configured using an optional + `adapter` arg e.g. `Cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME, + adapter=Replay429Adapter())`. - [IMPROVED] Made the 429 response code backoff optional and configurable. To enable the backoff add - an `adapter` arg of a `Replay429Adapter` with the desired number of retries and initial backoff. To replicate - the 2.0.0 behaviour use: `adapter=Replay429Adapter(retries=10, initialBackoff=0.25)`. If `retries` or - `initialBackoff` are not specified they will default to 3 retries and a 0.25 s initial backoff. + an `adapter` arg of a `Replay429Adapter` with the desired number of retries and initial + backoff. To replicate the 2.0.0 behaviour use: `adapter=Replay429Adapter(retries=10, + initialBackoff=0.25)`. If `retries` or `initialBackoff` are not specified they will default to 3 + retries and a 0.25 s initial backoff. - [IMPROVED] Additional error reason details appended to HTTP response message errors. - [FIX] `415 Client Error: Unsupported Media Type` when using keys with `db.all_docs`. - [FIX] Allowed strings as well as lists for search `group_sort` arguments. @@ -133,36 +151,65 @@ # 2.0.2 (2016-06-02) -- [IMPROVED] Updated documentation links from python-cloudant.readthedocs.org to python-cloudant.readthedocs.io. -- [FIX] Fixed issue with Windows platform compatibility,replaced usage of os.uname for the user-agent string. +- [IMPROVED] Updated documentation links from python-cloudant.readthedocs.org to + python-cloudant.readthedocs.io. +- [FIX] Fixed issue with Windows platform compatibility,replaced usage of os.uname for the + user-agent string. - [FIX] Fixed readthedocs link in README.rst to resolve to documentation home page. # 2.0.1 (2016-06-02) -- [IMPROVED] Updated documentation links from python-cloudant.readthedocs.org to python-cloudant.readthedocs.io. -- [FIX] Fixed issue with Windows platform compatibility,replaced usage of os.uname for the user-agent string. +- [IMPROVED] Updated documentation links from python-cloudant.readthedocs.org to + python-cloudant.readthedocs.io. +- [FIX] Fixed issue with Windows platform compatibility,replaced usage of os.uname for the + user-agent string. - [FIX] Fixed readthedocs link in README.rst to resolve to documentation home page. # 2.0.0 (2016-05-02) -- [BREAKING] Renamed modules account.py, errors.py, indexes.py, views.py, to client.py, error.py, index.py, and view.py. -- [BREAKING] Removed the `make_result` method from `View` and `Query` classes. If you need to make a query or view result, use `CloudantDatabase.get_query_result`, `CouchDatabase.get_view_result`, or the `View.custom_result` context manager. Additionally, the `Result` and `QueryResult` classes can be called directly to construct a result object. -- [BREAKING] Refactored the `SearchIndex` class to now be the `TextIndex` class. Also renamed the `CloudantDatabase` convenience methods of `get_all_indexes`, `create_index`, and `delete_index` as `get_query_indexes`, `create_query_index`, and `delete_query_index` respectively. These changes were made to clarify that the changed class and the changed methods were specific to query index processing only. -- [BREAKING] Replace "session" and "url" feed constructor arguments with "source" which can be either a client or a database object. Changes also made to the client `db_updates` method signature and the database `changes` method signature. -- [BREAKING] Fixed `CloudantDatabase.share_database` to accept all valid permission roles. Changed the method signature to accept roles as a list argument. -- [BREAKING] Removed credentials module from the API and moved it to the tests folder since the functionality is outside of the scope of this library but is still be useful in unit/integration tests. -- [IMPROVED] Changed the handling of queries using the keys argument to issue a http POST request instead of a http GET request so that the request is no longer bound by any URL length limitation. -- [IMPROVED] Added support for Result/QueryResult data access via index value and added validation logic to `Result.__getitem__()`. -- [IMPROVED] Updated feed functionality to process `_changes` and `_db_updates` with their supported options. Also added an infinite feed option. +- [BREAKING] Renamed modules account.py, errors.py, indexes.py, views.py, to client.py, error.py, + index.py, and view.py. +- [BREAKING] Removed the `make_result` method from `View` and `Query` classes. If you need to make + a query or view result, use `CloudantDatabase.get_query_result`, `CouchDatabase.get_view_result`, + or the `View.custom_result` context manager. Additionally, the `Result` and `QueryResult` classes + can be called directly to construct a result object. +- [BREAKING] Refactored the `SearchIndex` class to now be the `TextIndex` class. Also renamed the + `CloudantDatabase` convenience methods of `get_all_indexes`, `create_index`, and `delete_index` as + `get_query_indexes`, `create_query_index`, and `delete_query_index` respectively. These changes + were made to clarify that the changed class and the changed methods were specific to query index + processing only. +- [BREAKING] Replace "session" and "url" feed constructor arguments with "source" which can be + either a client or a database object. Changes also made to the client `db_updates` method + signature and the database `changes` method signature. +- [BREAKING] Fixed `CloudantDatabase.share_database` to accept all valid permission roles. Changed + the method signature to accept roles as a list argument. +- [BREAKING] Removed credentials module from the API and moved it to the tests folder since the + functionality is outside of the scope of this library but is still be useful in unit/integration + tests. +- [IMPROVED] Changed the handling of queries using the keys argument to issue a http POST request + instead of a http GET request so that the request is no longer bound by any URL length limitation. +- [IMPROVED] Added support for Result/QueryResult data access via index value and added validation + logic to `Result.__getitem__()`. +- [IMPROVED] Updated feed functionality to process `_changes` and `_db_updates` with their supported + options. Also added an infinite feed option. - [NEW] Handled HTTP status code `429 Too Many Requests` with blocking backoff and retries. -- [NEW] Added support for CouchDB Admin Party mode. This library can now be used with CouchDB instances where everyone is Admin. -- [FIX] Fixed `Document.get_attachment` method to successfully create text and binary files based on http response Content-Type. The method also returns text, binary, and json content based on http response Content-Type. -- [FIX] Added validation to `Cloudant.bill`, `Cloudant.volume_usage`, and `Cloudant.requests_usage` methods to ensure that a valid year/month combination or neither are used as arguments. +- [NEW] Added support for CouchDB Admin Party mode. This library can now be used with CouchDB + instances where everyone is Admin. +- [FIX] Fixed `Document.get_attachment` method to successfully create text and binary files based on + http response Content-Type. The method also returns text, binary, and json content based on http + response Content-Type. +- [FIX] Added validation to `Cloudant.bill`, `Cloudant.volume_usage`, and `Cloudant.requests_usage` + methods to ensure that a valid year/month combination or neither are used as arguments. - [FIX] Fixed the handling of empty views in the DesignDocument. -- [FIX] The `CouchDatabase.create_document` method now handles documents and design documents correctly. If the document created is a design document then the locally cached object will be a DesignDocument otherwise it will be a Document. -- [CHANGE] Moved internal `Code` class, functions like `python_to_couch` and `type_or_none`, and constants into a _common_util module. -- [CHANGE] Updated User-Agent header format to be `python-cloudant//Python///`. -- [CHANGE] Completed the addition of unit tests that target a database server. Removed all mocked unit tests. +- [FIX] The `CouchDatabase.create_document` method now handles documents and design documents + correctly. If the document created is a design document then the locally cached object will be a + DesignDocument otherwise it will be a Document. +- [CHANGE] Moved internal `Code` class, functions like `python_to_couch` and `type_or_none`, and + constants into a _common_util module. +- [CHANGE] Updated User-Agent header format to be `python-cloudant//Python///`. +- [CHANGE] Completed the addition of unit tests that target a database server. Removed all mocked + unit tests. # 2.0.0b2 (2016-02-24) @@ -192,10 +239,14 @@ - [NEW] Added unit tests targeting CouchDB and Cloudant databases. -- [FIX] Fixed bug in database create validation check to work if response code is either 201 (created) or 202 (accepted). +- [FIX] Fixed bug in database create validation check to work if response code is either 201 + (created) or 202 (accepted). - [FIX] Fixed database iterator infinite loop problem and to now yield a Document object. -- [BREAKING] Removed previous bulk_docs method from the CouchDatabase class and renamed the previous bulk_insert method as bulk_docs. The previous bulk_docs functionality is available through the all_docs method using the "keys" parameter. -- [FIX] Made missing_revisions, revisions_diff, get_revision_limit, set_revision_limit, and view_cleanup API methods available for CouchDB as well as Cloudant. +- [BREAKING] Removed previous bulk_docs method from the CouchDatabase class and renamed the previous + bulk_insert method as bulk_docs. The previous bulk_docs functionality is available through the + all_docs method using the "keys" parameter. +- [FIX] Made missing_revisions, revisions_diff, get_revision_limit, set_revision_limit, and + view_cleanup API methods available for CouchDB as well as Cloudant. - [BREAKING] Moved the db_update method to the account module. - [FIX] Fixed missing_revisions to key on 'missing_revs'. - [FIX] Fixed set_revision_limit to encode the request data payload correctly. @@ -203,10 +254,13 @@ - [BREAKING] Renamed Document `field_append` method to `list_field_append`. - [BREAKING] Renamed Document `field_remove` method to `list_field_remove`. - [BREAKING] Renamed Document `field_replace` method to `field_set`. -- [FIX] The Document local dictionary `_id` key is now synched with `_document_id` private attribute. +- [FIX] The Document local dictionary `_id` key is now synched with `_document_id` private + attribute. - [FIX] The Document local dictionary is now refreshed after an add/update/delete of an attachment. - [FIX] The Document `fetch()` method now refreshes the Document local dictionary content correctly. -- [BREAKING] Replace the ReplicatorDatabase class with the Replicator class. A Replicator object has a database attribute that represents the _replicator database. This allows the Replicator to work for both a CloudantDatabase and a CouchDatabase. +- [BREAKING] Replace the ReplicatorDatabase class with the Replicator class. A Replicator object + has a database attribute that represents the _replicator database. This allows the Replicator to + work for both a CloudantDatabase and a CouchDatabase. - [REMOVED] Removed "not implemented" methods from the DesignDocument. - [FIX] Add implicit "_design/" prefix for DesignDocument document ids. From a0b92b3b5d02897cf417a74fb1a11c6dc8698555 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 27 Mar 2019 15:10:24 +0000 Subject: [PATCH 40/97] Prepare version 2.12.0 release. --- CHANGES.md | 10 ++++++---- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e2c4dedf..d2217438 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,13 +1,15 @@ -# Unreleased +# 2.12.0 (2019-03-28) - [NEW] Added partitioned database support. -- [IMPROVED] Updated `Result` iteration by paginating with views' `startkey` and queries' - `bookmark`. - [FIXED] Bug where document context manager performed remote save despite uncaught exceptions being raised inside `with` block. - [FIXED] Fixed parameter type of `selector` in docstring. -- [IMPROVED] Updated `Getting started` section with a `get_query_result` example. - [FIXED] Removed internal `Document._document_id` property to allow a safe use of dict's methods. +- [IMPROVED] Performance of `Result` iteration by releasing result objects immediately after they + are returned to the client. +- [IMPROVED] Updated `Getting started` section with a `get_query_result` example. +- [IMPROVED] Updated `Result` iteration by paginating with views' `startkey` and queries' + `bookmark`. # 2.11.0 (2019-01-21) diff --git a/VERSION b/VERSION index 57f9a3c9..d8b69897 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.11.1-SNAPSHOT +2.12.0 diff --git a/docs/conf.py b/docs/conf.py index 3df55b2e..c400d37b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '2.11.1-SNAPSHOT' +version = '2.12.0' # The full version, including alpha/beta/rc tags. -release = '2.11.1-SNAPSHOT' +release = '2.12.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 89d142f9..93752011 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.11.1-SNAPSHOT' +__version__ = '2.12.0' # pylint: disable=wrong-import-position import contextlib From 77d37f4c3b7e8cab181b2ec675b401718a408d5b Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 28 Mar 2019 10:41:43 +0000 Subject: [PATCH 41/97] Update version to 2.12.1-SNAPSHOT. --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index d8b69897..91d3a8d4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.12.0 +2.12.1-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index c400d37b..764be732 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '2.12.0' +version = '2.12.1-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.12.0' +release = '2.12.1-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 93752011..677df2ed 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.12.0' +__version__ = '2.12.1-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From 83b0a6557a42061dfc4150f062ae0a27e42b195a Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 28 Mar 2019 11:12:11 +0000 Subject: [PATCH 42/97] Include README in package metadata. --- setup.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/setup.py b/setup.py index cac89fd5..e02192cd 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ """ +from os import path from setuptools import setup, find_packages requirements_file = open('requirements.txt') @@ -28,8 +29,14 @@ version = version_file.read().strip() version_file.close() +this_directory = path.abspath(path.dirname(__file__)) +with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + setup_args = { 'description': 'Cloudant / CouchDB Client Library', + 'long_description': long_description, + 'long_description_content_type': 'text/markdown', 'include_package_data': True, 'install_requires': requirements, 'name': 'cloudant', From e0ba190f6ba07fe3522a668747128214ad573c7e Mon Sep 17 00:00:00 2001 From: Felix Date: Wed, 24 Apr 2019 20:44:33 +0200 Subject: [PATCH 43/97] Update documentation to match function signature (#445) s/get_list_result/get_list_function_result/ --- src/cloudant/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cloudant/database.py b/src/cloudant/database.py index baf2df67..17978468 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -932,7 +932,7 @@ def get_list_function_result(self, ddoc_id, list_name, view_name, **kwargs): # Assuming that 'view001' exists as part of the # 'ddoc001' design document in the remote database... # Retrieve documents where the list function is 'list1' - resp = db.get_list_result('ddoc001', 'list1', 'view001', limit=10) + resp = db.get_list_function_result('ddoc001', 'list1', 'view001', limit=10) for row in resp['rows']: # Process data (in text format). From aef879cd71ac12350c0b7391b196499a26bc6348 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Mon, 20 May 2019 11:25:00 -0400 Subject: [PATCH 44/97] Updated 'create_database' to raise exception when database name is invalid (#447) --- CHANGES.md | 4 +++ src/cloudant/client.py | 1 + src/cloudant/error.py | 8 +++++- tests/unit/client_tests.py | 53 +++++++++++++++++++++++++++++++++++++- 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d2217438..3ba7cc7f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +# Unreleased + +- [FIXED] Correctly raise exceptions from `create_database` calls. + # 2.12.0 (2019-03-28) - [NEW] Added partitioned database support. diff --git a/src/cloudant/client.py b/src/cloudant/client.py index b1993530..f09d3a69 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -290,6 +290,7 @@ def create_database(self, dbname, partitioned=False, **kwargs): except CloudantDatabaseException as ex: if ex.status_code == 412: raise CloudantClientException(412, dbname) + raise ex super(CouchDB, self).__setitem__(dbname, new_db) return new_db diff --git a/src/cloudant/error.py b/src/cloudant/error.py index a35345f2..348e1bf6 100644 --- a/src/cloudant/error.py +++ b/src/cloudant/error.py @@ -107,7 +107,13 @@ class CloudantDatabaseException(CloudantException): """ def __init__(self, code=100, *args): try: - msg = DATABASE[code].format(*args) + if code in DATABASE: + msg = DATABASE[code].format(*args) + elif isinstance(code, int): + msg = ' '.join(args) + else: + code = 100 + msg = DATABASE[code] except (KeyError, IndexError): code = 100 msg = DATABASE[code] diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index 0a544a15..bfced219 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -34,7 +34,8 @@ from cloudant._client_session import BasicSession, CookieSession from cloudant.client import Cloudant, CouchDB from cloudant.database import CloudantDatabase -from cloudant.error import CloudantArgumentError, CloudantClientException +from cloudant.error import (CloudantArgumentError, CloudantClientException, + CloudantDatabaseException) from cloudant.feed import Feed, InfiniteFeed from nose.plugins.attrib import attr from requests import ConnectTimeout, HTTPError @@ -398,6 +399,56 @@ def test_create_existing_database(self): self.client.delete_database(dbname) self.client.disconnect() + def test_create_invalid_database_name(self): + """ + Test creation of database with an invalid name + """ + dbname = 'invalidDbName_' + self.client.connect() + with self.assertRaises(CloudantDatabaseException) as cm: + self.client.create_database(dbname) + self.assertEqual(cm.exception.status_code, 400) + self.client.disconnect() + + @skip_if_not_cookie_auth + @mock.patch('cloudant._client_session.Session.request') + def test_create_with_server_error(self, m_req): + """ + Test creation of database with a server error + """ + dbname = self.dbname() + # mock 200 for authentication + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + + # mock 404 for head request when verifying if database exists + m_response_bad = mock.MagicMock() + type(m_response_bad).status_code = mock.PropertyMock(return_value=404) + + # mock 500 when trying to create the database + m_resp_service_error = mock.MagicMock() + type(m_resp_service_error).status_code = mock.PropertyMock( + return_value=500) + type(m_resp_service_error).text = mock.PropertyMock( + return_value='Internal Server Error') + + m_req.side_effect = [m_response_ok, m_response_bad, m_resp_service_error] + + self.client.connect() + with self.assertRaises(CloudantDatabaseException) as cm: + self.client.create_database(dbname) + + self.assertEqual(cm.exception.status_code, 500) + + self.assertEquals(m_req.call_count, 3) + m_req.assert_called_with( + 'PUT', + '/'.join([self.url, dbname]), + data=None, + params={'partitioned': 'false'}, + timeout=(30, 300) + ) + def test_delete_non_existing_database(self): """ Test deletion of non-existing database From 00abad732d01364e2c4e2960afb706214d8b50fe Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 22 May 2019 16:12:31 +0100 Subject: [PATCH 45/97] Added skip_if_iam to some tests that rely on a legacy password Some of the bluemix initialization tests make a connection to the service to test the init worked correctly. They use a username and password combination so can't work with an IAM API key without additional switching. While they pass in cases where both legacy creds and IAM creds are supplied, if only IAM creds are supplied they fail so should be disabled for IAM. --- tests/unit/client_tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index bfced219..f3ac4e76 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,7 +40,7 @@ from nose.plugins.attrib import attr from requests import ConnectTimeout, HTTPError -from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase +from .unit_t_db_base import skip_if_iam, skip_if_not_cookie_auth, UnitTestDbBase from .. import bytes_, str_ @@ -782,6 +782,7 @@ def test_cloudant_bluemix_context_helper_raise_error_for_missing_iam_and_creds(s str(err) ) + @skip_if_iam def test_cloudant_bluemix_dedicated_context_helper(self): """ Test that the cloudant_bluemix context helper works as expected when @@ -888,6 +889,7 @@ def test_bluemix_constructor_with_iam(self): finally: c.disconnect() + @skip_if_iam def test_bluemix_constructor_specify_instance_name(self): """ Test instantiating a client object using a VCAP_SERVICES environment From f2c12984c325124d72cbdf5154a33b32538da941 Mon Sep 17 00:00:00 2001 From: tretinha Date: Wed, 9 Oct 2019 10:00:24 -0300 Subject: [PATCH 46/97] Fix "DeprecationWarning" from "collections" (#451) DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working. --- src/cloudant/_common_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index f7885f49..d47beb56 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -19,7 +19,7 @@ import sys import platform -from collections import Sequence +from collections.abc import Sequence import json from ._2to3 import LONGTYPE, STRTYPE, NONETYPE, UNITYPE, iteritems_ From fec158e71b9d49fdae74cd1e59e2f74ad9186cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Tue, 15 Oct 2019 17:39:54 +0200 Subject: [PATCH 47/97] Fix broken Python 2 build (#453) f2c1298 accidentally broke the build as collections.abc is only available in Python 3.x. This commit adds a workaround that enables the import that works both on 2 and 3. --- src/cloudant/_common_util.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index d47beb56..6e765c1d 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -19,12 +19,16 @@ import sys import platform -from collections.abc import Sequence import json from ._2to3 import LONGTYPE, STRTYPE, NONETYPE, UNITYPE, iteritems_ from .error import CloudantArgumentError, CloudantException, CloudantClientException +try: + from collections.abc import Sequence +except ImportError: + from collections import Sequence + # Library Constants USER_AGENT = '/'.join([ From 026f615ff389bbf99a562becd56df7c8d950189f Mon Sep 17 00:00:00 2001 From: James Mackenzie Date: Tue, 11 Feb 2020 14:13:41 +0000 Subject: [PATCH 48/97] Fixup the DesignDocumentTests::test_get_info and CloudantIndexExceptionTests::test_index_usage_via_query tests for couch3 --- tests/unit/design_document_tests.py | 5 +++-- tests/unit/index_tests.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index c986e769..35f46714 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2020 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -776,7 +776,8 @@ def test_get_info(self): info = ddoc_remote.info() # Remove variable fields to make equality easier to check info['view_index'].pop('signature') - info['view_index'].pop('disk_size') + if 'disk_size' in info['view_index']: + info['view_index'].pop('disk_size') # Remove Cloudant/Couch 2 fields if present to allow test to pass on Couch 1.6 if 'sizes' in info['view_index']: info['view_index'].pop('sizes') diff --git a/tests/unit/index_tests.py b/tests/unit/index_tests.py index 59a952d6..4361644f 100644 --- a/tests/unit/index_tests.py +++ b/tests/unit/index_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2020 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -390,7 +390,7 @@ def test_index_usage_via_query(self): self.populate_db_with_documents(100) result = self.db.get_query_result(fields=['name', 'age'], selector={'age': {'$eq': 6}}, raw_result=True) - self.assertTrue(str(result['warning']).startswith("no matching index found")) + self.assertTrue(str(result['warning']).lower().startswith("no matching index found")) @attr(db='cloudant') class TextIndexTests(UnitTestDbBase): From ae61f6a063f36f2b5cd1ad6341bd10e00c0ce4af Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 14 Feb 2020 09:34:39 +0000 Subject: [PATCH 49/97] Trigger search index builds before asserting metadata --- tests/unit/design_document_tests.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index c986e769..0bd27121 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -827,6 +827,9 @@ def test_get_search_info(self): ddoc_remote = DesignDocument(self.db, '_design/ddoc001') ddoc_remote.fetch() + # Make a request to the search index to ensure it is built + self.db.get_search_result('_design/ddoc001', 'search001', query='name:julia*') + search_info = ddoc_remote.search_info('search001') # Check the search index name self.assertEqual(search_info['name'], '_design/ddoc001/search001', 'The search index name should be correct.') @@ -856,7 +859,8 @@ def test_get_search_disk_size(self): ddoc_remote = DesignDocument(self.db, '_design/ddoc001') ddoc_remote.fetch() - ddoc_remote.search_info('search001') # trigger index build + # Make a request to the search index to ensure it is built + self.db.get_search_result('_design/ddoc001', 'search001', query='name:julia*') search_disk_size = ddoc_remote.search_disk_size('search001') From ab92f5e0b48238e549aa08b2fd3bb9a8d7e75c30 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 14 Feb 2020 11:57:34 +0000 Subject: [PATCH 50/97] Removed data_size from view info assertion The data_size field is no longer present in CouchDB 3.x so remove it from the assertion for compatibility. --- tests/unit/design_document_tests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index f710e9e5..7ec3c10e 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -778,6 +778,8 @@ def test_get_info(self): info['view_index'].pop('signature') if 'disk_size' in info['view_index']: info['view_index'].pop('disk_size') + if 'data_size' in info['view_index']: + info['view_index'].pop('data_size') # Remove Cloudant/Couch 2 fields if present to allow test to pass on Couch 1.6 if 'sizes' in info['view_index']: info['view_index'].pop('sizes') @@ -790,8 +792,7 @@ def test_get_info(self): {'view_index': {'update_seq': 0, 'waiting_clients': 0, 'language': 'javascript', 'purge_seq': 0, 'compact_running': False, - 'waiting_commit': False, 'updater_running': False, - 'data_size': 0 + 'waiting_commit': False, 'updater_running': False }, 'name': name }) From 924c2097f2e68f7d31e553a123015ac249e524cb Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 19 Feb 2020 09:45:00 +0000 Subject: [PATCH 51/97] Extend test read timeout value --- tests/unit/replicator_tests.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/unit/replicator_tests.py b/tests/unit/replicator_tests.py index 610d3588..28d78fab 100644 --- a/tests/unit/replicator_tests.py +++ b/tests/unit/replicator_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2020 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -220,10 +220,13 @@ def test_create_replication(self): def test_timeout_in_create_replication(self): """ Test that a read timeout exception is thrown when creating a - replicator with a timeout value of 500 ms. + replicator with a read timeout value of 5 s. """ - # Setup client with a timeout - self.set_up_client(auto_connect=True, timeout=.5) + # Setup client with a read timeout (but the standard connect timeout) + # Note that this timeout applies to all connections from this client + # setting it too short can cause intermittent failures when responses + # are not quick enough. Setting it too long makes the test take longer. + self.set_up_client(auto_connect=True, timeout=(30,5)) self.db = self.client[self.test_target_dbname] self.target_db = self.client[self.test_dbname] # Construct a replicator with the updated client From 058b96762d63f20b3e018ed35aff2070ec228436 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 19 Feb 2020 11:12:37 +0000 Subject: [PATCH 52/97] Updated test matrix version Python 2 is EOL, removed from matrix. Updated CouchDB patch versions. --- .travis.yml | 11 +++++------ Jenkinsfile | 3 +-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a88780e..44abb089 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,14 +3,13 @@ sudo: required language: python python: - - "2.7" - - "3.6" + - "3.8" env: - - ADMIN_PARTY=true COUCHDB_VERSION=2.1.1 - - ADMIN_PARTY=false COUCHDB_VERSION=2.1.1 - - ADMIN_PARTY=true COUCHDB_VERSION=1.7.1 - - ADMIN_PARTY=false COUCHDB_VERSION=1.7.1 + - ADMIN_PARTY=true COUCHDB_VERSION=2.3.1 + - ADMIN_PARTY=false COUCHDB_VERSION=2.3.1 + - ADMIN_PARTY=true COUCHDB_VERSION=1.7.2 + - ADMIN_PARTY=false COUCHDB_VERSION=1.7.2 services: - docker diff --git a/Jenkinsfile b/Jenkinsfile index 7501c07e..aa2f2c06 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -61,10 +61,9 @@ stage('Checkout'){ } stage('Test'){ - def py2 = '2' def py3 = '3' def axes = [:] - [py2, py3].each { version -> + [py3].each { version -> ['basic','cookie','iam'].each { auth -> axes.put("Python${version}-${auth}", {setupPythonAndTest(version, auth)}) } From 47b8a5ab4f05f036ca97e6e5019d4634b230e094 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 19 Feb 2020 12:00:00 +0000 Subject: [PATCH 53/97] Corrected syntax to eliminate warning in 3.8 --- .travis.yml | 2 +- Jenkinsfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 44abb089..b49557a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ before_script: # command to run tests script: - pylint ./src/cloudant - - nosetests -A 'not db or ((db is "couch" or "couch" in db) and (not couchapi or couchapi <='${COUCHDB_VERSION:0:1}'))' -w ./tests/unit + - nosetests -A 'not db or ((db == "couch" or "couch" in db) and (not couchapi or couchapi <='${COUCHDB_VERSION:0:1}'))' -w ./tests/unit notifications: email: false diff --git a/Jenkinsfile b/Jenkinsfile index aa2f2c06..ddf21dd4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,7 +40,7 @@ def setupPythonAndTest(pythonVersion, testSuite) { pip install -r test-requirements.txt ${'simplejson'.equals(testSuite) ? 'pip install simplejson' : ''} pylint ./src/cloudant - nosetests -A 'not db or (db is "cloudant" or "cloudant" in db)' -w ./tests/unit --with-xunit + nosetests -A 'not db or (db == "cloudant" or "cloudant" in db)' -w ./tests/unit --with-xunit """ } finally { // Load the test results From d84a0ea12d07688f0d6391616869b89792a11a1f Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 19 Feb 2020 14:43:11 +0000 Subject: [PATCH 54/97] Updated CouchDB version check in tests --- tests/unit/unit_t_db_base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/unit_t_db_base.py b/tests/unit/unit_t_db_base.py index 913d7b20..2067af0c 100644 --- a/tests/unit/unit_t_db_base.py +++ b/tests/unit/unit_t_db_base.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2020 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -109,7 +109,7 @@ def setUpClass(cls): if os.environ.get('DB_USER') is None: # Get couchdb docker node name - if os.environ.get('COUCHDB_VERSION') == '2.1.1': + if os.environ.get('COUCHDB_VERSION') == '2.3.1': os.environ['NODENAME'] = requests.get( '{0}/_membership'.format(os.environ['DB_URL'])).json()['all_nodes'][0] os.environ['DB_USER_CREATED'] = '1' @@ -117,7 +117,7 @@ def setUpClass(cls): unicode_(uuid.uuid4()) ) os.environ['DB_PASSWORD'] = 'password' - if os.environ.get('COUCHDB_VERSION') == '2.1.1': + if os.environ.get('COUCHDB_VERSION') == '2.3.1': resp = requests.put( '{0}/_node/{1}/_config/admins/{2}'.format( os.environ['DB_URL'], @@ -143,7 +143,7 @@ def tearDownClass(cls): """ if (os.environ.get('RUN_CLOUDANT_TESTS') is None and os.environ.get('DB_USER_CREATED') is not None): - if os.environ.get('COUCHDB_VERSION') == '2.1.1': + if os.environ.get('COUCHDB_VERSION') == '2.3.1': resp = requests.delete( '{0}://{1}:{2}@{3}/_node/{4}/_config/admins/{5}'.format( os.environ['DB_URL'].split('://', 1)[0], From 8331efae8dc7f432267c7f7b96062af8a4cd5a59 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 19 Feb 2020 15:21:05 +0000 Subject: [PATCH 55/97] Fixed invalid DB name test The HEAD on an invalid DB name that doesn't exist returns a 400 in 1.7.2 but a 404 in 1.7.1 and 2.x --- tests/unit/client_tests.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index f3ac4e76..2726de33 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2020 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -405,9 +405,10 @@ def test_create_invalid_database_name(self): """ dbname = 'invalidDbName_' self.client.connect() - with self.assertRaises(CloudantDatabaseException) as cm: + with self.assertRaises((CloudantDatabaseException, HTTPError)) as cm: self.client.create_database(dbname) - self.assertEqual(cm.exception.status_code, 400) + code = cm.exception.status_code if hasattr(cm.exception, 'status_code') else cm.exception.response.status_code + self.assertEqual(code, 400) self.client.disconnect() @skip_if_not_cookie_auth From 65fcb81fadecdb27e3127b9ab2d9c9cf38cc92ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Tue, 14 Apr 2020 14:03:39 +0200 Subject: [PATCH 56/97] Updated CHANGES.md for 2.13.0 release --- CHANGES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 3ba7cc7f..0a09653b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ -# Unreleased +# 2.13.0 (2020-04-16) - [FIXED] Correctly raise exceptions from `create_database` calls. +- [FIXED] Fix `DeprecationWarning` from `collections`. # 2.12.0 (2019-03-28) From 28cc072bc261b48716dcee1f091944258aecdf89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Tue, 14 Apr 2020 14:04:02 +0200 Subject: [PATCH 57/97] Updated version to 2.13.0 --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 91d3a8d4..fb2c0766 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.12.1-SNAPSHOT +2.13.0 diff --git a/docs/conf.py b/docs/conf.py index 764be732..beb4f08b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '2.12.1-SNAPSHOT' +version = '2.13.0' # The full version, including alpha/beta/rc tags. -release = '2.12.1-SNAPSHOT' +release = '2.13.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 677df2ed..0b8abc5f 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.12.1-SNAPSHOT' +__version__ = '2.13.0' # pylint: disable=wrong-import-position import contextlib From a45d21949baa9d753d899ce926ca68a3d1fd5371 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 16 Apr 2020 16:50:56 +0100 Subject: [PATCH 58/97] Updated version to 2.13.1-SNAPSHOT --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index fb2c0766..ca3389d9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.13.0 +2.13.1-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index beb4f08b..ada059aa 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '2.13.0' +version = '2.13.1-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.13.0' +release = '2.13.1-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 0b8abc5f..0b162070 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.13.0' +__version__ = '2.13.1-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From 06735afacadf8773517cbeaa9843dfa129c3b5da Mon Sep 17 00:00:00 2001 From: Dipjyoti Bisharad Date: Fri, 8 May 2020 20:19:30 +0530 Subject: [PATCH 59/97] Fixed design document creation when partitioned parameter is false (#467) Set default value for 'partitioned' parameter to false when creating a design document. Co-authored-by: dipjyoti bisharad #466 --- CHANGES.md | 4 ++++ src/cloudant/design_document.py | 2 ++ tests/unit/database_partition_tests.py | 11 +++++++++ tests/unit/database_tests.py | 2 +- tests/unit/design_document_tests.py | 31 ++++++++++++++++---------- 5 files changed, 37 insertions(+), 13 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0a09653b..9e7155db 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +# UNRELEASED + +- [FIXED] Set default value for `partitioned` parameter to false when creating a design document. + # 2.13.0 (2020-04-16) - [FIXED] Correctly raise exceptions from `create_database` calls. diff --git a/src/cloudant/design_document.py b/src/cloudant/design_document.py index 34594b95..d3ea387a 100644 --- a/src/cloudant/design_document.py +++ b/src/cloudant/design_document.py @@ -50,6 +50,8 @@ def __init__(self, database, document_id=None, partitioned=False): if partitioned: self.setdefault('options', {'partitioned': True}) + else: + self.setdefault('options', {'partitioned': False}) self._nested_object_names = frozenset(['views', 'indexes', 'lists', 'shows']) for prop in self._nested_object_names: diff --git a/tests/unit/database_partition_tests.py b/tests/unit/database_partition_tests.py index 2c595f65..52b76e0d 100644 --- a/tests/unit/database_partition_tests.py +++ b/tests/unit/database_partition_tests.py @@ -48,6 +48,17 @@ def test_create_partitioned_design_document(self): r.raise_for_status() self.assertTrue(r.json()['options']['partitioned']) + + def test_create_non_partitioned_design_document(self): + ddoc_id = 'empty_ddoc' + + ddoc = DesignDocument(self.db, ddoc_id, partitioned=False) + ddoc.save() + + r = self.db.r_session.get(ddoc.document_url) + r.raise_for_status() + + self.assertFalse(r.json()['options']['partitioned']) def test_partitioned_all_docs(self): for partition_key in self.populate_db_with_partitioned_documents(5, 25): diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index d57ba654..f054dd62 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -364,7 +364,7 @@ def test_retrieve_design_document(self): """ # Get an empty design document object that does not exist remotely local_ddoc = self.db.get_design_document('_design/ddoc01') - self.assertEqual(local_ddoc, {'_id': '_design/ddoc01', 'indexes': {}, + self.assertEqual(local_ddoc, {'_id': '_design/ddoc01', 'indexes': {}, 'options': {'partitioned': False}, 'views': {}, 'lists': {}, 'shows': {}}) # Add the design document to the database map_func = 'function(doc) {\n emit(doc._id, 1); \n}' diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index 7ec3c10e..68a40505 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -353,6 +353,7 @@ def test_fetch_map_reduce(self): self.assertEqual(ddoc_remote, { '_id': '_design/ddoc001', '_rev': ddoc['_rev'], + 'options': {'partitioned': False}, 'lists': {}, 'shows': {}, 'indexes': {}, @@ -415,7 +416,7 @@ def test_fetch_no_views(self): ddoc_remote = DesignDocument(self.db, '_design/ddoc001') ddoc_remote.fetch() self.assertEqual(set(ddoc_remote.keys()), - {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + {'_id', '_rev', 'indexes', 'views', 'options', 'lists', 'shows'}) self.assertEqual(ddoc_remote['_id'], '_design/ddoc001') self.assertTrue(ddoc_remote['_rev'].startswith('1-')) self.assertEqual(ddoc_remote['_rev'], ddoc['_rev']) @@ -432,6 +433,7 @@ def test_fetch_query_views(self): data = { '_id': '_design/ddoc001', 'indexes': {}, + 'options': {'partitioned': False}, 'lists': {}, 'shows': {}, 'language': 'query', @@ -463,6 +465,7 @@ def test_fetch_text_indexes(self): data = { '_id': '_design/ddoc001', 'language': 'query', + 'options': {'partitioned': False}, 'lists': {}, 'shows': {}, 'indexes': {'index001': @@ -499,6 +502,7 @@ def test_fetch_text_indexes_and_query_views(self): 'language': 'query', 'lists': {}, 'shows': {}, + 'options': {'partitioned': False}, 'views': { 'view001': {'map': {'fields': {'name': 'asc', 'age': 'asc'}}, 'reduce': '_count', @@ -687,7 +691,7 @@ def test_save_with_no_views(self): ddoc.save() # Ensure that locally cached DesignDocument contains an # empty views dict. - self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'indexes', 'options', 'views', 'lists', 'shows'}) self.assertEqual(ddoc['_id'], '_design/ddoc001') self.assertTrue(ddoc['_rev'].startswith('1-')) self.assertEqual(ddoc.views, {}) @@ -695,7 +699,7 @@ def test_save_with_no_views(self): # include a views sub-document. resp = self.client.r_session.get(ddoc.document_url) raw_ddoc = response_to_json_dict(resp) - self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) + self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev','options'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -1076,6 +1080,7 @@ def test_fetch_search_index(self): self.assertEqual(ddoc_remote, { '_id': '_design/ddoc001', '_rev': ddoc['_rev'], + 'options': {'partitioned': False}, 'indexes': { 'search001': {'index': search_index}, 'search002': {'index': search_index, 'analyzer': 'simple'}, @@ -1100,7 +1105,7 @@ def test_fetch_no_search_index(self): ddoc_remote = DesignDocument(self.db, '_design/ddoc001') ddoc_remote.fetch() self.assertEqual(set(ddoc_remote.keys()), - {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + {'_id', '_rev', 'indexes', 'options', 'views', 'lists', 'shows'}) self.assertEqual(ddoc_remote['_id'], '_design/ddoc001') self.assertTrue(ddoc_remote['_rev'].startswith('1-')) self.assertEqual(ddoc_remote['_rev'], ddoc['_rev']) @@ -1183,14 +1188,14 @@ def test_save_with_no_search_indexes(self): ddoc.save() # Ensure that locally cached DesignDocument contains an # empty search indexes and views dict. - self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'indexes','options', 'views', 'lists', 'shows'}) self.assertEqual(ddoc['_id'], '_design/ddoc001') self.assertTrue(ddoc['_rev'].startswith('1-')) # Ensure that remotely saved design document does not # include a search indexes sub-document. resp = self.client.r_session.get(ddoc.document_url) raw_ddoc = response_to_json_dict(resp) - self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) + self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev','options'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -1413,6 +1418,7 @@ def test_fetch_list_functions(self): self.assertEqual(ddoc_remote, { '_id': '_design/ddoc001', '_rev': ddoc['_rev'], + 'options': {'partitioned': False}, 'lists': { 'list001': list_func, 'list002': list_func, @@ -1437,7 +1443,7 @@ def test_fetch_no_list_functions(self): ddoc_remote = DesignDocument(self.db, '_design/ddoc001') ddoc_remote.fetch() self.assertEqual(set(ddoc_remote.keys()), - {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + {'_id', '_rev', 'options', 'indexes', 'views', 'lists', 'shows'}) self.assertEqual(ddoc_remote['_id'], '_design/ddoc001') self.assertTrue(ddoc_remote['_rev'].startswith('1-')) self.assertEqual(ddoc_remote['_rev'], ddoc['_rev']) @@ -1452,14 +1458,14 @@ def test_save_with_no_list_functions(self): ddoc = DesignDocument(self.db, '_design/ddoc001') ddoc.save() # Ensure that locally cached DesignDocument contains lists dict - self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'lists', 'shows', 'indexes', 'views'}) + self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'lists', 'options', 'shows', 'indexes', 'views'}) self.assertEqual(ddoc['_id'], '_design/ddoc001') self.assertTrue(ddoc['_rev'].startswith('1-')) # Ensure that remotely saved design document does not # include a lists sub-document. resp = self.client.r_session.get(ddoc.document_url) raw_ddoc = response_to_json_dict(resp) - self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) + self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev','options'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -1714,6 +1720,7 @@ def test_fetch_show_functions(self): self.assertEqual(ddoc_remote, { '_id': '_design/ddoc001', '_rev': ddoc['_rev'], + 'options': {'partitioned': False}, 'lists': {}, 'shows': { 'show001': show_func, @@ -1738,7 +1745,7 @@ def test_fetch_no_show_functions(self): ddoc_remote = DesignDocument(self.db, '_design/ddoc001') ddoc_remote.fetch() self.assertEqual(set(ddoc_remote.keys()), - {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + {'_id', '_rev', 'indexes', 'options', 'views', 'lists', 'shows'}) self.assertEqual(ddoc_remote['_id'], '_design/ddoc001') self.assertTrue(ddoc_remote['_rev'].startswith('1-')) self.assertEqual(ddoc_remote['_rev'], ddoc['_rev']) @@ -1753,14 +1760,14 @@ def test_save_with_no_show_functions(self): ddoc = DesignDocument(self.db, '_design/ddoc001') ddoc.save() # Ensure that locally cached DesignDocument contains shows dict - self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'lists', 'shows', 'indexes', 'views'}) + self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'lists','options', 'shows', 'indexes', 'views'}) self.assertEqual(ddoc['_id'], '_design/ddoc001') self.assertTrue(ddoc['_rev'].startswith('1-')) # Ensure that remotely saved design document does not # include a shows sub-document. resp = self.client.r_session.get(ddoc.document_url) raw_ddoc = response_to_json_dict(resp) - self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) + self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev','options'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) From 37894ceb7bab61667a8c399470a1fcc8d461a27e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Mon, 11 May 2020 16:16:53 +0200 Subject: [PATCH 60/97] Disable false-positive linter error (#469) * Disable false-positive linter error The build has been failing recently with a false positive linter error. Pylint was complaining that a method is not callable, even though the method is defined right on the following line. As I could not find either the reason pylint was failing or a solution that resolves this issue, this commit disables the check for that specific line pylint was failing on. * Fix assertions for non-partitioned db * Pin pylint version --- src/cloudant/feed.py | 2 +- test-requirements.txt | 2 +- tests/unit/design_document_tests.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/cloudant/feed.py b/src/cloudant/feed.py index ef2c90fc..f038ebfe 100644 --- a/src/cloudant/feed.py +++ b/src/cloudant/feed.py @@ -144,7 +144,7 @@ def __next__(self): """ Provides Python3 compatibility. """ - return self.next() + return self.next() # pylint: disable=not-callable def next(self): """ diff --git a/test-requirements.txt b/test-requirements.txt index 88c13ffb..8d95c01d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,5 +2,5 @@ mock==1.3.0 nose sphinx sphinx_rtd_theme -pylint +pylint==2.5.2 flaky diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index 68a40505..52352bcc 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -390,7 +390,8 @@ def test_fetch_dbcopy(self): # before comparison also. Compare the removed values with # the expected content in each case. self.assertEqual(db_copy, ddoc['views']['view002'].pop('dbcopy')) - self.assertEqual({'epi': {'dbcopy': {'view002': db_copy}}}, ddoc_remote.pop('options')) + self.assertEqual({'epi': {'dbcopy': {'view002': db_copy}}, 'partitioned': False}, ddoc_remote.pop('options')) + self.assertEqual({'partitioned': False}, ddoc.pop('options')) self.assertEqual(ddoc_remote, ddoc) self.assertTrue(ddoc_remote['_rev'].startswith('1-')) self.assertEqual(ddoc_remote, { @@ -1553,7 +1554,8 @@ def test_geospatial_index(self): 'indexes': {}, 'views': {}, 'lists': {}, - 'shows': {} + 'shows': {}, + 'options': {'partitioned': False} }) # Document with geospatial point geodoc = Document(self.db, 'doc001') From dfa5a7bb11bafa44b50d4dd3edfbdf7793c8aa07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Wed, 13 May 2020 15:34:42 +0200 Subject: [PATCH 61/97] Remove `CHANGELOG.md` from the PR template (#471) This repo only has a `CHANGES.md` and `CHANGELOG.md` is not applicable. This PR removes the `CHANGELOG.md` from the template. --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 8a9c853a..5b5697c2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,7 +5,7 @@ Thanks for your hard work, please ensure all items are complete before opening. - [ ] Tick to sign-off your agreement to the [Developer Certificate of Origin (DCO) 1.1](../blob/master/DCO1.1.txt) - [ ] Added tests for code changes _or_ test/build only changes -- [ ] Updated the change log file (`CHANGES.md`|`CHANGELOG.md`) _or_ test/build only changes +- [ ] Updated the change log file (`CHANGES.md`) _or_ test/build only changes - [ ] Completed the PR template below: ## Description From 14646b7c38caeceface6e2a35f94392356886b9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Thu, 14 May 2020 13:31:03 +0200 Subject: [PATCH 62/97] Make `create_query_index` always supply `partitioned` (#470) `create_query_index` does not always supply `partitioned` which is always required by the service. This commit changes that so that `partitioned` is now always present in the `create_query_index` requests when defined. This fixes #468 --- CHANGES.md | 1 + docs/getting_started.rst | 8 ++++++-- src/cloudant/database.py | 2 +- src/cloudant/index.py | 6 +++--- tests/unit/index_tests.py | 15 +++++++++++++++ 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9e7155db..6ea317b1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ # UNRELEASED - [FIXED] Set default value for `partitioned` parameter to false when creating a design document. +- [FIXED] Corrected setting of `partitioned` flag for `create_query_index` requests. # 2.13.0 (2020-04-16) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 9cd4b65e..c92387ca 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -284,8 +284,12 @@ when constructing the new ``DesignDocument`` class. ddoc.add_view('myview','function(doc) { emit(doc.foo, doc.bar); }') ddoc.save() -Similarly, to define a partitioned Cloudant Query index you must set the -``partitioned=True`` optional. + +To define a partitioned Cloudant Query index you may set the +``partitioned=True`` optional, but it is not required as the index will be +partitioned by default in a partitioned database. Conversely, you must +set the ``partitioned=False`` optional if you wish to create a global +(non-partitioned) index in a partitioned database. .. code-block:: python diff --git a/src/cloudant/database.py b/src/cloudant/database.py index 17978468..3586c93f 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -1100,7 +1100,7 @@ def create_query_index( design_document_id=None, index_name=None, index_type='json', - partitioned=False, + partitioned=None, **kwargs ): """ diff --git a/src/cloudant/index.py b/src/cloudant/index.py index 051c004b..c66c7ac2 100644 --- a/src/cloudant/index.py +++ b/src/cloudant/index.py @@ -47,7 +47,7 @@ class Index(object): :func:`~cloudant.database.CloudantDatabase.create_query_index`. """ - def __init__(self, database, design_document_id=None, name=None, partitioned=False, **kwargs): + def __init__(self, database, design_document_id=None, name=None, partitioned=None, **kwargs): self._database = database self._r_session = self._database.r_session self._ddoc_id = design_document_id @@ -154,8 +154,8 @@ def create(self): self._def_check() payload['index'] = self._def - if self._partitioned: - payload['partitioned'] = True + if self._partitioned is not None: + payload['partitioned'] = bool(self._partitioned) headers = {'Content-Type': 'application/json'} resp = self._r_session.post( diff --git a/tests/unit/index_tests.py b/tests/unit/index_tests.py index 4361644f..29a3d51c 100644 --- a/tests/unit/index_tests.py +++ b/tests/unit/index_tests.py @@ -554,6 +554,21 @@ def test_create_a_search_index_invalid_selector_value(self): '<{} \'dict\'>'.format('type' if PY2 else 'class') ) + def test_create_unpartitioned_query_index(self): + """ + Test that create_query_index works on an unpartitioned database + """ + ddoc = DesignDocument(self.db, document_id="unpartitioned_query_index_ddoc") + ddoc["language"] = "query" + ddoc.save() + index = self.db.create_query_index( + design_document_id="_design/unpartitioned_query_index_ddoc", + fields=["key"], + partitioned=False + ) + index.create() + self.assertGreater(len(self.db.get_query_indexes()), 0) + def test_search_index_via_query(self): """ Test that a created TEXT index will produce expected query results. From d080687766a3101b1e876957674983b956851a76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eigel=20Ildik=C3=B3?= Date: Fri, 12 Jun 2020 14:36:04 +0200 Subject: [PATCH 63/97] Add private endpoint usage with IAM to docs (#475) --- docs/getting_started.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index c92387ca..37c5b92c 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -111,6 +111,13 @@ You can easily connect to your Cloudant account using an IAM API key: # Authenticate using an IAM API key client = Cloudant.iam(ACCOUNT_NAME, API_KEY, connect=True) +If you need to authenticate to a server outside of the `cloudant.com` domain, you can use the `url` parameter: + +.. code-block:: python + + # Authenticate using an IAM API key to an account outside of the cloudant.com domain + client = Cloudant.iam(None, API_KEY, url='https://private.endpoint.example', connect=True) + **************** Resource sharing From 4e2c377f9c955f04b2d911d4480a00a308ce581e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Mon, 17 Aug 2020 10:36:58 +0200 Subject: [PATCH 64/97] Make setup.py compatible with python 2 (#477) --- CHANGES.md | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 6ea317b1..0fe2e075 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ - [FIXED] Set default value for `partitioned` parameter to false when creating a design document. - [FIXED] Corrected setting of `partitioned` flag for `create_query_index` requests. +- [FIXED] Added a workaround for installation on Python 2. # 2.13.0 (2020-04-16) diff --git a/setup.py b/setup.py index e02192cd..ac2e3876 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ """ +from io import open from os import path from setuptools import setup, find_packages From 296cf8110da40dfd2528f5431340a79d7e5fdc1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Mon, 17 Aug 2020 16:09:52 +0200 Subject: [PATCH 65/97] Prepare 2.14.0 release --- CHANGES.md | 2 ++ VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0fe2e075..93cc5339 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # UNRELEASED +# 2.14.0 (2020-08-17) + - [FIXED] Set default value for `partitioned` parameter to false when creating a design document. - [FIXED] Corrected setting of `partitioned` flag for `create_query_index` requests. - [FIXED] Added a workaround for installation on Python 2. diff --git a/VERSION b/VERSION index ca3389d9..edcfe40d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.13.1-SNAPSHOT +2.14.0 diff --git a/docs/conf.py b/docs/conf.py index ada059aa..44f27314 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '2.13.1-SNAPSHOT' +version = '2.14.0' # The full version, including alpha/beta/rc tags. -release = '2.13.1-SNAPSHOT' +release = '2.14.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 0b162070..193a099c 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.13.1-SNAPSHOT' +__version__ = '2.14.0' # pylint: disable=wrong-import-position import contextlib From 3f4ae5d7c3dc79a5b85a2b481b998570c262b0fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Mon, 17 Aug 2020 16:42:49 +0200 Subject: [PATCH 66/97] Remove "Unreleased" from CHANGES.md --- CHANGES.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 93cc5339..9188a90d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,3 @@ -# UNRELEASED - # 2.14.0 (2020-08-17) - [FIXED] Set default value for `partitioned` parameter to false when creating a design document. From 8c703409053fc2a47d25ba9e3da43b1527370baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Tue, 18 Aug 2020 17:15:37 +0200 Subject: [PATCH 67/97] Update version to 2.14.1-SNAPSHOT (#480) --- CHANGES.md | 2 ++ VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9188a90d..93cc5339 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,5 @@ +# UNRELEASED + # 2.14.0 (2020-08-17) - [FIXED] Set default value for `partitioned` parameter to false when creating a design document. diff --git a/VERSION b/VERSION index edcfe40d..20738ec2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.14.0 +2.14.1-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index 44f27314..1e338e6d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '2.14.0' +version = '2.14.1-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.14.0' +release = '2.14.1-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 193a099c..7e011096 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.14.0' +__version__ = '2.14.1-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From e80a07eb8a9723e2179872e832f0a47cd1a4fce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Tue, 18 Aug 2020 18:28:39 +0200 Subject: [PATCH 68/97] Remove Python 2 compatibility (#478) Python 2 was EoLed back in January. This commit removes it from the list of supported environments. --- CHANGES.md | 1 + README.md | 2 +- docs/compatibility.rst | 1 - setup.py | 2 -- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 93cc5339..b40e6fb1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,5 @@ # UNRELEASED +- [REMOVED] Removed Python 2 compatibility from the supported environments. # 2.14.0 (2020-08-17) diff --git a/README.md b/README.md index 0cc5edcf..7eaf83ba 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Build Status](https://travis-ci.org/cloudant/python-cloudant.svg?branch=master)](https://travis-ci.org/cloudant/python-cloudant) [![Readthedocs](https://readthedocs.org/projects/pip/badge/)](http://python-cloudant.readthedocs.io) -[![Compatibility](https://img.shields.io/badge/python-2.7%2C%203.5-blue.svg)](http://python-cloudant.readthedocs.io/en/latest/compatibility.html) +[![Compatibility](https://img.shields.io/badge/python-3.5-blue.svg)](http://python-cloudant.readthedocs.io/en/latest/compatibility.html) [![pypi](https://img.shields.io/pypi/v/cloudant.svg)](https://pypi.python.org/pypi/cloudant) This is the official Cloudant library for Python. diff --git a/docs/compatibility.rst b/docs/compatibility.rst index 1bb98cce..b756a65e 100644 --- a/docs/compatibility.rst +++ b/docs/compatibility.rst @@ -11,5 +11,4 @@ Note that some features are Cloudant specific. This library has been tested with the following versions of Python -* `Python™ 2.7 `_ * `Python™ 3.5 `_ diff --git a/setup.py b/setup.py index ac2e3876..a8ac4ef8 100644 --- a/setup.py +++ b/setup.py @@ -55,8 +55,6 @@ 'Topic :: Software Development :: Libraries', 'Development Status :: 5 - Production/Stable', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5' ] From 943ad117d0cb416ffb589c35e144b14eb32726f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Fri, 4 Sep 2020 11:10:36 +0200 Subject: [PATCH 69/97] Remove incorrect restriction from docs for bookmarks (#481) --- CHANGES.md | 1 + src/cloudant/database.py | 3 +-- src/cloudant/query.py | 9 +++------ src/cloudant/result.py | 3 +-- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b40e6fb1..e68156a0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ # UNRELEASED - [REMOVED] Removed Python 2 compatibility from the supported environments. +- [FIXED] Fixed the documentation for `bookmarks`. # 2.14.0 (2020-08-17) diff --git a/src/cloudant/database.py b/src/cloudant/database.py index 3586c93f..ccde12da 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -1242,8 +1242,7 @@ def get_query_result(self, selector, fields=None, raw_result=False, wrapped in a QueryResult or if the response JSON is returned. Defaults to False. :param str bookmark: A string that enables you to specify which page of - results you require. Only valid for queries using indexes of type - *text*. + results you require. :param int limit: Maximum number of results returned. Only valid if used with ``raw_result=True``. :param int page_size: Sets the page size for result iteration. Default diff --git a/src/cloudant/query.py b/src/cloudant/query.py index d6a0aebc..2362f80d 100644 --- a/src/cloudant/query.py +++ b/src/cloudant/query.py @@ -69,8 +69,7 @@ class Query(dict): :param CloudantDatabase database: A Cloudant database instance used by the Query. :param str bookmark: A string that enables you to specify which page of - results you require. Only valid for queries using indexes of type - *text*. + results you require. :param list fields: A list of fields to be returned by the query. :param int limit: Maximum number of results returned. :param int r: Read quorum needed for the result. Each document is read from @@ -141,8 +140,7 @@ def __call__(self, **kwargs): and set ``raw_result=True`` instead. :param str bookmark: A string that enables you to specify which page of - results you require. Only valid for queries using indexes of type - *text*. + results you require. :param list fields: A list of fields to be returned by the query. :param int limit: Maximum number of results returned. :param int r: Read quorum needed for the result. Each document is read @@ -204,8 +202,7 @@ def custom_result(self, **options): data = rslt[100:200] :param str bookmark: A string that enables you to specify which page of - results you require. Only valid for queries using indexes of type - *text*. + results you require. :param list fields: A list of fields to be returned by the query. :param int page_size: Sets the page size for result iteration. Default is 100. diff --git a/src/cloudant/result.py b/src/cloudant/result.py index d3dd9a09..e484cd07 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -488,8 +488,7 @@ class QueryResult(Result): :param query: A reference to the query callable that returns the JSON content result to be wrapped. :param str bookmark: A string that enables you to specify which page of - results you require. Only valid for queries using indexes of type - *text*. + results you require. :param list fields: A list of fields to be returned by the query. :param int page_size: Sets the page size for result iteration. Default is 100. From 1d634d69c48e23932a1ae7b0b5befefe1c274047 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 20 Nov 2020 12:18:30 +0000 Subject: [PATCH 70/97] Correct replicator states Add the `failed` state to `follow_replication`. The replicator tests were missing some other states introduced in CouchDB 2.1 scheduler. --- CHANGES.md | 1 + src/cloudant/replicator.py | 10 ++++++++-- tests/unit/replicator_tests.py | 9 ++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e68156a0..40299e13 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ # UNRELEASED - [REMOVED] Removed Python 2 compatibility from the supported environments. - [FIXED] Fixed the documentation for `bookmarks`. +- [FIXED] Also exit `follow_replication` for `failed` state. # 2.14.0 (2020-08-17) diff --git a/src/cloudant/replicator.py b/src/cloudant/replicator.py index e3ec3550..f1b44643 100644 --- a/src/cloudant/replicator.py +++ b/src/cloudant/replicator.py @@ -193,7 +193,12 @@ def update_state(): repl_doc, state = update_state() if repl_doc: yield repl_doc - if state is not None and state in ['error', 'completed']: + # This is a little awkward, since 2.1 the terminal states are + # "failed" and "completed", so those should be the exit states, but + # for backwards compatibility with older versions "error" is also + # needed. The code has always exited for "error" state even long + # after 2.1 was available so that behaviour is retained. + if state is not None and state in ['error', 'failed', 'completed']: return # Now listen on changes feed for the state @@ -202,7 +207,8 @@ def update_state(): repl_doc, state = update_state() if repl_doc is not None: yield repl_doc - if state is not None and state in ['error', 'completed']: + # See note about these states + if state is not None and state in ['error', 'failed', 'completed']: return def stop_replication(self, repl_id): diff --git a/tests/unit/replicator_tests.py b/tests/unit/replicator_tests.py index 28d78fab..9eb56b56 100644 --- a/tests/unit/replicator_tests.py +++ b/tests/unit/replicator_tests.py @@ -322,12 +322,14 @@ def test_retrieve_replication_state(self): ) self.replication_ids.append(repl_id) repl_state = None - valid_states = ['completed', 'error', 'triggered', 'running', None] + # note triggered is for versions prior to 2.1 + valid_states = ['completed', 'error', 'initializing', 'triggered', 'pending', 'running', 'failed', 'crashing', None] finished = False + # Wait for 5 minutes or a terminal replication state for _ in range(300): repl_state = self.replicator.replication_state(repl_id) self.assertTrue(repl_state in valid_states) - if repl_state in ('error', 'completed'): + if repl_state in ('error', 'failed', 'completed'): finished = True break time.sleep(1) @@ -407,7 +409,8 @@ def test_follow_replication(self): repl_id ) self.replication_ids.append(repl_id) - valid_states = ('completed', 'error', 'triggered', 'running', None) + # note triggered is for versions prior to 2.1 + valid_states = ['completed', 'error', 'initializing', 'triggered', 'pending', 'running', 'failed', 'crashing', None] repl_states = [] if 'scheduler' in self.client.features(): state_key = 'state' From a105ae7c191328ad8bdcc75bce12e9ba5f5ad72f Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Fri, 20 Nov 2020 13:04:15 -0500 Subject: [PATCH 71/97] Fix paging logic to check 'id' field exists for grouped view queries Added CHANGES entry Dict members aren't checked by `hasattr`. Use `get` instead and keep the value so we don't have to fetch it twice. --- CHANGES.md | 1 + src/cloudant/result.py | 5 +++-- tests/unit/database_tests.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 40299e13..ad1d3788 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ - [REMOVED] Removed Python 2 compatibility from the supported environments. - [FIXED] Fixed the documentation for `bookmarks`. - [FIXED] Also exit `follow_replication` for `failed` state. +- [FIXED] Fixed result paging for grouped view queries. # 2.14.0 (2020-08-17) diff --git a/src/cloudant/result.py b/src/cloudant/result.py index e484cd07..319d7268 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -397,9 +397,10 @@ def _iterator(self, response): # if we are in a view, keys could be duplicate so we # need to start from the right docid - if last['id']: + last_doc_id = last.get('id') + if last_doc_id is not None: response = self._call(startkey=last['key'], - startkey_docid=last['id']) + startkey_docid=last_doc_id) # reduce result keys are unique by definition else: response = self._call(startkey=last['key']) diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index f054dd62..c8c5cfe0 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -408,6 +408,34 @@ def test_retrieve_view_results(self): self.assertIsInstance(rslt, Result) self.assertEqual(rslt[:1], rslt['julia099']) + def test_retrieve_grouped_view_result_with_page_size(self): + """ + Test retrieving Result wrapped output from a design document grouped view + that uses a custom page size + + The view used here along with group=True will generate rows of + data where each key will be grouped into groups of 2. Such as: + {'key': 0, 'value': 2}, + {'key': 1, 'value': 2}, + ... + """ + map_func = 'function(doc) {\n emit(Math.floor(doc.age / 2), 1); \n}' + data = {'_id': '_design/ddoc01','views': {'view01': {"map": map_func, "reduce": "_count"}}} + self.db.create_document(data) + self.populate_db_with_documents(5) + + rslt = self.db.get_view_result( + '_design/ddoc01', + 'view01', + group=True, + page_size=1) + self.assertIsInstance(rslt, Result) + i = 0 + for row in rslt: + self.assertIsNotNone(row) + self.assertEqual(row['key'], i) + i += 1 + def test_retrieve_raw_view_results(self): """ Test retrieving raw output from a design document view From 43254c75cafdc19b098c5b04f19907879352a197 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Fri, 8 Jan 2021 12:17:26 -0500 Subject: [PATCH 72/97] Add migration guide (#488) * Added migration guide to cloudant-python-sdk library * Updated README with migration guide section --- MIGRATION.md | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 5 +++ 2 files changed, 100 insertions(+) create mode 100644 MIGRATION.md diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 00000000..5503e2cc --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,95 @@ +# Migrating to the `cloudant-python-sdk` library +This document is to assist in migrating from the `python-cloudant` (module: `cloudant`) to the newly supported [`cloudant-python-sdk`](https://github.com/IBM/cloudant-python-sdk) (module: `ibmcloudant`). + +## Initializing the client connection +There are several ways to create a client connection in `cloudant-python-sdk`: +1. [Environment variables](https://github.com/IBM/cloudant-python-sdk#authentication-with-environment-variables) +2. [External configuration file](https://github.com/IBM/cloudant-python-sdk#authentication-with-external-configuration) +3. [Programmatically](https://github.com/IBM/cloudant-python-sdk#programmatic-authentication) + +[See the README](https://github.com/IBM/cloudant-python-sdk#code-examples) for code examples on using environment variables. + +## Other differences +1. The `cloudant-python-sdk` library does not support local dictionary caching of database and document objects. +1. There are no context managers in `cloudant-python-sdk`. To reproduce the behaviour of the `python-cloudant` +context managers in `cloudant-python-sdk` users need to explicitly call the specific operations against the +remote HTTP API. For example, in the case of the document context manager, this would mean doing both a `get_document` +to fetch and a `put_document` to save. +1. In `cloudant-python-sdk` View, Search, and Query (aka `_find` endpoint) operation responses contain raw JSON +content like using `raw_result=True` in `python-cloudant`. + +## Request mapping +Here's a list of the top 5 most frequently used `python-cloudant` operations and the `cloudant-python-sdk` equivalent API operation documentation link: + +| `python-cloudant` operation | `cloudant-python-sdk` API operation documentation link | +|---------------------------------------|---------------------------------| +|`Document('db_name', 'docid').fetch()` |[`getDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdocument)| +|`db.get_view_result()` |[`postView`](https://cloud.ibm.com/apidocs/cloudant?code=python#postview)| +|`db.get_query_result()` |[`postFind`](https://cloud.ibm.com/apidocs/cloudant?code=python#postfind)| +| `doc.exists()` |[`headDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#headdocument)| +|`Document('db_name', 'docid').save()` |[`putDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#putdocument)| + + +[A table](#reference-table) with the whole list of operations is provided at the end of this guide. + +The `cloudant-python-sdk` library is generated from a more complete API spec and provides a significant number of operations that do not exist in `python-cloudant`. See [the IBM Cloud API Documentation](https://cloud.ibm.com/apidocs/cloudant) to review request parameter and body options, code examples, and additional details for every endpoint. + +## Known Issues +There's an [outline of known issues](https://github.com/IBM/cloudant-python-sdk/blob/master/KNOWN_ISSUES.md) in the `cloudant-python-sdk` repository. + +## Reference table +The table below contains a list of `python-cloudant` functions and the `cloudant-python-sdk` equivalent API operation documentation link. The `cloudant-python-sdk` operation documentation link will contain the new function in a code sample e.g. `getServerInformation` link will contain a code example with `get_server_information()`. + +**Note:** There are many API operations included in the new `cloudant-python-sdk` that are not available in the `python-cloudant` library. The [API documentation](https://cloud.ibm.com/apidocs/cloudant?code=python) contains the full list of operations. + +| `python-cloudant` function | `cloudant-python-sdk` API operation documentation link | +|-----------------|---------------------| +|`metadata()`|[getServerInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getserverinformation)| +|`all_dbs()`|[getAllDbs](https://cloud.ibm.com/apidocs/cloudant?code=python#getalldbs)| +|`db_updates()/infinite_db_updates()`|[getDbUpdates](https://cloud.ibm.com/apidocs/cloudant?code=python#getdbupdates)| +|`Replicator.create_replication()`|[postReplicate](https://cloud.ibm.com/apidocs/cloudant?code=python#postreplicate)| +|`Scheduler.get_doc()`|[getSchedulerDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerdocument)| +|`Scheduler.list_jobs()`|[getSchedulerJobs](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerjobs)| +|`session()`|[getSessionInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getsessioninformation)| +|`uuids()`|[getUuids](https://cloud.ibm.com/apidocs/cloudant?code=python#getuuids)| +|`db.delete()`|[deleteDatabase](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedatabase)| +|`db.metadata()`|[getDatabaseInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getdatabaseinformation)| +|`db.create_document()`|[postDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#postdocument)| +|`db.create()`|[putDatabase](https://cloud.ibm.com/apidocs/cloudant?code=python#putdatabase)| +|`db.all_docs()/db.keys()`|[postAllDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postalldocs)| +|`db.bulk_docs()`|[postBulkDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postbulkdocs)| +|`db.changes()/db.infinite_changes()`|[postChanges](https://cloud.ibm.com/apidocs/cloudant?code=python#postchanges)| +|`DesignDocument(db, '_design/doc').delete()`|[deleteDesignDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedesigndocument)| +|`db.get_design_document()/DesignDocument(db, '_design/doc').fetch()`|[getDesignDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getdesigndocument)| +|`DesignDocument(db, '_design/doc').save()`|[putDesignDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putdesigndocument)| +|`DesignDocument(db, '_design/doc').info()`|[getDesignDocumentInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getdesigndocumentinformation)| +|`db.get_search_result()`|[postSearch](https://cloud.ibm.com/apidocs/cloudant?code=python#postsearch)| +|`db.get_view_result()`|[postView](https://cloud.ibm.com/apidocs/cloudant?code=python#postview)| +|`db.list_design_documents()`|[postDesignDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postdesigndocs)| +|`db.get_query_result()`|[postFind](https://cloud.ibm.com/apidocs/cloudant?code=python#postfind)| +|`db.get_query_indexes()`|[getIndexesInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getindexesinformation)| +|`db.create_query_index()`|[postIndex](https://cloud.ibm.com/apidocs/cloudant?code=python#postindex)| +|`db.delete_query_index()`|[deleteIndex](https://cloud.ibm.com/apidocs/cloudant?code=python#deleteindex)| +|`Document(db, 'docid').delete()`|[getLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getlocaldocument)| +|`Document(db, 'docid').create()`|[putLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putlocaldocument)| +|`db.missing_revisions()`|[postMissingRevs](https://cloud.ibm.com/apidocs/cloudant?code=python#postmissingrevs)| +|`db.partition_metadata()`|[getPartitionInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getpartitioninformation)| +|`db.partitioned_all_docs()`|[postPartitionAllDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionalldocs)| +|`db.get_partitioned_search_result()`|[postPartitionSearch](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionsearch)| +|`db.get_partitioned_view_result()`|[postPartitionView](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionview)| +|`db.get_partitioned_query_result()`|[postPartitionFind](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionfind)| +|`db.revisions_diff()`|[postRevsDiff](https://cloud.ibm.com/apidocs/cloudant?code=python#postrevsdiff)| +|`db.get_security_document()/db.security_document()`|[getSecurity](https://cloud.ibm.com/apidocs/cloudant?code=python#getsecurity)| +|`db.share_database()`|[putSecurity](https://cloud.ibm.com/apidocs/cloudant?code=python#putsecurity)| +|`db.shards()`|[getShardsInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getshardsinformation)| +|`Document(db, 'docid').delete()`|[getLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getlocaldocument)| +|`Document(db, 'docid').fetch()`|[getDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getdocument)| +|`Document(db, 'docid').exists()`|[headDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#headdocument)| +|`Document(db, 'docid').save()`|[putDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putdocument)| +|`Document(db, 'docid').delete_attachment()`|[deleteAttachment](https://cloud.ibm.com/apidocs/cloudant?code=python#deleteattachment)| +|`Document(db, 'docid').get_attachment()`|[getAttachment](https://cloud.ibm.com/apidocs/cloudant?code=python#getattachment)| +|`Document(db, 'docid').put_attachment()`|[putAttachment](https://cloud.ibm.com/apidocs/cloudant?code=python#putattachment)| +|`generate_api_key()`|[postApiKeys](https://cloud.ibm.com/apidocs/cloudant?code=python#postapikeys)| +|`SecurityDocument().save()`|[putCloudantSecurityConfiguration](https://cloud.ibm.com/apidocs/cloudant?code=python#putcloudantsecurityconfiguration)| +|`cors_configuration()/cors_origin()`|[getCorsInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getcorsinformation)| +|`update_cors_configuration()`|[putCorsConfiguration](https://cloud.ibm.com/apidocs/cloudant?code=python#putcorsconfiguration)| diff --git a/README.md b/README.md index 7eaf83ba..f12b2814 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ This is the official Cloudant library for Python. * [Using in Other Projects](#using-in-other-projects) * [License](#license) * [Issues](#issues) +* [Migrating to `cloudant-python-sdk` library](#migrating-to-cloudant-python-sdk-library) ## Installation and Usage @@ -87,3 +88,7 @@ to see if the problem has already been reported. Note that the default search includes only open issues, but it may already have been closed. * Cloudant customers should contact Cloudant support for urgent issues. * When opening a new issue [here in github](../../issues) please complete the template fully. + +## Migrating to `cloudant-python-sdk` library +We have a newly supported Cloudant Python SDK named [cloudant-python-sdk](https://github.com/IBM/cloudant-python-sdk). +For advice on migrating from this module see [MIGRATION.md](MIGRATION.md). From 20cefd5ddd6c2975224ddfd4831de98a238f6c2f Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Thu, 28 Jan 2021 10:43:30 -0500 Subject: [PATCH 73/97] Update local document operations in migration guide (#489) --- MIGRATION.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index 5503e2cc..53845e7d 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -70,8 +70,9 @@ The table below contains a list of `python-cloudant` functions and the `cloudant |`db.get_query_indexes()`|[getIndexesInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getindexesinformation)| |`db.create_query_index()`|[postIndex](https://cloud.ibm.com/apidocs/cloudant?code=python#postindex)| |`db.delete_query_index()`|[deleteIndex](https://cloud.ibm.com/apidocs/cloudant?code=python#deleteindex)| -|`Document(db, 'docid').delete()`|[getLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getlocaldocument)| -|`Document(db, 'docid').create()`|[putLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putlocaldocument)| +|`Document(db, '_local/docid').fetch()`|[getLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getlocaldocument)| +|`Document(db, '_local/docid').save()`|[putLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putlocaldocument)| +|`Document(db, '_local/docid').delete()`|[deleteLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#deletelocaldocument)| |`db.missing_revisions()`|[postMissingRevs](https://cloud.ibm.com/apidocs/cloudant?code=python#postmissingrevs)| |`db.partition_metadata()`|[getPartitionInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getpartitioninformation)| |`db.partitioned_all_docs()`|[postPartitionAllDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionalldocs)| @@ -82,7 +83,7 @@ The table below contains a list of `python-cloudant` functions and the `cloudant |`db.get_security_document()/db.security_document()`|[getSecurity](https://cloud.ibm.com/apidocs/cloudant?code=python#getsecurity)| |`db.share_database()`|[putSecurity](https://cloud.ibm.com/apidocs/cloudant?code=python#putsecurity)| |`db.shards()`|[getShardsInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getshardsinformation)| -|`Document(db, 'docid').delete()`|[getLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getlocaldocument)| +|`Document(db, 'docid').delete()`|[deleteDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedocument)| |`Document(db, 'docid').fetch()`|[getDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getdocument)| |`Document(db, 'docid').exists()`|[headDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#headdocument)| |`Document(db, 'docid').save()`|[putDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putdocument)| From 33781463ff85c8bb535bd6ede2049e4d3bec6c4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eigel=20Ildik=C3=B3?= Date: Wed, 10 Feb 2021 16:47:14 +0100 Subject: [PATCH 74/97] Add replication and scheduler operations to the migration guide (#490) --- MIGRATION.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index 53845e7d..29be9810 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -1,4 +1,4 @@ -# Migrating to the `cloudant-python-sdk` library +# Migrating to the `cloudant-python-sdk` library This document is to assist in migrating from the `python-cloudant` (module: `cloudant`) to the newly supported [`cloudant-python-sdk`](https://github.com/IBM/cloudant-python-sdk) (module: `ibmcloudant`). ## Initializing the client connection @@ -11,11 +11,11 @@ There are several ways to create a client connection in `cloudant-python-sdk`: ## Other differences 1. The `cloudant-python-sdk` library does not support local dictionary caching of database and document objects. -1. There are no context managers in `cloudant-python-sdk`. To reproduce the behaviour of the `python-cloudant` -context managers in `cloudant-python-sdk` users need to explicitly call the specific operations against the -remote HTTP API. For example, in the case of the document context manager, this would mean doing both a `get_document` +1. There are no context managers in `cloudant-python-sdk`. To reproduce the behaviour of the `python-cloudant` +context managers in `cloudant-python-sdk` users need to explicitly call the specific operations against the +remote HTTP API. For example, in the case of the document context manager, this would mean doing both a `get_document` to fetch and a `put_document` to save. -1. In `cloudant-python-sdk` View, Search, and Query (aka `_find` endpoint) operation responses contain raw JSON +1. In `cloudant-python-sdk` View, Search, and Query (aka `_find` endpoint) operation responses contain raw JSON content like using `raw_result=True` in `python-cloudant`. ## Request mapping @@ -47,8 +47,11 @@ The table below contains a list of `python-cloudant` functions and the `cloudant |`metadata()`|[getServerInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getserverinformation)| |`all_dbs()`|[getAllDbs](https://cloud.ibm.com/apidocs/cloudant?code=python#getalldbs)| |`db_updates()/infinite_db_updates()`|[getDbUpdates](https://cloud.ibm.com/apidocs/cloudant?code=python#getdbupdates)| -|`Replicator.create_replication()`|[postReplicate](https://cloud.ibm.com/apidocs/cloudant?code=python#postreplicate)| +|`Replicator.stop_replication()`|[deleteReplicationDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#deletereplicationdocument)| +|`Replicator.replication_state()`|[getReplicationDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getreplicationdocument)| +|`Replicator.create_replication()`|[putReplicationDocument](https://cloud.ibm.com/apidocs/cloudant?code=#putreplicationdocument)| |`Scheduler.get_doc()`|[getSchedulerDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerdocument)| +|`Scheduler.list_docs()`|[getSchedulerDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerdocs)| |`Scheduler.list_jobs()`|[getSchedulerJobs](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerjobs)| |`session()`|[getSessionInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getsessioninformation)| |`uuids()`|[getUuids](https://cloud.ibm.com/apidocs/cloudant?code=python#getuuids)| From 18245a94a8b6551517cf5134efefc5a84b516f59 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 14 May 2021 15:51:34 +0100 Subject: [PATCH 75/97] Make IAM token server configurable in tests --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index ddf21dd4..d88f017c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,6 +13,7 @@ def getEnvForSuite(suiteName) { case 'iam': // Setting IAM_API_KEY forces tests to run using an IAM enabled client. envVars.add("IAM_API_KEY=$DB_IAM_API_KEY") + envVars.add("IAM_TOKEN_URL=$SDKS_TEST_IAM_URL") break case 'cookie': case 'simplejson': From c409805ccb148d2f0da2e3a2bd5e72bbed1ffed9 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 14 May 2021 16:01:03 +0100 Subject: [PATCH 76/97] Add new test server env vars --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d88f017c..e655a45c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ def getEnvForSuite(suiteName) { // Base environment variables def envVars = [ - "CLOUDANT_ACCOUNT=$DB_USER", + "DB_URL=${SDKS_TEST_SERVER_URL}", "RUN_CLOUDANT_TESTS=1", "SKIP_DB_UPDATES=1" // Disable pending resolution of case 71610 ] @@ -29,8 +29,8 @@ def setupPythonAndTest(pythonVersion, testSuite) { // Unstash the source on this node unstash name: 'source' // Set up the environment and test - withCredentials([usernamePassword(credentialsId: 'clientlibs-test', usernameVariable: 'DB_USER', passwordVariable: 'DB_PASSWORD'), - string(credentialsId: 'clientlibs-test-iam', variable: 'DB_IAM_API_KEY')]) { + withCredentials([usernamePassword(credentialsId: 'testServerLegacy', usernameVariable: 'DB_USER', passwordVariable: 'DB_PASSWORD'), + string(credentialsId: 'testServerIamApiKey', variable: 'DB_IAM_API_KEY')]) { withEnv(getEnvForSuite("${testSuite}")) { try { sh """ From ebcb98c544c2e24b284e0c10970ba3d72c1783d7 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Mon, 17 May 2021 12:04:34 +0100 Subject: [PATCH 77/97] Fix client tests to not always use account --- tests/unit/client_tests.py | 40 ++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index 2726de33..07c4159b 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2020 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,6 +27,7 @@ import sys import unittest from time import sleep +from urllib.parse import urlparse import mock import requests @@ -213,7 +214,7 @@ def test_auto_renew_enabled_with_auto_connect(self): @skip_if_not_cookie_auth def test_session(self): """ - Test getting session information. + Test getting session information. Session info is None if CouchDB Admin Party mode was selected. """ try: @@ -563,7 +564,7 @@ def test_get_cached_db_object_via_get(self): self.client.connect() # Default returns None self.assertIsNone(self.client.get('no_such_db')) - # Creates the database remotely and adds it to the + # Creates the database remotely and adds it to the # client database cache db = self.client.create_database(dbname) # Locally cached database object is returned @@ -702,7 +703,7 @@ def test_cloudant_context_helper(self): Test that the cloudant context helper works as expected. """ try: - with cloudant(self.user, self.pwd, account=self.account) as c: + with cloudant(self.user, self.pwd, url=self.url) as c: self.assertIsInstance(c, Cloudant) self.assertIsInstance(c.r_session, requests.Session) except Exception as err: @@ -718,7 +719,7 @@ def test_cloudant_bluemix_context_helper_with_legacy_creds(self): 'credentials': { 'username': self.user, 'password': self.pwd, - 'host': '{0}.cloudant.com'.format(self.account), + 'host': urlparse(self.url).hostname, 'port': 443, 'url': self.url }, @@ -744,7 +745,7 @@ def test_cloudant_bluemix_context_helper_with_iam(self): 'credentials': { 'apikey': self.iam_api_key, 'username': self.user, - 'host': '{0}.cloudant.com'.format(self.account), + 'host': urlparse(self.url).hostname, 'port': 443, 'url': self.url }, @@ -766,7 +767,7 @@ def test_cloudant_bluemix_context_helper_raise_error_for_missing_iam_and_creds(s instance_name = 'Cloudant NoSQL DB-lv' vcap_services = {'cloudantNoSQLDB': [{ 'credentials': { - 'host': '{0}.cloudant.com'.format(self.account), + 'host': urlparse(self.url).hostname, 'port': 443, 'url': self.url }, @@ -795,7 +796,7 @@ def test_cloudant_bluemix_dedicated_context_helper(self): 'credentials': { 'username': self.user, 'password': self.pwd, - 'host': '{0}.cloudant.com'.format(self.account), + 'host': urlparse(self.url).hostname, 'port': 443, 'url': self.url }, @@ -818,10 +819,10 @@ def test_constructor_with_account(self): """ # Ensure that the client is new del self.client - self.client = Cloudant(self.user, self.pwd, account=self.account) + self.client = Cloudant('user', 'pass', account='foo') self.assertEqual( self.client.server_url, - 'https://{0}.cloudant.com'.format(self.account) + 'https://foo.cloudant.com' ) @skip_if_not_cookie_auth @@ -835,7 +836,7 @@ def test_bluemix_constructor_with_legacy_creds(self): 'credentials': { 'username': self.user, 'password': self.pwd, - 'host': '{0}.cloudant.com'.format(self.account), + 'host': urlparse(self.url).hostname, 'port': 443, 'url': self.url }, @@ -870,7 +871,7 @@ def test_bluemix_constructor_with_iam(self): 'credentials': { 'apikey': self.iam_api_key, 'username': self.user, - 'host': '{0}.cloudant.com'.format(self.account), + 'host': urlparse(self.url).hostname, 'port': 443 }, 'name': instance_name @@ -901,7 +902,7 @@ def test_bluemix_constructor_specify_instance_name(self): 'credentials': { 'username': self.user, 'password': self.pwd, - 'host': '{0}.cloudant.com'.format(self.account), + 'host': urlparse(self.url).hostname, 'port': 443, 'url': self.url }, @@ -934,7 +935,7 @@ def test_bluemix_constructor_with_multiple_services(self): { 'credentials': { 'apikey': '1234api', - 'host': '{0}.cloudant.com'.format(self.account), + 'host': urlparse(self.url).hostname, 'port': 443, 'url': self.url }, @@ -973,10 +974,11 @@ def test_connect_headers(self): """ try: self.client.connect() - self.assertEqual( - self.client.r_session.headers['X-Cloudant-User'], - self.account - ) + if (self.account): + self.assertEqual( + self.client.r_session.headers['X-Cloudant-User'], + self.account + ) agent = self.client.r_session.headers.get('User-Agent') ua_parts = agent.split('/') self.assertEqual(len(ua_parts), 6) @@ -1413,4 +1415,4 @@ def test_update_cors_configuration(self): self.client.disconnect() if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From 0f031cde8a1ea6c6b91ecfc353d9aad9242a8e50 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 20 May 2021 14:38:45 +0100 Subject: [PATCH 78/97] Correct use of username as account name in `Cloudant.bluemix()` --- CHANGES.md | 2 ++ src/cloudant/client.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ad1d3788..8885ae26 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,8 @@ - [FIXED] Fixed the documentation for `bookmarks`. - [FIXED] Also exit `follow_replication` for `failed` state. - [FIXED] Fixed result paging for grouped view queries. +- [FIXED] Incorrect use of username as account name in `Cloudant.bluemix()`. +- [IMPROVED] Documented use of None account name and url override for `Cloudant.iam()`. # 2.14.0 (2020-08-17) diff --git a/src/cloudant/client.py b/src/cloudant/client.py index f09d3a69..47c8555c 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2019 IBM Corp. All rights reserved. +# Copyright (c) 2015, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -853,7 +853,7 @@ def bluemix(cls, vcap_services, instance_name=None, service_name=None, **kwargs) raise CloudantClientException(103) if hasattr(service, 'iam_api_key'): - return Cloudant.iam(service.username, + return Cloudant.iam(None, service.iam_api_key, url=service.url, **kwargs) @@ -867,7 +867,7 @@ def iam(cls, account_name, api_key, **kwargs): """ Create a Cloudant client that uses IAM authentication. - :param account_name: Cloudant account name. + :param account_name: Cloudant account name; or use None and a url kwarg. :param api_key: IAM authentication API key. """ return cls(None, From 5f3ca45c442847046f3a5dfd34f6cfcd5b5929f5 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sun, 30 May 2021 16:45:38 +0200 Subject: [PATCH 79/97] Include LICENSE in pip package --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 96ebb5e1..7c050d91 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include requirements.txt VERSION +include requirements.txt VERSION LICENSE From 105fe2055562adbeb7c5ae41f97712794bf8716b Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Thu, 3 Jun 2021 09:28:45 -0400 Subject: [PATCH 80/97] Remove CouchDB 1.x from Travis CI (#502) --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b49557a3..9f770aac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,6 @@ python: env: - ADMIN_PARTY=true COUCHDB_VERSION=2.3.1 - ADMIN_PARTY=false COUCHDB_VERSION=2.3.1 - - ADMIN_PARTY=true COUCHDB_VERSION=1.7.2 - - ADMIN_PARTY=false COUCHDB_VERSION=1.7.2 services: - docker From f0c0883080a4dcf403a8f3b6e717a4de224f965b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eigel=20Ildik=C3=B3?= Date: Wed, 16 Jun 2021 15:05:39 +0200 Subject: [PATCH 81/97] Use custom encoder (if provided) for all view `key` params not just `keys` (#501) --- CHANGES.md | 1 + src/cloudant/_common_util.py | 11 +++++++---- tests/unit/database_tests.py | 9 +++++++++ tests/unit/param_translation_tests.py | 14 +++++++------- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8885ae26..522d5ede 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ - [FIXED] Fixed result paging for grouped view queries. - [FIXED] Incorrect use of username as account name in `Cloudant.bluemix()`. - [IMPROVED] Documented use of None account name and url override for `Cloudant.iam()`. +- [FIXED] Use custom encoder (if provided) for all view `key` params not just `keys`. # 2.14.0 (2020-08-17) diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 6e765c1d..962f1dec 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -160,7 +160,7 @@ def feed_arg_types(feed_type): return _COUCH_DB_UPDATES_ARG_TYPES return _CHANGES_ARG_TYPES -def python_to_couch(options): +def python_to_couch(options, encoder=None): """ Translates query options from python style options into CouchDB/Cloudant query options. For example ``{'include_docs': True}`` will @@ -171,13 +171,14 @@ def python_to_couch(options): :func:`~cloudant.view.View.__call__` callable, both used to retrieve data. :param dict options: Python style parameters to be translated. + :param encoder: Custom encoder, defaults to None :returns: Dictionary of translated CouchDB/Cloudant query parameters """ translation = dict() for key, val in iteritems_(options): py_to_couch_validate(key, val) - translation.update(_py_to_couch_translate(key, val)) + translation.update(_py_to_couch_translate(key, val, encoder)) return translation def py_to_couch_validate(key, val): @@ -201,7 +202,7 @@ def py_to_couch_validate(key, val): if val not in ('ok', 'update_after'): raise CloudantArgumentError(135, val) -def _py_to_couch_translate(key, val): +def _py_to_couch_translate(key, val, encoder=None): """ Performs the conversion of the Python parameter value to its CouchDB equivalent. @@ -209,6 +210,8 @@ def _py_to_couch_translate(key, val): try: if key in ['keys', 'endkey_docid', 'startkey_docid', 'stale', 'update']: return {key: val} + if key in ['endkey', 'key', 'startkey']: + return {key: json.dumps(val, cls=encoder)} if val is None: return {key: None} arg_converter = TYPE_CONVERTERS.get(type(val)) @@ -249,7 +252,7 @@ def get_docs(r_session, url, encoder=None, headers=None, **params): keys = None if keys_list is not None: keys = json.dumps({'keys': keys_list}, cls=encoder) - f_params = python_to_couch(params) + f_params = python_to_couch(params, encoder) resp = None if keys is not None: # If we're using POST we are sending JSON so add the header diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index c8c5cfe0..a4e4bad1 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -512,6 +512,15 @@ def test_all_docs_get_with_long_type(self): data = self.db.all_docs(limit=1, skip=LONG_NUMBER) self.assertEqual(len(data.get('rows')), 1) + def test_all_docs_get_uses_custom_encoder(self): + """ + Test that all_docs uses the custom encoder. + """ + self.set_up_client(auto_connect=True, encoder="AEncoder") + database = self.client[self.test_dbname] + with self.assertRaises(CloudantArgumentError): + database.all_docs(endkey=['foo', 10]) + def test_custom_result_context_manager(self): """ Test using the database custom result context manager diff --git a/tests/unit/param_translation_tests.py b/tests/unit/param_translation_tests.py index 4da0d05d..6aac7210 100644 --- a/tests/unit/param_translation_tests.py +++ b/tests/unit/param_translation_tests.py @@ -36,7 +36,7 @@ def test_valid_descending(self): {'descending': 'true'} ) self.assertEqual( - python_to_couch({'descending': False}), + python_to_couch({'descending': False}), {'descending': 'false'} ) @@ -44,9 +44,9 @@ def test_valid_endkey(self): """ Test endkey translation is successful. """ - self.assertEqual(python_to_couch({'endkey': 10}), {'endkey': 10}) + self.assertEqual(python_to_couch({'endkey': 10}), {'endkey': '10'}) # Test with long type - self.assertEqual(python_to_couch({'endkey': LONG_NUMBER}), {'endkey': LONG_NUMBER}) + self.assertEqual(python_to_couch({'endkey': LONG_NUMBER}), {'endkey': str(LONG_NUMBER)}) self.assertEqual( python_to_couch({'endkey': 'foo'}), {'endkey': '"foo"'} @@ -120,9 +120,9 @@ def test_valid_key(self): """ Test key translation is successful. """ - self.assertEqual(python_to_couch({'key': 10}), {'key': 10}) + self.assertEqual(python_to_couch({'key': 10}), {'key': '10'}) # Test with long type - self.assertEqual(python_to_couch({'key': LONG_NUMBER}), {'key': LONG_NUMBER}) + self.assertEqual(python_to_couch({'key': LONG_NUMBER}), {'key': str(LONG_NUMBER)}) self.assertEqual(python_to_couch({'key': 'foo'}), {'key': '"foo"'}) self.assertEqual( python_to_couch({'key': ['foo', 10]}), @@ -194,9 +194,9 @@ def test_valid_startkey(self): """ Test startkey translation is successful. """ - self.assertEqual(python_to_couch({'startkey': 10}), {'startkey': 10}) + self.assertEqual(python_to_couch({'startkey': 10}), {'startkey': '10'}) # Test with long type - self.assertEqual(python_to_couch({'startkey': LONG_NUMBER}), {'startkey': LONG_NUMBER}) + self.assertEqual(python_to_couch({'startkey': LONG_NUMBER}), {'startkey': str(LONG_NUMBER)}) self.assertEqual( python_to_couch({'startkey': 'foo'}), {'startkey': '"foo"'} From 19b14ab9adb38d84885587b71c56a66a7789893c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eigel=20Ildik=C3=B3?= Date: Mon, 21 Jun 2021 09:48:15 +0200 Subject: [PATCH 82/97] Support boolean type for `key`, `endkey`, and `startkey` in view requests (#504) --- CHANGES.md | 1 + src/cloudant/_common_util.py | 11 ++++++----- tests/unit/param_translation_tests.py | 18 +++++++++++++++--- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 522d5ede..56700503 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,7 @@ - [FIXED] Incorrect use of username as account name in `Cloudant.bluemix()`. - [IMPROVED] Documented use of None account name and url override for `Cloudant.iam()`. - [FIXED] Use custom encoder (if provided) for all view `key` params not just `keys`. +- [FIXED] Support boolean type for `key`, `endkey`, and `startkey` in view requests. # 2.14.0 (2020-08-17) diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 962f1dec..adce56fa 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -56,20 +56,20 @@ RESULT_ARG_TYPES = { 'descending': (bool,), - 'endkey': (int, LONGTYPE, STRTYPE, Sequence,), + 'endkey': (int, LONGTYPE, STRTYPE, Sequence, bool,), 'endkey_docid': (STRTYPE,), 'group': (bool,), 'group_level': (int, LONGTYPE, NONETYPE,), 'include_docs': (bool,), 'inclusive_end': (bool,), - 'key': (int, LONGTYPE, STRTYPE, Sequence,), + 'key': (int, LONGTYPE, STRTYPE, Sequence, bool,), 'keys': (list,), 'limit': (int, LONGTYPE, NONETYPE,), 'reduce': (bool,), 'skip': (int, LONGTYPE, NONETYPE,), 'stable': (bool,), 'stale': (STRTYPE,), - 'startkey': (int, LONGTYPE, STRTYPE, Sequence,), + 'startkey': (int, LONGTYPE, STRTYPE, Sequence, bool,), 'startkey_docid': (STRTYPE,), 'update': (STRTYPE,), } @@ -191,12 +191,13 @@ def py_to_couch_validate(key, val): # Validate argument values and ensure that a boolean is not passed in # if an integer is expected if (not isinstance(val, RESULT_ARG_TYPES[key]) or - (type(val) is bool and int in RESULT_ARG_TYPES[key])): + (type(val) is bool and bool not in RESULT_ARG_TYPES[key] and + int in RESULT_ARG_TYPES[key])): raise CloudantArgumentError(117, key, RESULT_ARG_TYPES[key]) if key == 'keys': for key_list_val in val: if (not isinstance(key_list_val, RESULT_ARG_TYPES['key']) or - type(key_list_val) is bool): + isinstance(key_list_val, bool)): raise CloudantArgumentError(134, RESULT_ARG_TYPES['key']) if key == 'stale': if val not in ('ok', 'update_after'): diff --git a/tests/unit/param_translation_tests.py b/tests/unit/param_translation_tests.py index 6aac7210..fda3c002 100644 --- a/tests/unit/param_translation_tests.py +++ b/tests/unit/param_translation_tests.py @@ -55,6 +55,10 @@ def test_valid_endkey(self): python_to_couch({'endkey': ['foo', 10]}), {'endkey': '["foo", 10]'} ) + self.assertEqual( + python_to_couch({'endkey': True}), + {'endkey': 'true'} + ) def test_valid_endkey_docid(self): """ @@ -128,6 +132,10 @@ def test_valid_key(self): python_to_couch({'key': ['foo', 10]}), {'key': '["foo", 10]'} ) + self.assertEqual( + python_to_couch({'key': True}), + {'key': 'true'} + ) def test_valid_keys(self): """ @@ -205,6 +213,10 @@ def test_valid_startkey(self): python_to_couch({'startkey': ['foo', 10]}), {'startkey': '["foo", 10]'} ) + self.assertEqual( + python_to_couch({'startkey': True}), + {'startkey': 'true'} + ) def test_valid_startkey_docid(self): """ @@ -247,7 +259,7 @@ def test_invalid_endkey(self): """ msg = 'Argument endkey not instance of expected type:' with self.assertRaises(CloudantArgumentError) as cm: - python_to_couch({'endkey': True}) + python_to_couch({'endkey': {'foo': 'bar'}}) self.assertTrue(str(cm.exception).startswith(msg)) def test_invalid_endkey_docid(self): @@ -302,7 +314,7 @@ def test_invalid_key(self): """ msg = 'Argument key not instance of expected type:' with self.assertRaises(CloudantArgumentError) as cm: - python_to_couch({'key': True}) + python_to_couch({'key': {'foo': 'bar'}}) self.assertTrue(str(cm.exception).startswith(msg)) def test_invalid_keys_not_list(self): @@ -372,7 +384,7 @@ def test_invalid_startkey(self): """ msg = 'Argument startkey not instance of expected type:' with self.assertRaises(CloudantArgumentError) as cm: - python_to_couch({'startkey': True}) + python_to_couch({'startkey': {'foo': 'bar'}}) self.assertTrue(str(cm.exception).startswith(msg)) def test_invalid_startkey_docid(self): From 436eddb45093a6f8aab26f778a7ed5aabc6aba5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eigel=20Ildik=C3=B3?= Date: Mon, 21 Jun 2021 16:02:48 +0200 Subject: [PATCH 83/97] Override dict's get method (#505) Co-authored-by: dominickj-tdi <49956725+dominickj-tdi@users.noreply.github.com> --- CHANGES.md | 1 + src/cloudant/database.py | 18 ++++++++++++++ tests/unit/database_tests.py | 46 ++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 56700503..5b641708 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ - [IMPROVED] Documented use of None account name and url override for `Cloudant.iam()`. - [FIXED] Use custom encoder (if provided) for all view `key` params not just `keys`. - [FIXED] Support boolean type for `key`, `endkey`, and `startkey` in view requests. +- [NEW] Override `dict.get` method for `CouchDatabase` to add `remote` parameter allowing it to retrieve a remote document if specified. # 2.14.0 (2020-08-17) diff --git a/src/cloudant/database.py b/src/cloudant/database.py index ccde12da..4ad93f90 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -706,6 +706,24 @@ def __getitem__(self, key): raise KeyError(key) return doc + def get(self, key, remote=False): + """ + Overrides dict's get method. This gets an item from the database or cache + like __getitem__, but instead of throwing an exception if the item is not + found, it simply returns None. + + :param bool remote: Dictates whether a remote request is made to + retrieve the doc, if it is not present in the local cache. + Defaults to False. + """ + if remote: + try: + return self.__getitem__(key) + except KeyError: + return None + else: + return super(CouchDatabase, self).get(key) + def __contains__(self, key): """ Overrides dictionary __contains__ behavior to check if a document diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index a4e4bad1..3ba12584 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -256,6 +256,8 @@ def test_create_document_with_id(self): data = {'_id': 'julia06', 'name': 'julia', 'age': 6} doc = self.db.create_document(data) self.assertEqual(self.db['julia06'], doc) + self.assertEqual(self.db.get('julia06'), doc) + self.assertEqual(self.db.get('julia06', remote=True), doc) self.assertEqual(doc['_id'], data['_id']) self.assertTrue(doc['_rev'].startswith('1-')) self.assertEqual(doc['name'], data['name']) @@ -271,6 +273,42 @@ def test_create_document_with_id(self): 'Document with id julia06 already exists.' ) + def test_get_non_existing_document_from_remote(self): + """ + Test dict's get on non existing document from remote. + """ + doc = self.db.get('non-existing', remote=True) + self.assertIsNone(doc) + + def test_get_non_existing_document_from_cache(self): + """ + Test dict's get on non existing document from cache. + """ + doc = self.db.get('non-existing') + self.assertIsNone(doc) + + def test_get_document_from_cache(self): + """ + Test dict's get on a document from cache. + """ + doc = Document(self.db, document_id='julia06') + self.db['julia06'] = doc + self.assertEqual(self.db.get('julia06'), doc) + # doc is fetched from the local dict preferentially to remote even with remote=True + self.assertEqual(self.db.get('julia06', remote=True), doc) + self.assertEqual(self.db['julia06'], doc) + + def test_get_document_from_remote(self): + """ + Test dict's get on a document from remote. + """ + data = {'_id': 'julia06','name': 'julia06', 'age': 6} + doc = self.db.create_document(data) + self.db.clear() + self.assertIsNone(self.db.get('julia06')) + self.assertEqual(self.db.get('julia06', remote=True), doc) + self.assertEqual(self.db['julia06'], doc) + def test_create_document_that_already_exists(self): """ Test creating a document that already exists @@ -278,6 +316,8 @@ def test_create_document_that_already_exists(self): data = {'_id': 'julia'} doc = self.db.create_document(data) self.assertEqual(self.db['julia'], doc) + self.assertEqual(self.db.get('julia'), doc) + self.assertEqual(self.db.get('julia', remote=True), doc) self.assertTrue(doc['_rev'].startswith('1-')) # attempt to recreate document self.db.create_document(data, throw_on_exists=False) @@ -289,6 +329,8 @@ def test_create_document_without_id(self): data = {'name': 'julia', 'age': 6} doc = self.db.create_document(data) self.assertEqual(self.db[doc['_id']], doc) + self.assertEqual(self.db.get(doc['_id']), doc) + self.assertEqual(self.db.get(doc['_id'], remote=True), doc) self.assertTrue(doc['_rev'].startswith('1-')) self.assertEqual(doc['name'], data['name']) self.assertEqual(doc['age'], data['age']) @@ -302,6 +344,8 @@ def test_create_design_document(self): data = {'_id': '_design/julia06', 'name': 'julia', 'age': 6} doc = self.db.create_document(data) self.assertEqual(self.db['_design/julia06'], doc) + self.assertEqual(self.db.get('_design/julia06'), doc) + self.assertEqual(self.db.get('_design/julia06', remote=True), doc) self.assertEqual(doc['_id'], data['_id']) self.assertTrue(doc['_rev'].startswith('1-')) self.assertEqual(doc['name'], data['name']) @@ -316,6 +360,8 @@ def test_create_empty_document(self): """ empty_doc = self.db.new_document() self.assertEqual(self.db[empty_doc['_id']], empty_doc) + self.assertEqual(self.db.get(empty_doc['_id']), empty_doc) + self.assertEqual(self.db.get(empty_doc['_id'], remote=True), empty_doc) self.assertTrue(all(x in ['_id', '_rev'] for x in empty_doc.keys())) self.assertTrue(empty_doc['_rev'].startswith('1-')) From 66e4b188030bfa8ed4a71a0a8980f34fd8f34153 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 3 Jun 2021 16:30:22 +0100 Subject: [PATCH 84/97] Add deprecation warnings Co-authored-by: Esteban Laver --- CHANGES.md | 10 ++++++---- README.md | 14 ++++++++++++++ src/cloudant/__init__.py | 4 ++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5b641708..21fa1a50 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,13 +1,15 @@ # UNRELEASED -- [REMOVED] Removed Python 2 compatibility from the supported environments. +- [NEW] Override `dict.get` method for `CouchDatabase` to add `remote` parameter allowing it to + retrieve a remote document if specified. - [FIXED] Fixed the documentation for `bookmarks`. - [FIXED] Also exit `follow_replication` for `failed` state. - [FIXED] Fixed result paging for grouped view queries. - [FIXED] Incorrect use of username as account name in `Cloudant.bluemix()`. -- [IMPROVED] Documented use of None account name and url override for `Cloudant.iam()`. - [FIXED] Use custom encoder (if provided) for all view `key` params not just `keys`. - [FIXED] Support boolean type for `key`, `endkey`, and `startkey` in view requests. -- [NEW] Override `dict.get` method for `CouchDatabase` to add `remote` parameter allowing it to retrieve a remote document if specified. +- [DEPRECATED] This library is now deprecated and will be EOL on Dec 31 2021. +- [REMOVED] Removed Python 2 compatibility from the supported environments. +- [IMPROVED] Documented use of `None` account name and url override for `Cloudant.iam()`. # 2.14.0 (2020-08-17) @@ -139,7 +141,7 @@ # 2.2.0 (2016-10-20) -- [NEW] Added auto connect feature to the client constructor.
 +- [NEW] Added auto connect feature to the client constructor. - [FIXED] Requests session is no longer valid after disconnect. # 2.1.1 (2016-10-03) diff --git a/README.md b/README.md index f12b2814..029edf07 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,17 @@ +# DEPRECATED + +**This library is now deprecated and will be end-of-life on Dec 31 2021.** + +The library remains supported until the end-of-life date, +but will receive only _critical_ maintenance updates. + +Please see the [Migration Guide](./MIGRATION.md) for advice +about migrating to our replacement library +[cloudant-python-sdk](https://github.com/IBM/cloudant-python-sdk). + +For FAQs and additional information please refer to the +[Cloudant blog](https://blog.cloudant.com/2021/06/30/Cloudant-SDK-Transition.html). + # Cloudant Python Client [![Build Status](https://travis-ci.org/cloudant/python-cloudant.svg?branch=master)](https://travis-ci.org/cloudant/python-cloudant) diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 7e011096..f3910174 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -19,10 +19,14 @@ # pylint: disable=wrong-import-position import contextlib +import warnings # pylint: disable=wrong-import-position from .client import Cloudant, CouchDB from ._common_util import CloudFoundryService +warnings.warn('The module cloudant is now deprecated. The replacement is ibmcloudant.', + DeprecationWarning) + @contextlib.contextmanager def cloudant(user, passwd, **kwargs): """ From a166650f795852e506adf3c758a612300fb28392 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Wed, 28 Jul 2021 00:34:32 -0400 Subject: [PATCH 85/97] Add validation for doc ID and attachment names --- CHANGES.md | 5 + Jenkinsfile | 2 +- src/cloudant/_common_util.py | 28 +- src/cloudant/_messages.py | 6 +- src/cloudant/design_document.py | 9 +- src/cloudant/document.py | 24 +- tests/unit/document_validation_tests.py | 909 ++++++++++++++++++++++++ 7 files changed, 973 insertions(+), 10 deletions(-) create mode 100644 tests/unit/document_validation_tests.py diff --git a/CHANGES.md b/CHANGES.md index 21fa1a50..66d1f820 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,11 @@ - [DEPRECATED] This library is now deprecated and will be EOL on Dec 31 2021. - [REMOVED] Removed Python 2 compatibility from the supported environments. - [IMPROVED] Documented use of `None` account name and url override for `Cloudant.iam()`. +- [IMPROVED] - Document IDs and attachment names are now rejected if they could cause an unexpected + Cloudant request. We have seen that some applications pass unsantized document IDs to SDK functions + (e.g. direct from user requests). In response to this we have updated many functions to reject + obviously invalid paths. However, for complete safety applications must still validate that + document IDs and attachment names match expected patterns. # 2.14.0 (2020-08-17) diff --git a/Jenkinsfile b/Jenkinsfile index e655a45c..cc2d0ad4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,7 +25,7 @@ def getEnvForSuite(suiteName) { } def setupPythonAndTest(pythonVersion, testSuite) { - node { + node('sdks-executor') { // Unstash the source on this node unstash name: 'source' // Set up the environment and test diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index adce56fa..433e09a6 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. +# Copyright © 2015, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -31,6 +31,8 @@ # Library Constants +DESIGN_PREFIX = '_design/' +LOCAL_PREFIX = '_local/' USER_AGENT = '/'.join([ 'python-cloudant', sys.modules['cloudant'].__version__, @@ -301,6 +303,30 @@ def response_to_json_dict(response, **kwargs): response.encoding = 'utf-8' return json.loads(response.text, **kwargs) +def assert_document_type_id(docid): + """ + Validate the document ID. Raises an error if the ID is an `_` prefixed name + that isn't either `_design` or `_local`. + :return: + """ + invalid = False + if docid.startswith('_'): + if docid.startswith(DESIGN_PREFIX) and DESIGN_PREFIX != docid: + invalid = False + elif docid.startswith(LOCAL_PREFIX) and LOCAL_PREFIX != docid: + invalid = False + else: + invalid = True + if invalid: + raise CloudantArgumentError(137, docid) + +def assert_attachment_name(attname): + """ + Validate the document attachment's name. Raises an error if `_` prefixed name exists. + :return: + """ + if attname.startswith('_'): + raise CloudantArgumentError(138, attname) # Classes diff --git a/src/cloudant/_messages.py b/src/cloudant/_messages.py index 07a23d62..b428998c 100644 --- a/src/cloudant/_messages.py +++ b/src/cloudant/_messages.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright © 2016, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -65,7 +65,9 @@ # Common_util 134: 'Key list element not of expected type: {0}', 135: 'Invalid value for stale option {0} must be ok or update_after.', - 136: 'Error converting argument {0}: {1}' + 136: 'Error converting argument {0}: {1}', + 137: 'Invalid document ID: {0}', + 138: 'Invalid attachment name: {0}' } CLIENT = { diff --git a/src/cloudant/design_document.py b/src/cloudant/design_document.py index d3ea387a..df9fb44d 100644 --- a/src/cloudant/design_document.py +++ b/src/cloudant/design_document.py @@ -16,7 +16,8 @@ API module/class for interacting with a design document in a database. """ from ._2to3 import iteritems_, url_quote_plus, STRTYPE -from ._common_util import QUERY_LANGUAGE, codify, response_to_json_dict +from ._common_util import QUERY_LANGUAGE, codify, response_to_json_dict, \ + assert_document_type_id, DESIGN_PREFIX from .document import Document from .view import View, QueryIndexView from .error import CloudantArgumentError, CloudantDesignDocumentException @@ -44,8 +45,10 @@ class DesignDocument(Document): databases. """ def __init__(self, database, document_id=None, partitioned=False): - if document_id and not document_id.startswith('_design/'): - document_id = '_design/{0}'.format(document_id) + if document_id: + assert_document_type_id(document_id) + if document_id and not document_id.startswith(DESIGN_PREFIX): + document_id = '{0}{1}'.format(DESIGN_PREFIX, document_id) super(DesignDocument, self).__init__(database, document_id) if partitioned: diff --git a/src/cloudant/document.py b/src/cloudant/document.py index 38082054..96267bc9 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright © 2015, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ from requests.exceptions import HTTPError from ._2to3 import url_quote, url_quote_plus -from ._common_util import response_to_json_dict +from ._common_util import response_to_json_dict, assert_document_type_id, assert_attachment_name from .error import CloudantDocumentException @@ -96,6 +96,15 @@ def document_url(self): url_quote(self['_id'][8:], safe='') )) + # handle _local document url + if self['_id'].startswith('_local/'): + return '/'.join(( + self._database_host, + url_quote_plus(self._database_name), + '_local', + url_quote(self['_id'][7:], safe='') + )) + # handle document url return '/'.join(( self._database_host, @@ -113,6 +122,8 @@ def exists(self): if '_id' not in self or self['_id'] is None: return False + assert_document_type_id(self['_id']) + resp = self.r_session.head(self.document_url) if resp.status_code not in [200, 404]: resp.raise_for_status() @@ -161,6 +172,8 @@ def fetch(self): """ if self.document_url is None: raise CloudantDocumentException(101) + if '_id' in self: + assert_document_type_id(self['_id']) resp = self.r_session.get(self.document_url) resp.raise_for_status() self.clear() @@ -309,6 +322,8 @@ def delete(self): if not self.get("_rev"): raise CloudantDocumentException(103) + assert_document_type_id(self['_id']) + del_resp = self.r_session.delete( self.document_url, params={"rev": self["_rev"]}, @@ -375,7 +390,8 @@ def get_attachment( """ # need latest rev self.fetch() - attachment_url = '/'.join((self.document_url, attachment)) + assert_attachment_name(attachment) + attachment_url = '/'.join((self.document_url, url_quote(attachment, safe=''))) if headers is None: headers = {'If-Match': self['_rev']} else: @@ -418,6 +434,7 @@ def delete_attachment(self, attachment, headers=None): """ # need latest rev self.fetch() + assert_attachment_name(attachment) attachment_url = '/'.join((self.document_url, attachment)) if headers is None: headers = {'If-Match': self['_rev']} @@ -459,6 +476,7 @@ def put_attachment(self, attachment, content_type, data, headers=None): """ # need latest rev self.fetch() + assert_attachment_name(attachment) attachment_url = '/'.join((self.document_url, attachment)) if headers is None: headers = { diff --git a/tests/unit/document_validation_tests.py b/tests/unit/document_validation_tests.py new file mode 100644 index 00000000..021d8d6f --- /dev/null +++ b/tests/unit/document_validation_tests.py @@ -0,0 +1,909 @@ +#!/usr/bin/env python +# Copyright © 2021 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +from enum import Enum +from unittest.mock import Mock, patch + +from mock import create_autospec + +import requests +from urllib.parse import urlparse + +from cloudant import database +from cloudant.design_document import DesignDocument +from cloudant.document import Document +from cloudant.error import CloudantArgumentError + +class ValidationExceptionMsg(Enum): + DOC = 'Invalid document ID:' + ATTACHMENT = 'Invalid attachment name:' + +class Expect(Enum): + VALIDATION_EXCEPTION_DOCID = CloudantArgumentError(137, '') + VALIDATION_EXCEPTION_ATT = CloudantArgumentError(138, '') + RESPONSE_404 = 404 + RESPONSE_200 = 200 + RESPONSE_201 = 201 + + +class ValidationTests(unittest.TestCase): + """ + Document validation unit tests + """ + def setUp(self): + self.doc_r_session_patcher = patch('cloudant.document.Document.r_session') + self.requests_get_patcher = patch('requests.get') + + self.addCleanup(patch.stopall) + + self.doc_r_session_mock = self.doc_r_session_patcher.start() + self.requests_get_mock = self.requests_get_patcher.start() + + self.db = create_autospec(database) + self.db.client = Mock() + self.db.client.server_url = 'http://mocked.url.com' + self.db.database_url = 'http://mocked.url.com/my_db' + self.db.database_name = 'mydb' + + def teardown(self): + self.addCleanup(patch.stopall) + del self.db + del self.doc_r_session_patcher + del self.requests_get_patcher + del self.doc_r_session_mock + del self.requests_get_mock + + # GET and HEAD _all_docs + # EXPECTED: validation failure + def test_get_invalid_all_docs(self): + """ + Test GET/HEAD request for invalid '_all_docs' document ID + """ + self.get_document_variants('_all_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _design/foo + # EXPECTED: 200 + def test_get_valid_ddoc(self): + """ + Test GET/HEAD request for valid '_design/foo' document ID + """ + self.get_document_variants('_design/foo', Expect.RESPONSE_200.value, path_segment_count=3) + self.get_document_variants('_design/foo', Expect.RESPONSE_200.value, True, path_segment_count=3) + + # GET and HEAD _design + # EXPECTED: Validation exception + def test_get_invalid_design(self): + """ + Test GET/HEAD request for invalid '_design' document ID + """ + self.get_document_variants('_design', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_document_variants('_design', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # GET and HEAD /_design/foo with a slash + # EXPECTED: 404 + def test_get_missing_ddoc_with_slash(self): + """ + Test GET/HEAD request for missing '/_design/foo' document ID + """ + self.get_document_variants('/_design/foo', Expect.RESPONSE_404.value, path_segment_count=2) + + # GET and HEAD _design/foo/_view/bar + # EXPECTED: 404 + def test_get_invalid_view(self): + """ + Test GET/HEAD request for missing '_design/foo' document ID + """ + self.get_document_variants('_design/foo/_view/bar', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_view/bar', Expect.RESPONSE_404.value, True, path_segment_count=3) + + # GET and HEAD _design/foo/_info + # EXPECTED: 404 + def test_get_invalid_view_info(self): + """ + Test GET/HEAD request for missing '_design/foo/_info' document ID + """ + self.get_document_variants('_design/foo/_info', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_info', Expect.RESPONSE_404.value, True, path_segment_count=3) + + # GET and HEAD _design/foo/_search/bar + # EXPECTED: 404 + def test_get_invalid_search(self): + """ + Test GET/HEAD request for missing '_design/foo/_search/bar' document ID + """ + self.get_document_variants('_design/foo/_search/bar', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_search/bar', Expect.RESPONSE_404.value, True, path_segment_count=3) + + # GET and HEAD _design/foo/_search_info/bar + # EXPECTED: 404 + def test_get_invalid_search_info(self): + """ + Test GET/HEAD request for missing '_design/foo/_search_info/bar' document ID + """ + self.get_document_variants('_design/foo/_search_info/bar', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_search_info/bar', Expect.RESPONSE_404.value, True, path_segment_count=3) + + # GET and HEAD _design/foo/_geo/bar + # EXPECTED: 404 + def test_get_missing_geo(self): + """ + Test GET/HEAD request for missing '_design/foo/_geo/bar' document ID + """ + self.get_document_variants('_design/foo/_geo/bar', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_geo/bar', Expect.RESPONSE_404.value, True, path_segment_count=3) + # with a parameter + self.get_document_variants('_design/foo/_geo/bar?bbox=-50.52,-4.46,54.59,1.45', Expect.RESPONSE_404.value, + path_segment_count=3) + self.get_document_variants('_design/foo/_geo/bar?bbox=-50.52,-4.46,54.59,1.45', Expect.RESPONSE_404.value, True, + path_segment_count=3) + + # GET and HEAD _design/foo/_geo_info/bar + # EXPECTED: 404 + def test_get_missing_geo_info(self): + """ + Test GET/HEAD request for missing '_design/foo/_geo_info/bar' document ID + """ + self.get_document_variants('_design/foo/_geo_info/bar', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_geo_info/bar', Expect.RESPONSE_404.value, True, path_segment_count=3) + + # GET and HEAD _local/foo + # EXPECTED: 200 + def test_get_local_doc(self): + """ + Test GET/HEAD request for valid '_local/foo' document ID + """ + self.get_document_variants('_local/foo', Expect.RESPONSE_200.value, path_segment_count=3) + + # GET and HEAD _local + # EXPECTED: Validation exception + def test_get_invalid_local(self): + """ + Test GET/HEAD request for invalid '_local' document ID + """ + self.get_document_variants('_local', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _local_docs + # EXPECTED: Validation exception + def test_get_invalid_local_docs(self): + """ + Test GET/HEAD request for invalid '_local_docs' document ID + """ + self.get_document_variants('_local_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _design_docs + # EXPECTED: Validation exception + def test_get_invalid_design_docs(self): + """ + Test GET/HEAD request for invalid '_design_docs' document ID + """ + self.get_document_variants('_design_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _changes + # EXPECTED: Validation exception + def test_get_invalid_changes(self): + """ + Test GET/HEAD request for invalid '_changes' document ID + """ + self.get_document_variants('_changes', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _ensure_full_commit + # EXPECTED: Validation exception + def test_get_invalid_ensure_full_commit(self): + """ + Test GET/HEAD request for invalid '_ensure_full_commit' document ID + """ + self.get_document_variants('_ensure_full_commit', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _index + # EXPECTED: Validation exception + def test_get_invalid_index(self): + """ + Test GET/HEAD request for invalid '_index' document ID + """ + self.get_document_variants('_index', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _revs_limit + # EXPECTED: Validation exception + def test_get_invalid_revs_limit(self): + """ + Test GET/HEAD request for invalid '_revs_limit' document ID + """ + self.get_document_variants('_revs_limit', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _security + # EXPECTED: Validation exception + def test_get_invalid_security(self): + """ + Test GET/HEAD request for invalid '_security' document ID + """ + self.get_document_variants('_security', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _shards + # EXPECTED: Validation exception + def test_get_invalid_shards(self): + """ + Test GET/HEAD request for invalid '_shards' document ID + """ + self.get_document_variants('_shards', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # DELETE _index/_design/foo/json/bar + # EXPECTED: Validation exception + def test_delete_invalid_index(self): + """ + Test DELETE request for invalid '_index/_design/foo/json/bar' document ID + """ + self.delete_document_variants('_index/_design/foo/json/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # DELETE _design/foo + # EXPECTED: 200 + def test_delete_valid_ddoc(self): + """ + Test DELETE request for valid '_design/foo' document ID + """ + self.delete_document_variants('_design/foo', Expect.RESPONSE_200.value, path_segment_count=3) + + # DELETE _design + # EXPECTED: Validation exception + def test_delete_invalid_ddoc(self): + """ + Test DELETE request for invalid '_design' document ID + """ + # no trailing '/' on _design prefix + self.delete_document_variants('_design', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # DELETE _local/foo + # EXPECTED: 200 + def test_delete_valid_local_doc(self): + """ + Test DELETE request for valid '_local/foo' document ID + """ + self.delete_document_variants('_local/foo', Expect.RESPONSE_200.value, path_segment_count=3) + + # DELETE _local + # EXPECTED: Validation exception + def test_delete_invalid_local(self): + """ + Test DELETE request for invalid '_local' document ID + """ + # no trailing '/' on _local prefix + self.delete_document_variants('_local', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # PUT _design/foo + # EXPECTED: 201 + def test_put_valid_ddoc(self): + """ + Test PUT request for valid '_design/foo' document ID + """ + self.put_document_variants('_design/foo', Expect.RESPONSE_201.value, path_segment_count=3) + + # PUT _design + # EXPECTED: Validation exception + def test_put_invalid_ddoc(self): + """ + Test PUT request for invalid '_design' document ID + """ + self.put_document_variants('_design', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # PUT _local/foo + # EXPECTED: 201 + def test_put_valid_local_doc(self): + """ + Test PUT request for valid '_local/foo' document ID + """ + self.put_document_variants('_local/foo', Expect.RESPONSE_201.value, path_segment_count=3) + + # PUT _local + # EXPECTED: Validation exception + def test_put_invalid_local_doc(self): + """ + Test PUT request for invalid '_local' document ID + """ + self.put_document_variants('_local', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # PUT _revs_limit + # EXPECTED: Validation exception + def test_put_invalid_revs_limit(self): + """ + Test PUT request for invalid '_revs_limit' document ID + """ + self.put_document_variants('_revs_limit', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # PUT _security + # EXPECTED: Validation exception + def test_put_invalid_security(self): + """ + Test PUT request for invalid '_security' document ID + """ + self.put_document_variants('_security', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET _design/foo/bar + # EXPECTED: 200 + def test_get_valid_ddoc_attachment(self): + """ + Test PUT request for valid '_design/foo/bar' document ID + """ + self.get_doc_attachment_variants('_design/foo', 'bar', Expect.RESPONSE_200.value, True, path_segment_count=4) + + # PUT _design/foo/bar + # EXPECTED: 201 + def test_put_valid_ddoc_attachment(self): + """ + Test PUT request for valid '_design/foo/bar' document ID + """ + self.put_doc_attachment_variants('_design/foo', 'bar', Expect.RESPONSE_201.value, True, path_segment_count=4) + + # DELETE _design/foo/bar + # EXPECTED: 200 + def test_delete_valid_ddoc_attachment(self): + """ + Test DELETE request for valid '_design/foo/bar' document ID + """ + self.delete_doc_attachment_variants('_design/foo', 'bar', Expect.RESPONSE_200.value, True, path_segment_count=4) + + # GET _design/foo + # EXPECTED: Validation exception + def test_get_invalid_ddoc_attachment(self): + """ + Test GET request for invalid '_design/foo' document ID + """ + # with ddoc option enabled + self.get_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.get_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # PUT _design/foo + # EXPECTED: Validation exception + def test_put_invalid_ddoc_attachment(self): + """ + Test PUT request for invalid '_design/foo' document ID + """ + # with ddoc option enabled + self.put_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.put_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # DELETE _design/foo + # EXPECTED: Validation exception + def test_delete_invalid_ddoc_attachment(self): + """ + Test DELETE request for invalid '_design/foo' document ID + """ + # with ddoc option enabled + self.delete_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.delete_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # DELETE _index/_design/foo/json/bar + # EXPECTED: Validation exception + def test_delete_index_via_attachment(self): + """ + Test DELETE requests for invalid '_index/_design/foo/json/bar' + """ + self.delete_doc_attachment_variants('_index', '_design/foo/json/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_index', '_design/foo/json/bar', + Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.delete_doc_attachment_variants('_index/_design', 'foo/json/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_index/_design', 'foo/json/bar', + Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.delete_doc_attachment_variants('_index/_design/foo', 'json/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_index/_design/foo', 'json/bar', + Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.delete_doc_attachment_variants('_index/_design/foo/json', 'bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_index/_design/foo/json', 'bar', + Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # GET _design/foo/_view/bar + def test_get_view_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_view/bar' + """ + # EXPECTED: 404 + self.get_doc_attachment_variants('_design/foo/_view', 'bar', Expect.RESPONSE_404.value, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_view', 'bar', Expect.RESPONSE_404.value, True, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo', '/_view/bar', Expect.RESPONSE_404.value, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo', '/_view/bar', Expect.RESPONSE_404.value, True, path_segment_count=4) + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + self.get_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.get_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # PUT _design/foo/_view/bar + def test_put_view_via_ddoc_attachment(self): + """ + Test PUT requests for '_design/foo/_view/bar' + """ + # EXPECTED: Validation exception + self.put_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.put_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + self.put_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.put_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.put_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.put_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # DELETE _design/foo/_view/bar + def test_delete_view_via_ddoc_attachment(self): + """ + Test DELETE requests for '_design/foo/_view/bar' + """ + # EXPECTED: Validation exception + self.delete_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.delete_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + self.delete_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.delete_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # GET _design/foo/_info + def test_get_view_info_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_info' + """ + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_info', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_info', Expect.VALIDATION_EXCEPTION_ATT.value, True) + self.get_doc_attachment_variants('_design', 'foo/_info', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design', 'foo/_info', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.get_doc_attachment_variants('_design/', 'foo/_info', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design/', 'foo/_info', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # GET _design/foo/_search/bar + def test_get_search_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_search/bar' + """ + # EXPECTED: 404 + self.get_doc_attachment_variants('_design/foo/_search', 'bar', Expect.RESPONSE_404.value, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_search', 'bar', Expect.RESPONSE_404.value, True, + path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_search', 'bar?q=*.*', Expect.RESPONSE_404.value, + path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_search', 'bar?q=*.*', Expect.RESPONSE_404.value, True, + path_segment_count=4) + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_search/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_search/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + self.get_doc_attachment_variants('_design', 'foo/_search/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design', 'foo/_search/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.get_doc_attachment_variants('_design/', 'foo/_search/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design/', 'foo/_search/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # GET _design/foo/_search_info/bar + def test_get_search_info_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_search_info/bar' + """ + # EXPECTED: 404 + self.get_doc_attachment_variants('_design/foo/_search_info', 'bar', Expect.RESPONSE_404.value, + path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_search_info', 'bar', Expect.RESPONSE_404.value, True, + path_segment_count=4) + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_search_info/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_search_info/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + + # GET _design/foo/_geo/bar + def test_get_geo_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_geo/bar' + """ + # EXPECTED: 404 + self.get_doc_attachment_variants('_design/foo/_geo', 'bar', Expect.RESPONSE_404.value, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_geo', 'bar', Expect.RESPONSE_404.value, True, + path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_geo', 'bar?bbox=-50.52,-4.46,54.59,1.45', + Expect.RESPONSE_404.value, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_geo', 'bar?bbox=-50.52,-4.46,54.59,1.45', + Expect.RESPONSE_404.value, True, path_segment_count=4) + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_geo/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_geo/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + + # GET _design/foo/_geo_info/bar + def test_get_geo_info_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_geo_info/bar' + """ + # EXPECTED: 404 + self.get_doc_attachment_variants('_design/foo/_geo_info', 'bar', Expect.RESPONSE_404.value, + path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_geo_info', 'bar', Expect.RESPONSE_404.value, True, + path_segment_count=4) + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_geo_info/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_geo_info/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + + # GET _partition/foo + # EXPECTED: Validation exception + def test_get_invalid_partition_info(self): + """ + Test GET requests for '_partition/foo' + """ + self.get_document_variants('_partition/foo', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET _partition/foo + # EXPECTED: Validation exception + def test_get_invalid_partition_info_via_attachment(self): + """ + Test GET requests for '_partition/foo' + """ + self.get_doc_attachment_variants('_partition', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET _partition/foo/_all_docs + # EXPECTED: Validation exception + def test_get_partition_info(self): + """ + Test GET requests for '_partition/foo/_all_docs' + """ + self.get_document_variants('_partition/foo/_all_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET _partition/foo/_all_docs + # EXPECTED: Validation exception + def test_get_invalid_partition_all_docs_via_attachment(self): + """ + Test GET requests for '_partition/foo/_all_docs' + """ + self.get_doc_attachment_variants('_partition', 'foo/_all_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_partition/foo', '_all_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + + """UTIL FUNCTIONS""" + def mocked_get_requests(self, rev=None, override_status_code=None): + """ + Create a mock GET request for documents with the expected status code + :param rev: the doc's revision (default None) + :param override_status_code: override the status code for handling + inner `fetch` request call within `get_attachment` + :return: mocked Response object + """ + resp_mock = create_autospec(requests.Response) + if override_status_code is not None: + resp_mock.status_code = override_status_code + else: + resp_mock.status_code = self.expected_enum + if (resp_mock.status_code == 200 or resp_mock.status_code == 201 + and self.doc_id is not None): + if rev is not None: + resp_mock.text = f"""{{"_id": "{self.doc_id}", "_rev": "{rev}"}}""" + else: + resp_mock.text = f"""{{"_id": "{self.doc_id}", "_rev": "1-abc"}}""" + elif resp_mock.status_code == 404: + resp_mock.raise_for_status.side_effect = requests.exceptions.HTTPError + resp_mock.encoding = None + + return resp_mock + + def mocked_get_att_requests(self): + """ + Create a mock GET request for attachments with the expected status code + """ + self.expected_att_content = f"""this is a text attachment""" + # first fetch doc call with rev + fetch_mock = self.mocked_get_requests(rev=None, override_status_code=200) + # second get to attachment + resp_mock = create_autospec(requests.Response) + resp_mock.status_code = self.expected_enum + if self.expected_enum == 200 and self.doc_id is not None and self.att_name is not None: + resp_mock.text = self.expected_att_content + if self.expected_enum == 404: + resp_mock.raise_for_status.side_effect = requests.exceptions.HTTPError + self.doc_r_session_mock.get.side_effect = [fetch_mock, resp_mock] + + def mocked_head_requests(self, override_status_code=None): + """ + Create a mock HEAD request for documents and attachments with the expected status code + """ + resp_mock = create_autospec(requests.Response) + if override_status_code is not None: + resp_mock.status_code = override_status_code + else: + resp_mock.status_code = self.expected_enum + self.doc_r_session_mock.head = Mock(return_value=resp_mock) + + def mocked_delete_requests(self): + """ + Create a mock DELETE request for documents with the expected status code + """ + resp_mock = create_autospec(requests.Response) + resp_mock.status_code = self.expected_enum + if self.expected_enum == 201 and self.doc_id is not None: + resp_mock.text = f"""{{"id": "{self.doc_id}", "rev": "2-abc", "ok": true}}""" + self.doc_r_session_mock.delete = Mock(return_value=resp_mock) + + def mocked_delete_att_requests(self): + """ + Create a mock DELETE request for attachments with the expected status code + """ + # first `fetch` document call with rev + self.doc_r_session_mock.get = Mock(return_value=self.mocked_get_requests(rev=None, override_status_code=200)) + # second delete to attachment + resp_mock = create_autospec(requests.Response) + resp_mock.status_code = self.expected_enum + resp_mock.encoding = None + if self.expected_enum == 200 and self.doc_id is not None and self.att_name is not None: + resp_mock.text = f"""{{"id": "{self.doc_id}", "rev": "2-abc", "ok": true}}""" + elif self.expected_enum == 404: + resp_mock.raise_for_status.side_effect = requests.exceptions.HTTPError + + self.doc_r_session_mock.delete = Mock(return_value=resp_mock) + + def mocked_put_doc_requests(self): + """ + Create a mock PUT request for documents with the expected status code + """ + # mock 'doc.exists' request call within 'doc.save' function + self.mocked_head_requests(200) + resp_mock = create_autospec(requests.Response) + resp_mock.status_code = self.expected_enum + resp_mock.encoding = None + if self.expected_enum == 201 and self.doc_id is not None: + resp_mock.text = f"""{{"id": "{self.doc_id}", "rev": "1-abc", "ok": true}}""" + if self.expected_enum == 404: + resp_mock.raise_for_status.side_effect = requests.exceptions.HTTPError + self.doc_r_session_mock.put = Mock(return_value=resp_mock) + + def mocked_put_att_requests(self): + """ + Create a mock PUT request for attachments with the expected status code + """ + # first `fetch` document call within `put_attachment` + fetch_mock = self.mocked_get_requests(rev=None, override_status_code=200) + # create Response object for PUT attachment + resp_mock = create_autospec(requests.Response) + resp_mock.status_code = self.expected_enum + resp_mock.encoding = None + if self.expected_enum == 201 and self.doc_id is not None: + resp_mock.text = f"""{{"id": "{self.doc_id}", "rev": "2-def", "ok": true}}""" + if self.expected_enum == 404: + resp_mock.raise_for_status.side_effect = requests.exceptions.HTTPError + # final fetch doc call + second_fetch_mock = self.mocked_get_requests(rev='2-def', override_status_code=200) + self.doc_r_session_mock.get.side_effect = [fetch_mock, second_fetch_mock] + self.doc_r_session_mock.put = Mock(return_value=resp_mock) + + def get_document_variants(self, doc_id, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute GET/HEAD document requests + """ + self.doc_id = doc_id + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_head_requests() + self.head_document() + self.doc_r_session_mock.get.return_value = self.mocked_get_requests() + self.fetch_document() + self.assert_path_segments(self.doc_r_session_mock.get.call_args_list, path_segment_count) + + def get_doc_attachment_variants(self, doc_id, att_name, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute GET attachment requests + """ + self.att_name = att_name + self.doc_id = doc_id + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_get_att_requests() + self.get_doc_attachment() + self.assert_path_segments(self.doc_r_session_mock.get.call_args_list, path_segment_count) + + def put_document_variants(self, doc_id, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute PUT document requests + """ + self.doc_id = doc_id + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_put_doc_requests() + self.put_document() + self.assert_path_segments(self.doc_r_session_mock.put.call_args_list, path_segment_count) + + def put_doc_attachment_variants(self, doc_id, att_name, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute PUT attachment requests + """ + self.att_name = att_name + self.doc_id = doc_id + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_put_att_requests() + self.put_doc_attachment() + self.assert_path_segments(self.doc_r_session_mock.put.call_args_list, path_segment_count) + + def delete_document_variants(self, doc_id, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute DELETE document requests + """ + self.doc_id = doc_id + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_delete_requests() + self.delete_document() + self.assert_path_segments(self.doc_r_session_mock.delete.call_args_list, path_segment_count) + + def delete_doc_attachment_variants(self, doc_id, attname, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute DELETE attachment requests + """ + self.doc_id = doc_id + self.att_name = attname + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_delete_att_requests() + self.delete_doc_attachment() + self.assert_path_segments(self.doc_r_session_mock.delete.call_args_list, path_segment_count) + + """HTTP REQUEST FUNCTIONS""" + def head_document(self): + try: + resp = self.create_doc(self.doc_id, self.is_ddoc).exists() + if self.expected_enum == 200 or self.expected_enum == 201: + self.assertTrue(resp) + elif self.expected_enum == 404: + self.assertFalse(resp) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + + def delete_document(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + doc['_rev'] = '1-abc' + doc.delete() + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, doc['_id']) + self.assertFalse('rev' in doc) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + def fetch_document(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + doc.fetch() + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, doc['_id']) + self.assertIsNotNone(doc['_rev']) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + def put_document(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + doc.save() + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, doc['_id']) + self.assertIsNotNone(doc['_rev']) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + def delete_doc_attachment(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + doc['_rev'] = '1-abc' + resp = doc.delete_attachment(self.att_name) + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, doc['_id']) + self.assertEqual(self.doc_id, resp['id']) + self.assertEqual(doc['_rev'], resp['rev']) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + def get_doc_attachment(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + resp_att = doc.get_attachment(self.att_name, attachment_type='text') + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, doc['_id']) + self.assertIsNotNone(resp_att) + self.assertEqual(resp_att, self.expected_att_content) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + def put_doc_attachment(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + resp_att = doc.put_attachment(self.att_name, content_type='utf-8', data='test') + self.assertIsNotNone(resp_att) + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, resp_att['id']) + self.assertEqual(resp_att['id'], doc['_id']) + self.assertEqual(doc['_rev'], resp_att['rev']) + self.assertEqual(resp_att['ok'], True) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + """HELPER FUNCTIONS""" + def create_doc(self, doc_id=None, is_ddoc=False): + """ + Function to create and return a Document or DesignDocument object. + """ + if is_ddoc: + if doc_id is not None: + doc = DesignDocument(self.db, doc_id) + else: + doc = DesignDocument(self.db) + elif doc_id is not None: + doc = Document(self.db, doc_id) + else: + doc = Document(self.db) + self.assertIsNone(doc.get('_rev')) + return doc + + def assert_exception_msg(self, cae): + """ + Function to assert whether the exception message is for an invalid + document ID or an attachment name. + """ + self.assertTrue(id(self.expected_enum), id(cae)) + # Check that actual exception message starts with the expected msg + if str(cae).startswith(str(self.expected_enum)): + # Figure out which exception msg to assert against + if str(cae).startswith(ValidationExceptionMsg.ATTACHMENT.value): + self.assertEqual(str(cae), f"""{ValidationExceptionMsg.ATTACHMENT.value} {self.att_name}""") + elif str(cae).startswith(ValidationExceptionMsg.DOC.value): + self.assertEqual(str(cae), f"""{ValidationExceptionMsg.DOC.value} {self.doc_id}""") + else: + self.fail('Expected CloudantArgumentError message should equal actual error message.') + + def assert_path_segments(self, actual_call_args_list, exp_segment_count): + """ + Function to assert the number of path segments from a mock request + """ + # If there's no segment count, verify that the test case expects an argument error + if exp_segment_count is None: + self.assertTrue(isinstance(self.expected_enum, CloudantArgumentError), 'Path segment count should exist ' + 'when testing against valid ' + 'document or attachment names.') + else: + # get latest call in list + url, headers = actual_call_args_list[len(actual_call_args_list) - 1] + # there should only be one mocked url + self.assertEqual(len(url), 1) + # parse path of url and remove first / path segment + path = urlparse(url[0]).path[1:] + actual_segment_count = len(path.split('/')) + self.assertEqual(actual_segment_count, exp_segment_count) From f05f0921716684c4aac0c61bedbe7cb7b0823655 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 26 Aug 2021 10:53:20 +0100 Subject: [PATCH 86/97] Update CHANGES for 2.15.0 --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 66d1f820..d5cf74e5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,4 @@ -# UNRELEASED +# 2.15.0 (2021-08-26) - [NEW] Override `dict.get` method for `CouchDatabase` to add `remote` parameter allowing it to retrieve a remote document if specified. - [FIXED] Fixed the documentation for `bookmarks`. From 760c8c99373b7b02dc32cc2ff994071ed4938997 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 26 Aug 2021 10:55:33 +0100 Subject: [PATCH 87/97] Update version to 2.15.0 --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 20738ec2..68e69e40 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.14.1-SNAPSHOT +2.15.0 diff --git a/docs/conf.py b/docs/conf.py index 1e338e6d..403f5bd7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '2.14.1-SNAPSHOT' +version = '2.15.0' # The full version, including alpha/beta/rc tags. -release = '2.14.1-SNAPSHOT' +release = '2.15.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index f3910174..720b8709 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.14.1-SNAPSHOT' +__version__ = '2.15.0' # pylint: disable=wrong-import-position import contextlib From 3bb26d75fa255802a5f308bbf9cff1ba3b34439b Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 26 Aug 2021 12:17:13 +0100 Subject: [PATCH 88/97] Update verson to 2.15.1-SNAPSHOT --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 68e69e40..3a7d90b0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.15.0 +2.15.1-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index 403f5bd7..c8d4ad26 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '2.15.0' +version = '2.15.1-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.15.0' +release = '2.15.1-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 720b8709..66769dad 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.15.0' +__version__ = '2.15.1-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From c2ffcd6393e9115acc02c81135424dd9325cdfd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eigel=20Ildik=C3=B3?= Date: Tue, 21 Dec 2021 10:56:10 +0100 Subject: [PATCH 89/97] Improve migration guide (#509) --- MIGRATION.md | 109 ++++++++++++++++++++++++++------------------------- 1 file changed, 56 insertions(+), 53 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index 29be9810..21c9ae5f 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -17,6 +17,10 @@ remote HTTP API. For example, in the case of the document context manager, this to fetch and a `put_document` to save. 1. In `cloudant-python-sdk` View, Search, and Query (aka `_find` endpoint) operation responses contain raw JSON content like using `raw_result=True` in `python-cloudant`. +1. Replay adapters are replaced by the [automatic retries](https://github. + com/IBM/ibm-cloud-sdk-common/#automatic-retries) feature for failed requests. +1. Error handling is not transferable from `python-cloudant` to `cloudant-python-sdk`. For more information go to the [Error handling section](https://cloud.ibm.com/apidocs/cloudant?code=python#error-handling) in our API docs. +1. Custom HTTP client configurations in `python-cloudant` are not transferable to `python-java-sdk`. For more information go to the [Configuring the HTTP client section(https://githubcom/IBM/ibm-cloud-sdk-common/#configuring-the-http-client) in the IBM Cloud SDK Common README. ## Request mapping Here's a list of the top 5 most frequently used `python-cloudant` operations and the `cloudant-python-sdk` equivalent API operation documentation link: @@ -44,56 +48,55 @@ The table below contains a list of `python-cloudant` functions and the `cloudant | `python-cloudant` function | `cloudant-python-sdk` API operation documentation link | |-----------------|---------------------| -|`metadata()`|[getServerInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getserverinformation)| -|`all_dbs()`|[getAllDbs](https://cloud.ibm.com/apidocs/cloudant?code=python#getalldbs)| -|`db_updates()/infinite_db_updates()`|[getDbUpdates](https://cloud.ibm.com/apidocs/cloudant?code=python#getdbupdates)| -|`Replicator.stop_replication()`|[deleteReplicationDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#deletereplicationdocument)| -|`Replicator.replication_state()`|[getReplicationDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getreplicationdocument)| -|`Replicator.create_replication()`|[putReplicationDocument](https://cloud.ibm.com/apidocs/cloudant?code=#putreplicationdocument)| -|`Scheduler.get_doc()`|[getSchedulerDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerdocument)| -|`Scheduler.list_docs()`|[getSchedulerDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerdocs)| -|`Scheduler.list_jobs()`|[getSchedulerJobs](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerjobs)| -|`session()`|[getSessionInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getsessioninformation)| -|`uuids()`|[getUuids](https://cloud.ibm.com/apidocs/cloudant?code=python#getuuids)| -|`db.delete()`|[deleteDatabase](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedatabase)| -|`db.metadata()`|[getDatabaseInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getdatabaseinformation)| -|`db.create_document()`|[postDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#postdocument)| -|`db.create()`|[putDatabase](https://cloud.ibm.com/apidocs/cloudant?code=python#putdatabase)| -|`db.all_docs()/db.keys()`|[postAllDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postalldocs)| -|`db.bulk_docs()`|[postBulkDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postbulkdocs)| -|`db.changes()/db.infinite_changes()`|[postChanges](https://cloud.ibm.com/apidocs/cloudant?code=python#postchanges)| -|`DesignDocument(db, '_design/doc').delete()`|[deleteDesignDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedesigndocument)| -|`db.get_design_document()/DesignDocument(db, '_design/doc').fetch()`|[getDesignDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getdesigndocument)| -|`DesignDocument(db, '_design/doc').save()`|[putDesignDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putdesigndocument)| -|`DesignDocument(db, '_design/doc').info()`|[getDesignDocumentInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getdesigndocumentinformation)| -|`db.get_search_result()`|[postSearch](https://cloud.ibm.com/apidocs/cloudant?code=python#postsearch)| -|`db.get_view_result()`|[postView](https://cloud.ibm.com/apidocs/cloudant?code=python#postview)| -|`db.list_design_documents()`|[postDesignDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postdesigndocs)| -|`db.get_query_result()`|[postFind](https://cloud.ibm.com/apidocs/cloudant?code=python#postfind)| -|`db.get_query_indexes()`|[getIndexesInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getindexesinformation)| -|`db.create_query_index()`|[postIndex](https://cloud.ibm.com/apidocs/cloudant?code=python#postindex)| -|`db.delete_query_index()`|[deleteIndex](https://cloud.ibm.com/apidocs/cloudant?code=python#deleteindex)| -|`Document(db, '_local/docid').fetch()`|[getLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getlocaldocument)| -|`Document(db, '_local/docid').save()`|[putLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putlocaldocument)| -|`Document(db, '_local/docid').delete()`|[deleteLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#deletelocaldocument)| -|`db.missing_revisions()`|[postMissingRevs](https://cloud.ibm.com/apidocs/cloudant?code=python#postmissingrevs)| -|`db.partition_metadata()`|[getPartitionInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getpartitioninformation)| -|`db.partitioned_all_docs()`|[postPartitionAllDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionalldocs)| -|`db.get_partitioned_search_result()`|[postPartitionSearch](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionsearch)| -|`db.get_partitioned_view_result()`|[postPartitionView](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionview)| -|`db.get_partitioned_query_result()`|[postPartitionFind](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionfind)| -|`db.revisions_diff()`|[postRevsDiff](https://cloud.ibm.com/apidocs/cloudant?code=python#postrevsdiff)| -|`db.get_security_document()/db.security_document()`|[getSecurity](https://cloud.ibm.com/apidocs/cloudant?code=python#getsecurity)| -|`db.share_database()`|[putSecurity](https://cloud.ibm.com/apidocs/cloudant?code=python#putsecurity)| -|`db.shards()`|[getShardsInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getshardsinformation)| -|`Document(db, 'docid').delete()`|[deleteDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedocument)| -|`Document(db, 'docid').fetch()`|[getDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getdocument)| -|`Document(db, 'docid').exists()`|[headDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#headdocument)| -|`Document(db, 'docid').save()`|[putDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putdocument)| -|`Document(db, 'docid').delete_attachment()`|[deleteAttachment](https://cloud.ibm.com/apidocs/cloudant?code=python#deleteattachment)| -|`Document(db, 'docid').get_attachment()`|[getAttachment](https://cloud.ibm.com/apidocs/cloudant?code=python#getattachment)| -|`Document(db, 'docid').put_attachment()`|[putAttachment](https://cloud.ibm.com/apidocs/cloudant?code=python#putattachment)| -|`generate_api_key()`|[postApiKeys](https://cloud.ibm.com/apidocs/cloudant?code=python#postapikeys)| -|`SecurityDocument().save()`|[putCloudantSecurityConfiguration](https://cloud.ibm.com/apidocs/cloudant?code=python#putcloudantsecurityconfiguration)| -|`cors_configuration()/cors_origin()`|[getCorsInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getcorsinformation)| -|`update_cors_configuration()`|[putCorsConfiguration](https://cloud.ibm.com/apidocs/cloudant?code=python#putcorsconfiguration)| +|`metadata()`|[`getServerInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getserverinformation)| +|`all_dbs()`|[`getAllDbs`](https://cloud.ibm.com/apidocs/cloudant?code=python#getalldbs)| +|`db_updates()/infinite_db_updates()`|[`getDbUpdates`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdbupdates)| +|`Replicator.stop_replication()`|[`deleteReplicationDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#deletereplicationdocument)| +|`Replicator.replication_state()`|[`getReplicationDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getreplicationdocument)| +|`Replicator.create_replication()`|[`putReplicationDocument`](https://cloud.ibm.com/apidocs/cloudant?code=#putreplicationdocument)| +|`Scheduler.get_doc()`|[`getSchedulerDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerdocument)| +|`Scheduler.list_docs()`|[`getSchedulerDocs`](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerdocs)| +|`Scheduler.list_jobs()`|[`getSchedulerJobs`](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerjobs)| +|`session()`|[`getSessionInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getsessioninformation)| +|`uuids()`|[`getUuids`](https://cloud.ibm.com/apidocs/cloudant?code=python#getuuids)| +|`db.delete()`|[`deleteDatabase`](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedatabase)| +|`db.metadata()`|[`getDatabaseInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdatabaseinformation)| +|`db.create_document()`|[`postDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#postdocument)| +|`db.create()`|[`putDatabase`](https://cloud.ibm.com/apidocs/cloudant?code=python#putdatabase)| +|`db.all_docs()/db.keys()`|[`postAllDocs`](https://cloud.ibm.com/apidocs/cloudant?code=python#postalldocs)| +|`db.bulk_docs()`|[`postBulkDocs`](https://cloud.ibm.com/apidocs/cloudant?code=python#postbulkdocs)| +|`db.changes()/db.infinite_changes()`|[`postChanges`](https://cloud.ibm.com/apidocs/cloudant?code=python#postchanges-databases)| +|`DesignDocument(db, '_design/doc').delete()`|[`deleteDesignDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedesigndocument)| +|`db.get_design_document()/DesignDocument(db, '_design/doc').fetch()`|[`getDesignDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdesigndocument)| +|`DesignDocument(db, '_design/doc').save()`|[`putDesignDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#putdesigndocument)| +|`DesignDocument(db, '_design/doc').info()`|[`getDesignDocumentInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdesigndocumentinformation)| +|`db.get_search_result()`|[`postSearch`](https://cloud.ibm.com/apidocs/cloudant?code=python#postsearch)| +|`db.get_view_result()`|[`postView`](https://cloud.ibm.com/apidocs/cloudant?code=python#postview)| +|`db.list_design_documents()`|[`postDesignDocs`](https://cloud.ibm.com/apidocs/cloudant?code=python#postdesigndocs)| +|`db.get_query_result()`|[`postFind`](https://cloud.ibm.com/apidocs/cloudant?code=python#postfind)| +|`db.get_query_indexes()`|[`getIndexesInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getindexesinformation)| +|`db.create_query_index()`|[`postIndex`](https://cloud.ibm.com/apidocs/cloudant?code=python#postindex)| +|`db.delete_query_index()`|[`deleteIndex`](https://cloud.ibm.com/apidocs/cloudant?code=python#deleteindex)| +|`Document(db, '_local/docid').fetch()`|[`getLocalDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getlocaldocument)| +|`Document(db, '_local/docid').save()`|[`putLocalDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#putlocaldocument)| +|`Document(db, '_local/docid').delete()`|[`deleteLocalDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#deletelocaldocument)| +|`db.missing_revisions()/db.revisions_diff()`|[`postRevsDiff`](https://cloud.ibm.com/apidocs/cloudant?code=python#postrevsdiff)| +|`db.partition_metadata()`|[`getPartitionInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getpartitioninformation)| +|`db.partitioned_all_docs()`|[`postPartitionAllDocs`](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionalldocs)| +|`db.get_partitioned_search_result()`|[`postPartitionSearch`](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionsearch)| +|`db.get_partitioned_view_result()`|[`postPartitionView`](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionview)| +|`db.get_partitioned_query_result()`|[`postPartitionFind`](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionfind-partitioned-databases)| +|`db.get_security_document()/db.security_document()`|[`getSecurity`](https://cloud.ibm.com/apidocs/cloudant?code=python#getsecurity)| +|`db.share_database()`|[`putSecurity`](https://cloud.ibm.com/apidocs/cloudant?code=python#putsecurity)| +|`db.shards()`|[`getShardsInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getshardsinformation)| +|`Document(db, 'docid').delete()`|[`deleteDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedocument)| +|`Document(db, 'docid').fetch()`|[`getDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdocument)| +|`Document(db, 'docid').exists()`|[`headDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#headdocument)| +|`Document(db, 'docid').save()`|[`putDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#putdocument)| +|`Document(db, 'docid').delete_attachment()`|[`deleteAttachment`](https://cloud.ibm.com/apidocs/cloudant?code=python#deleteattachment)| +|`Document(db, 'docid').get_attachment()`|[`getAttachment`](https://cloud.ibm.com/apidocs/cloudant?code=python#getattachment)| +|`Document(db, 'docid').put_attachment()`|[`putAttachment`](https://cloud.ibm.com/apidocs/cloudant?code=python#putattachment)| +|`generate_api_key()`|[`postApiKeys`](https://cloud.ibm.com/apidocs/cloudant?code=python#postapikeys)| +|`SecurityDocument().save()`|[`putCloudantSecurityConfiguration`](https://cloud.ibm.com/apidocs/cloudant?code=python#putcloudantsecurity)| +|`cors_configuration()/cors_origin()`|[`getCorsInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getcorsinformation)| +|`update_cors_configuration()`|[`putCorsConfiguration`](https://cloud.ibm.com/apidocs/cloudant?code=python#putcorsconfiguration)| From a92c6593ba5db69fda64b462a9847d7e06cc9cf4 Mon Sep 17 00:00:00 2001 From: Ildiko Eigel Date: Tue, 21 Dec 2021 10:58:05 +0100 Subject: [PATCH 90/97] Remove assertion from test_get_feed_descending --- tests/unit/changes_tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/changes_tests.py b/tests/unit/changes_tests.py index d39ab159..8d937214 100644 --- a/tests/unit/changes_tests.py +++ b/tests/unit/changes_tests.py @@ -245,7 +245,6 @@ def test_get_feed_descending(self): self.assertTrue(current < last) except AssertionError: self.assertEqual(current, last) - self.assertTrue(len(change['seq']) > len(last_seq)) seq_list.append(change['seq']) last_seq = change['seq'] self.assertEqual(len(seq_list), 50) @@ -522,7 +521,7 @@ def test_invalid_style_value(self): with self.assertRaises(CloudantArgumentError) as cm: invalid_feed = [x for x in feed] self.assertEqual( - str(cm.exception), + str(cm.exception), 'Invalid value (foo) for style option. Must be main_only, or all_docs.') if __name__ == '__main__': From e113b8f0da43d5cb9f90a84502eb5cf32d3ba1de Mon Sep 17 00:00:00 2001 From: Ildiko Eigel Date: Tue, 21 Dec 2021 10:58:54 +0100 Subject: [PATCH 91/97] Update date in copyright header --- tests/unit/changes_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/changes_tests.py b/tests/unit/changes_tests.py index 8d937214..5ec2f02e 100644 --- a/tests/unit/changes_tests.py +++ b/tests/unit/changes_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2016, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 8d968f4c048a320c2398436333b225d792837762 Mon Sep 17 00:00:00 2001 From: Ildiko Eigel Date: Tue, 21 Dec 2021 11:34:34 +0100 Subject: [PATCH 92/97] Update comment --- tests/unit/changes_tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/changes_tests.py b/tests/unit/changes_tests.py index 5ec2f02e..b0e9cb3e 100644 --- a/tests/unit/changes_tests.py +++ b/tests/unit/changes_tests.py @@ -227,8 +227,7 @@ def test_get_feed_descending(self): Test getting content back for a descending feed. When testing, the sequence identifier is in the form of -. Often times the number prefix sorts as expected when using descending but sometimes the - number prefix is repeated. In these cases the check is to see if the following - random character sequence suffix is longer than its predecessor. + number prefix is repeated. """ self.populate_db_with_documents(50) feed = Feed(self.db, descending=True) From ab941812c84cfbcf90c85ae63fb034fbe136780d Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Wed, 22 Dec 2021 14:19:22 +0000 Subject: [PATCH 93/97] Use assertEqual instead of assertEquals for Python 3.11 compatibility. --- tests/unit/changes_tests.py | 6 +- tests/unit/client_tests.py | 16 +-- tests/unit/database_partition_tests.py | 22 ++--- tests/unit/database_tests.py | 130 ++++++++++++------------- tests/unit/design_document_tests.py | 6 +- tests/unit/document_tests.py | 8 +- tests/unit/iam_auth_tests.py | 4 +- tests/unit/index_tests.py | 120 +++++++++++------------ tests/unit/replicator_mock_tests.py | 12 +-- 9 files changed, 162 insertions(+), 162 deletions(-) diff --git a/tests/unit/changes_tests.py b/tests/unit/changes_tests.py index b0e9cb3e..3df0cef3 100644 --- a/tests/unit/changes_tests.py +++ b/tests/unit/changes_tests.py @@ -474,9 +474,9 @@ def test_get_feed_with_custom_filter_query_params(self): include_docs=False ) params = feed._translate(feed._options) - self.assertEquals(params['filter'], 'mailbox/new_mail') - self.assertEquals(params['foo'], 'bar') - self.assertEquals(params['include_docs'], 'false') + self.assertEqual(params['filter'], 'mailbox/new_mail') + self.assertEqual(params['foo'], 'bar') + self.assertEqual(params['include_docs'], 'false') def test_invalid_argument_type(self): """ diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index 07c4159b..35139168 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -266,7 +266,7 @@ def test_session_basic(self, m_req): timeout=None ) - self.assertEquals(all_dbs, ['animaldb']) + self.assertEqual(all_dbs, ['animaldb']) @mock.patch('cloudant._client_session.Session.request') def test_session_basic_with_no_credentials(self, m_req): @@ -333,7 +333,7 @@ def test_change_credentials_basic(self, m_req): auth=('baz', 'qux'), # uses HTTP Basic Auth timeout=None ) - self.assertEquals(all_dbs, ['animaldb']) + self.assertEqual(all_dbs, ['animaldb']) @skip_if_not_cookie_auth def test_basic_auth_str(self): @@ -442,7 +442,7 @@ def test_create_with_server_error(self, m_req): self.assertEqual(cm.exception.status_code, 500) - self.assertEquals(m_req.call_count, 3) + self.assertEqual(m_req.call_count, 3) m_req.assert_called_with( 'PUT', '/'.join([self.url, dbname]), @@ -730,7 +730,7 @@ def test_cloudant_bluemix_context_helper_with_legacy_creds(self): with cloudant_bluemix(vcap_services, instance_name=instance_name) as c: self.assertIsInstance(c, Cloudant) self.assertIsInstance(c.r_session, requests.Session) - self.assertEquals(c.session()['userCtx']['name'], self.user) + self.assertEqual(c.session()['userCtx']['name'], self.user) except Exception as err: self.fail('Exception {0} was raised.'.format(str(err))) @@ -809,7 +809,7 @@ def test_cloudant_bluemix_dedicated_context_helper(self): service_name=service_name) as c: self.assertIsInstance(c, Cloudant) self.assertIsInstance(c.r_session, requests.Session) - self.assertEquals(c.session()['userCtx']['name'], self.user) + self.assertEqual(c.session()['userCtx']['name'], self.user) except Exception as err: self.fail('Exception {0} was raised.'.format(str(err))) @@ -850,7 +850,7 @@ def test_bluemix_constructor_with_legacy_creds(self): c.connect() self.assertIsInstance(c, Cloudant) self.assertIsInstance(c.r_session, requests.Session) - self.assertEquals(c.session()['userCtx']['name'], self.user) + self.assertEqual(c.session()['userCtx']['name'], self.user) except Exception as err: self.fail('Exception {0} was raised.'.format(str(err))) @@ -916,7 +916,7 @@ def test_bluemix_constructor_specify_instance_name(self): c.connect() self.assertIsInstance(c, Cloudant) self.assertIsInstance(c.r_session, requests.Session) - self.assertEquals(c.session()['userCtx']['name'], self.user) + self.assertEqual(c.session()['userCtx']['name'], self.user) except Exception as err: self.fail('Exception {0} was raised.'.format(str(err))) @@ -960,7 +960,7 @@ def test_bluemix_constructor_with_multiple_services(self): c.connect() self.assertIsInstance(c, Cloudant) self.assertIsInstance(c.r_session, requests.Session) - self.assertEquals(c.session()['userCtx']['name'], self.user) + self.assertEqual(c.session()['userCtx']['name'], self.user) except Exception as err: self.fail('Exception {0} was raised.'.format(str(err))) diff --git a/tests/unit/database_partition_tests.py b/tests/unit/database_partition_tests.py index 52b76e0d..8b3690dd 100644 --- a/tests/unit/database_partition_tests.py +++ b/tests/unit/database_partition_tests.py @@ -63,7 +63,7 @@ def test_create_non_partitioned_design_document(self): def test_partitioned_all_docs(self): for partition_key in self.populate_db_with_partitioned_documents(5, 25): docs = self.db.partitioned_all_docs(partition_key) - self.assertEquals(len(docs['rows']), 25) + self.assertEqual(len(docs['rows']), 25) for doc in docs['rows']: self.assertTrue(doc['id'].startswith(partition_key + ':')) @@ -71,8 +71,8 @@ def test_partitioned_all_docs(self): def test_partition_metadata(self): for partition_key in self.populate_db_with_partitioned_documents(5, 25): meta = self.db.partition_metadata(partition_key) - self.assertEquals(meta['partition'], partition_key) - self.assertEquals(meta['doc_count'], 25) + self.assertEqual(meta['partition'], partition_key) + self.assertEqual(meta['doc_count'], 25) def test_partitioned_search(self): ddoc = DesignDocument(self.db, 'partitioned_search', partitioned=True) @@ -91,7 +91,7 @@ def test_partitioned_search(self): print(result) self.assertTrue(result['id'].startswith(partition_key + ':')) i += 1 - self.assertEquals(i, 10) + self.assertEqual(i, 10) def test_get_partitioned_index(self): index_name = 'test_partitioned_index' @@ -99,16 +99,16 @@ def test_get_partitioned_index(self): self.db.create_query_index(index_name=index_name, fields=['foo']) results = self.db.get_query_indexes() - self.assertEquals(len(results), 2) + self.assertEqual(len(results), 2) index_all_docs = results[0] - self.assertEquals(index_all_docs.name, '_all_docs') - self.assertEquals(type(index_all_docs), SpecialIndex) + self.assertEqual(index_all_docs.name, '_all_docs') + self.assertEqual(type(index_all_docs), SpecialIndex) self.assertFalse(index_all_docs.partitioned) index_partitioned = results[1] - self.assertEquals(index_partitioned.name, index_name) - self.assertEquals(type(index_partitioned), Index) + self.assertEqual(index_partitioned.name, index_name) + self.assertEqual(type(index_partitioned), Index) self.assertTrue(index_partitioned.partitioned) def test_partitioned_query(self): @@ -122,7 +122,7 @@ def test_partitioned_query(self): for result in results: self.assertTrue(result['_id'].startswith(partition_key + ':')) i += 1 - self.assertEquals(i, 10) + self.assertEqual(i, 10) def test_partitioned_view(self): ddoc = DesignDocument(self.db, 'partitioned_view', partitioned=True) @@ -138,4 +138,4 @@ def test_partitioned_view(self): self.assertTrue( result['id'].startswith(partition_key + ':')) i += 1 - self.assertEquals(i, 10) + self.assertEqual(i, 10) diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index 3ba12584..ae9898c7 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -885,7 +885,7 @@ def test_get_set_revision_limit(self, m_req): # Get new revisions limit. self.assertEqual(self.db.get_revision_limit(), 1234) - self.assertEquals(m_req.call_count, 3) + self.assertEqual(m_req.call_count, 3) @attr(db='couch') def test_view_clean_up(self): @@ -1084,19 +1084,19 @@ def test_create_json_index(self): ddoc = self.db[index.design_document_id] - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['indexes'], {}) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) + self.assertEqual(ddoc['indexes'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) index = ddoc['views'][index.name] - self.assertEquals(index['map']['fields']['age'], 'asc') - self.assertEquals(index['map']['fields']['name'], 'asc') - self.assertEquals(index['options']['def']['fields'], ['name', 'age']) - self.assertEquals(index['reduce'], '_count') + self.assertEqual(index['map']['fields']['age'], 'asc') + self.assertEqual(index['map']['fields']['name'], 'asc') + self.assertEqual(index['options']['def']['fields'], ['name', 'age']) + self.assertEqual(index['reduce'], '_count') @attr(couchapi=2) def test_delete_json_index(self): @@ -1369,22 +1369,22 @@ def test_create_text_index(self): ddoc = self.db[index.design_document_id] - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) - self.assertEquals(ddoc['views'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + self.assertEqual(ddoc['views'], {}) text_index = ddoc['indexes'][index.name] - self.assertEquals(text_index['analyzer']['default'], 'keyword') - self.assertEquals(text_index['analyzer']['fields']['$default'], 'standard') - self.assertEquals(text_index['analyzer']['name'], 'perfield') - self.assertEquals(text_index['index']['default_analyzer'], 'keyword') - self.assertEquals(text_index['index']['default_field'], {}) - self.assertEquals(text_index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) - self.assertEquals(text_index['index']['selector'], {}) + self.assertEqual(text_index['analyzer']['default'], 'keyword') + self.assertEqual(text_index['analyzer']['fields']['$default'], 'standard') + self.assertEqual(text_index['analyzer']['name'], 'perfield') + self.assertEqual(text_index['index']['default_analyzer'], 'keyword') + self.assertEqual(text_index['index']['default_field'], {}) + self.assertEqual(text_index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) + self.assertEqual(text_index['index']['selector'], {}) self.assertTrue(text_index['index']['index_array_lengths']) def test_create_all_fields_text_index(self): @@ -1396,22 +1396,22 @@ def test_create_all_fields_text_index(self): ddoc = self.db[index.design_document_id] - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) - self.assertEquals(ddoc['views'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + self.assertEqual(ddoc['views'], {}) index = ddoc['indexes'][index.name] - self.assertEquals(index['analyzer']['default'], 'keyword') - self.assertEquals(index['analyzer']['fields'], {'$default': 'standard'}) - self.assertEquals(index['analyzer']['name'], 'perfield') - self.assertEquals(index['index']['default_analyzer'], 'keyword') - self.assertEquals(index['index']['default_field'], {}) - self.assertEquals(index['index']['fields'], 'all_fields') - self.assertEquals(index['index']['selector'], {}) + self.assertEqual(index['analyzer']['default'], 'keyword') + self.assertEqual(index['analyzer']['fields'], {'$default': 'standard'}) + self.assertEqual(index['analyzer']['name'], 'perfield') + self.assertEqual(index['index']['default_analyzer'], 'keyword') + self.assertEqual(index['index']['default_field'], {}) + self.assertEqual(index['index']['fields'], 'all_fields') + self.assertEqual(index['index']['selector'], {}) self.assertTrue(index['index']['index_array_lengths']) def test_create_multiple_indexes_one_ddoc(self): @@ -1436,27 +1436,27 @@ def test_create_multiple_indexes_one_ddoc(self): ddoc = self.db['_design/ddoc001'] - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('2-')) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) json_index = ddoc['views']['json-index-001'] - self.assertEquals(json_index['map']['fields']['age'], 'asc') - self.assertEquals(json_index['map']['fields']['name'], 'asc') - self.assertEquals(json_index['options']['def']['fields'], ['name', 'age']) - self.assertEquals(json_index['reduce'], '_count') + self.assertEqual(json_index['map']['fields']['age'], 'asc') + self.assertEqual(json_index['map']['fields']['name'], 'asc') + self.assertEqual(json_index['options']['def']['fields'], ['name', 'age']) + self.assertEqual(json_index['reduce'], '_count') text_index = ddoc['indexes']['text-index-001'] - self.assertEquals(text_index['analyzer']['default'], 'keyword') - self.assertEquals(text_index['analyzer']['fields']['$default'], 'standard') - self.assertEquals(text_index['analyzer']['name'], 'perfield') - self.assertEquals(text_index['index']['default_analyzer'], 'keyword') - self.assertEquals(text_index['index']['default_field'], {}) - self.assertEquals(text_index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) - self.assertEquals(text_index['index']['selector'], {}) + self.assertEqual(text_index['analyzer']['default'], 'keyword') + self.assertEqual(text_index['analyzer']['fields']['$default'], 'standard') + self.assertEqual(text_index['analyzer']['name'], 'perfield') + self.assertEqual(text_index['index']['default_analyzer'], 'keyword') + self.assertEqual(text_index['index']['default_field'], {}) + self.assertEqual(text_index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) + self.assertEqual(text_index['index']['selector'], {}) self.assertTrue(text_index['index']['index_array_lengths']) def test_create_query_index_failure(self): @@ -1513,28 +1513,28 @@ def test_get_query_indexes_raw(self): indexes = self.db.get_query_indexes(raw_result=True) - self.assertEquals(indexes['total_rows'], 3) + self.assertEqual(indexes['total_rows'], 3) all_docs_index = indexes['indexes'][0] - self.assertEquals(all_docs_index['ddoc'], None) - self.assertEquals(all_docs_index['def']['fields'], [{'_id': 'asc'}]) - self.assertEquals(all_docs_index['name'], '_all_docs') - self.assertEquals(all_docs_index['type'], 'special') + self.assertEqual(all_docs_index['ddoc'], None) + self.assertEqual(all_docs_index['def']['fields'], [{'_id': 'asc'}]) + self.assertEqual(all_docs_index['name'], '_all_docs') + self.assertEqual(all_docs_index['type'], 'special') json_index = indexes['indexes'][1] - self.assertEquals(json_index['ddoc'], '_design/ddoc001') - self.assertEquals(json_index['def']['fields'], [{'name': 'asc'}, {'age': 'asc'}]) - self.assertEquals(json_index['name'], 'json-idx-001') - self.assertEquals(json_index['type'], 'json') + self.assertEqual(json_index['ddoc'], '_design/ddoc001') + self.assertEqual(json_index['def']['fields'], [{'name': 'asc'}, {'age': 'asc'}]) + self.assertEqual(json_index['name'], 'json-idx-001') + self.assertEqual(json_index['type'], 'json') text_index = indexes['indexes'][2] - self.assertEquals(text_index['ddoc'], '_design/ddoc001') - self.assertEquals(text_index['def']['default_analyzer'], 'keyword') - self.assertEquals(text_index['def']['default_field'], {}) - self.assertEquals(text_index['def']['fields'], []) - self.assertEquals(text_index['def']['selector'], {}) - self.assertEquals(text_index['name'], 'text-idx-001') - self.assertEquals(text_index['type'], 'text') + self.assertEqual(text_index['ddoc'], '_design/ddoc001') + self.assertEqual(text_index['def']['default_analyzer'], 'keyword') + self.assertEqual(text_index['def']['default_field'], {}) + self.assertEqual(text_index['def']['fields'], []) + self.assertEqual(text_index['def']['selector'], {}) + self.assertEqual(text_index['name'], 'text-idx-001') + self.assertEqual(text_index['type'], 'text') self.assertTrue(text_index['def']['index_array_lengths']) def test_get_query_indexes(self): diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index 52352bcc..86a2b5f1 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -843,9 +843,9 @@ def test_get_search_info(self): # Validate the metadata search_index_metadata = search_info['search_index'] self.assertIsNotNone(search_index_metadata) - self.assertEquals(search_index_metadata['doc_del_count'], 0, 'There should be no deleted docs.') + self.assertEqual(search_index_metadata['doc_del_count'], 0, 'There should be no deleted docs.') self.assertTrue(search_index_metadata['doc_count'] <= 100, 'There should be 100 or fewer docs.') - self.assertEquals(search_index_metadata['committed_seq'], 0, 'The committed_seq should be 0.') + self.assertEqual(search_index_metadata['committed_seq'], 0, 'The committed_seq should be 0.') self.assertTrue(search_index_metadata['pending_seq'] <= 101, 'The pending_seq should be 101 or fewer.') self.assertTrue(search_index_metadata['disk_size'] >0, 'The disk_size should be greater than 0.') @@ -1275,7 +1275,7 @@ def test_rewrite_rule(self): doc = Document(self.db, 'rewrite_doc') doc.save() resp = self.client.r_session.get('/'.join([ddoc.document_url, '_rewrite'])) - self.assertEquals( + self.assertEqual( response_to_json_dict(resp), { '_id': 'rewrite_doc', diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index 61fa9fac..09cb09c4 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -944,14 +944,14 @@ def object_hook(self, obj): raw_doc = self.db.all_docs(include_docs=True)['rows'][0]['doc'] - self.assertEquals(raw_doc['name'], 'julia') - self.assertEquals(raw_doc['dt']['_type'], 'datetime') - self.assertEquals(raw_doc['dt']['value'], '2018-07-09T15:11:10') + self.assertEqual(raw_doc['name'], 'julia') + self.assertEqual(raw_doc['dt']['_type'], 'datetime') + self.assertEqual(raw_doc['dt']['value'], '2018-07-09T15:11:10') doc2 = Document(self.db, doc['_id'], decoder=DTDecoder) doc2.fetch() - self.assertEquals(doc2['dt'], doc['dt']) + self.assertEqual(doc2['dt'], doc['dt']) if __name__ == '__main__': unittest.main() diff --git a/tests/unit/iam_auth_tests.py b/tests/unit/iam_auth_tests.py index f36af4e0..3e0c7cf2 100644 --- a/tests/unit/iam_auth_tests.py +++ b/tests/unit/iam_auth_tests.py @@ -88,12 +88,12 @@ def _mock_cookie(expires_secs=300): def test_iam_set_credentials(self): iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984') - self.assertEquals(iam._api_key, MOCK_API_KEY) + self.assertEqual(iam._api_key, MOCK_API_KEY) new_api_key = 'some_new_api_key' iam.set_credentials(None, new_api_key) - self.assertEquals(iam._api_key, new_api_key) + self.assertEqual(iam._api_key, new_api_key) @mock.patch('cloudant._client_session.ClientSession.request') def test_iam_get_access_token(self, m_req): diff --git a/tests/unit/index_tests.py b/tests/unit/index_tests.py index 29a3d51c..f12cf158 100644 --- a/tests/unit/index_tests.py +++ b/tests/unit/index_tests.py @@ -161,21 +161,21 @@ def test_create_an_index_using_ddoc_index_name(self): with DesignDocument(self.db, index.design_document_id) as ddoc: self.assertIsInstance(ddoc.get_view(index.name), QueryIndexView) - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['indexes'], {}) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) + self.assertEqual(ddoc['indexes'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) self.assertListEqual(list(ddoc['views'].keys()), ['index001']) view = ddoc['views'][index.name] - self.assertEquals(view['map']['fields']['age'], 'asc') - self.assertEquals(view['map']['fields']['name'], 'asc') - self.assertEquals(view['options']['def']['fields'], ['name', 'age']) - self.assertEquals(view['reduce'], '_count') + self.assertEqual(view['map']['fields']['age'], 'asc') + self.assertEqual(view['map']['fields']['name'], 'asc') + self.assertEqual(view['options']['def']['fields'], ['name', 'age']) + self.assertEqual(view['reduce'], '_count') def test_create_an_index_without_ddoc_index_name(self): """ @@ -189,21 +189,21 @@ def test_create_an_index_without_ddoc_index_name(self): with DesignDocument(self.db, index.design_document_id) as ddoc: self.assertIsInstance(ddoc.get_view(index.name), QueryIndexView) - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['indexes'], {}) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) + self.assertEqual(ddoc['indexes'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) self.assertListEqual(list(ddoc['views'].keys()), [index.name]) view = ddoc['views'][index.name] - self.assertEquals(view['map']['fields']['age'], 'asc') - self.assertEquals(view['map']['fields']['name'], 'asc') - self.assertEquals(view['options']['def']['fields'], ['name', 'age']) - self.assertEquals(view['reduce'], '_count') + self.assertEqual(view['map']['fields']['age'], 'asc') + self.assertEqual(view['map']['fields']['name'], 'asc') + self.assertEqual(view['options']['def']['fields'], ['name', 'age']) + self.assertEqual(view['reduce'], '_count') def test_create_an_index_with_empty_ddoc_index_name(self): """ @@ -217,21 +217,21 @@ def test_create_an_index_with_empty_ddoc_index_name(self): with DesignDocument(self.db, index.design_document_id) as ddoc: self.assertIsInstance(ddoc.get_view(index.name), QueryIndexView) - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['indexes'], {}) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) + self.assertEqual(ddoc['indexes'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) self.assertListEqual(list(ddoc['views'].keys()), [index.name]) view = ddoc['views'][index.name] - self.assertEquals(view['map']['fields']['age'], 'asc') - self.assertEquals(view['map']['fields']['name'], 'asc') - self.assertEquals(view['options']['def']['fields'], ['name', 'age']) - self.assertEquals(view['reduce'], '_count') + self.assertEqual(view['map']['fields']['age'], 'asc') + self.assertEqual(view['map']['fields']['name'], 'asc') + self.assertEqual(view['options']['def']['fields'], ['name', 'age']) + self.assertEqual(view['reduce'], '_count') def test_create_an_index_using_design_prefix(self): """ @@ -245,21 +245,21 @@ def test_create_an_index_using_design_prefix(self): with DesignDocument(self.db, index.design_document_id) as ddoc: self.assertIsInstance(ddoc.get_view(index.name), QueryIndexView) - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['indexes'], {}) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) + self.assertEqual(ddoc['indexes'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) self.assertListEqual(list(ddoc['views'].keys()), [index.name]) view = ddoc['views'][index.name] - self.assertEquals(view['map']['fields']['age'], 'asc') - self.assertEquals(view['map']['fields']['name'], 'asc') - self.assertEquals(view['options']['def']['fields'], ['name', 'age']) - self.assertEquals(view['reduce'], '_count') + self.assertEqual(view['map']['fields']['age'], 'asc') + self.assertEqual(view['map']['fields']['name'], 'asc') + self.assertEqual(view['options']['def']['fields'], ['name', 'age']) + self.assertEqual(view['reduce'], '_count') def test_create_uses_custom_encoder(self): """ @@ -446,22 +446,22 @@ def test_create_a_search_index_no_kwargs(self): self.assertEqual(index.design_document_id, '_design/ddoc001') self.assertEqual(index.name, 'index001') with DesignDocument(self.db, index.design_document_id) as ddoc: - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) - self.assertEquals(ddoc['views'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + self.assertEqual(ddoc['views'], {}) index = ddoc['indexes']['index001'] - self.assertEquals(index['analyzer']['default'], 'keyword') - self.assertEquals(index['analyzer']['fields']['$default'], 'standard') - self.assertEquals(index['analyzer']['name'], 'perfield') - self.assertEquals(index['index']['default_analyzer'], 'keyword') - self.assertEquals(index['index']['default_field'], {}) - self.assertEquals(index['index']['fields'], 'all_fields') - self.assertEquals(index['index']['selector'], {}) + self.assertEqual(index['analyzer']['default'], 'keyword') + self.assertEqual(index['analyzer']['fields']['$default'], 'standard') + self.assertEqual(index['analyzer']['name'], 'perfield') + self.assertEqual(index['index']['default_analyzer'], 'keyword') + self.assertEqual(index['index']['default_field'], {}) + self.assertEqual(index['index']['fields'], 'all_fields') + self.assertEqual(index['index']['selector'], {}) self.assertTrue(index['index']['index_array_lengths']) def test_create_a_search_index_with_kwargs(self): @@ -480,22 +480,22 @@ def test_create_a_search_index_with_kwargs(self): self.assertEqual(index.design_document_id, '_design/ddoc001') self.assertEqual(index.name, 'index001') with DesignDocument(self.db, index.design_document_id) as ddoc: - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) - self.assertEquals(ddoc['views'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + self.assertEqual(ddoc['views'], {}) index = ddoc['indexes']['index001'] - self.assertEquals(index['analyzer']['default'], 'keyword') - self.assertEquals(index['analyzer']['fields']['$default'], 'german') - self.assertEquals(index['analyzer']['name'], 'perfield') - self.assertEquals(index['index']['default_analyzer'], 'keyword') - self.assertEquals(index['index']['default_field']['analyzer'], 'german') - self.assertEquals(index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) - self.assertEquals(index['index']['selector'], {}) + self.assertEqual(index['analyzer']['default'], 'keyword') + self.assertEqual(index['analyzer']['fields']['$default'], 'german') + self.assertEqual(index['analyzer']['name'], 'perfield') + self.assertEqual(index['index']['default_analyzer'], 'keyword') + self.assertEqual(index['index']['default_field']['analyzer'], 'german') + self.assertEqual(index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) + self.assertEqual(index['index']['selector'], {}) self.assertTrue(index['index']['default_field']['enabled']) self.assertTrue(index['index']['index_array_lengths']) diff --git a/tests/unit/replicator_mock_tests.py b/tests/unit/replicator_mock_tests.py index 96589f9e..f49b04d4 100644 --- a/tests/unit/replicator_mock_tests.py +++ b/tests/unit/replicator_mock_tests.py @@ -86,9 +86,9 @@ def test_using_admin_party_source_and_target(self): rep.create_replication(src, tgt, repl_id=self.repl_id) kcall = m_replicator.create_document.call_args_list - self.assertEquals(len(kcall), 1) + self.assertEqual(len(kcall), 1) args, kwargs = kcall[0] - self.assertEquals(len(args), 1) + self.assertEqual(len(args), 1) expected_doc = { '_id': self.repl_id, @@ -118,9 +118,9 @@ def test_using_basic_auth_source_and_target(self): src, tgt, repl_id=self.repl_id, user_ctx=self.user_ctx) kcall = m_replicator.create_document.call_args_list - self.assertEquals(len(kcall), 1) + self.assertEqual(len(kcall), 1) args, kwargs = kcall[0] - self.assertEquals(len(args), 1) + self.assertEqual(len(args), 1) expected_doc = { '_id': self.repl_id, @@ -154,9 +154,9 @@ def test_using_iam_auth_source_and_target(self): src, tgt, repl_id=self.repl_id, user_ctx=self.user_ctx) kcall = m_replicator.create_document.call_args_list - self.assertEquals(len(kcall), 1) + self.assertEqual(len(kcall), 1) args, kwargs = kcall[0] - self.assertEquals(len(args), 1) + self.assertEqual(len(args), 1) expected_doc = { '_id': self.repl_id, From 5b1ecc215b2caea22ccc2d7310462df56be6e848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eigel=20Ildik=C3=B3?= Date: Tue, 11 Jan 2022 14:21:54 +0100 Subject: [PATCH 94/97] [MIGRATION] Add note about authentication errors and server errors (#512) --- MIGRATION.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index 21c9ae5f..a122d7dd 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -20,8 +20,21 @@ content like using `raw_result=True` in `python-cloudant`. 1. Replay adapters are replaced by the [automatic retries](https://github. com/IBM/ibm-cloud-sdk-common/#automatic-retries) feature for failed requests. 1. Error handling is not transferable from `python-cloudant` to `cloudant-python-sdk`. For more information go to the [Error handling section](https://cloud.ibm.com/apidocs/cloudant?code=python#error-handling) in our API docs. -1. Custom HTTP client configurations in `python-cloudant` are not transferable to `python-java-sdk`. For more information go to the [Configuring the HTTP client section(https://githubcom/IBM/ibm-cloud-sdk-common/#configuring-the-http-client) in the IBM Cloud SDK Common README. - +1. Custom HTTP client configurations in `python-cloudant` can be set differently in + `cloudant-python-sdk`. For more information go to the + [Configuring the HTTP client section](https://github.com/IBM/ibm-cloud-sdk-common/#configuring-the-http-client) + in the IBM Cloud SDK Common README. + +### Troubleshooting +1. Authentication errors occur during service instantiation. For example, the code `service = + CloudantV1.new_instance(service_name="EXAMPLE")` will fail with `ValueError: At least one of + iam_profile_name or iam_profile_id must be specified.` if required environment variables + prefixed with `EXAMPLE` are not set. +1. Server errors occur when running a request against the service. We suggest to + check server errors with + [`getServerInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getserverinformation) + which is the new alternative of `metadata()`. + ## Request mapping Here's a list of the top 5 most frequently used `python-cloudant` operations and the `cloudant-python-sdk` equivalent API operation documentation link: From fe141fcc3b3a3614e9b780317f3fb6efdebceb99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eigel=20Ildik=C3=B3?= Date: Thu, 27 Jan 2022 15:59:05 +0100 Subject: [PATCH 95/97] doc: note full deprecation (#513) --- CHANGES.md | 3 +++ README.md | 7 +++---- src/cloudant/__init__.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d5cf74e5..75402ba5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,6 @@ +# UNRELEASED +- [DEPRECATED] This library is end-of-life and no longer supported. + # 2.15.0 (2021-08-26) - [NEW] Override `dict.get` method for `CouchDatabase` to add `remote` parameter allowing it to retrieve a remote document if specified. diff --git a/README.md b/README.md index 029edf07..467c9dd3 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,8 @@ -# DEPRECATED +# :warning: NO LONGER MAINTAINED :warning: -**This library is now deprecated and will be end-of-life on Dec 31 2021.** +**This library is end-of-life and no longer supported.** -The library remains supported until the end-of-life date, -but will receive only _critical_ maintenance updates. +This repository will not be updated. The repository will be kept available in read-only mode. Please see the [Migration Guide](./MIGRATION.md) for advice about migrating to our replacement library diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 66769dad..8e4d7e33 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -24,7 +24,7 @@ from .client import Cloudant, CouchDB from ._common_util import CloudFoundryService -warnings.warn('The module cloudant is now deprecated. The replacement is ibmcloudant.', +warnings.warn('The module cloudant is now end-of-life. The replacement is ibmcloudant.', DeprecationWarning) @contextlib.contextmanager From 3636169ef2212f70668c70d169533653817a9ae7 Mon Sep 17 00:00:00 2001 From: Ivan Date: Wed, 9 Mar 2022 10:17:33 +0000 Subject: [PATCH 96/97] Update MIGRATION.md Fixing small typo that prevented a markdown link to render properly. --- MIGRATION.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index a122d7dd..3323e1d8 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -17,8 +17,7 @@ remote HTTP API. For example, in the case of the document context manager, this to fetch and a `put_document` to save. 1. In `cloudant-python-sdk` View, Search, and Query (aka `_find` endpoint) operation responses contain raw JSON content like using `raw_result=True` in `python-cloudant`. -1. Replay adapters are replaced by the [automatic retries](https://github. - com/IBM/ibm-cloud-sdk-common/#automatic-retries) feature for failed requests. +1. Replay adapters are replaced by the [automatic retries](https://github.com/IBM/ibm-cloud-sdk-common/#automatic-retries) feature for failed requests. 1. Error handling is not transferable from `python-cloudant` to `cloudant-python-sdk`. For more information go to the [Error handling section](https://cloud.ibm.com/apidocs/cloudant?code=python#error-handling) in our API docs. 1. Custom HTTP client configurations in `python-cloudant` can be set differently in `cloudant-python-sdk`. For more information go to the From 1fb0bb577e8fba6dbf95919d1e5f6326057e0b01 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Fri, 30 Aug 2024 16:36:34 +0000 Subject: [PATCH 97/97] deprecation warning --- docs/index.rst | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 0f642eed..a26ea611 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,14 +1 @@ -python-cloudant documentation -============================= - -This is the official Cloudant client library for Python. - -.. toctree:: - :maxdepth: 4 - - compatibility - getting_started - cloudant - -* :ref:`genindex` - +This library is end-of-life and no longer supported.