diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index ba295fa1..95d86bf6 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,5 +1,23 @@ -Please include the following information in your ticket. +Please [read these guidelines](http://ibm.biz/cdt-issue-guide) before opening an issue. -- Cloudant (python-cloudant) version(s) that are affected by this issue. -- Python version -- A small code sample that demonstrates the issue. + + +## Bug Description + +### 1. Steps to reproduce and the simplest code sample possible to demonstrate the issue + + +### 2. What you expected to happen + +### 3. What actually happened + +## Environment details + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6a0b96ea..5b5697c2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,23 +1,102 @@ + +## Checklist -- [ ] You have signed the CLA as per the instructions in [CONTRIBUTING.rst](https://github.com/cloudant/python-cloudant/blob/master/CONTRIBUTING.rst#contributor-license-agreement) -- [ ] You have added tests for any code changes -- [ ] You have updated the [CHANGES.rst](https://github.com/cloudant/python-cloudant/blob/master/CHANGES.rst) -- [ ] You have completed the PR template below: +- [ ] Tick to sign-off your agreement to the [Developer Certificate of Origin (DCO) 1.1](../blob/master/DCO1.1.txt) +- [ ] Added tests for code changes _or_ test/build only changes +- [ ] Updated the change log file (`CHANGES.md`) _or_ test/build only changes +- [ ] Completed the PR template below: -## What +## Description + + +## Approach + + + +## Schema & API Changes + + + +## Security and Privacy + + ## Testing -How to test your changes work, not required for documentation changes. + + +## Monitoring and Logging + diff --git a/.travis.yml b/.travis.yml index 4b2f9ae4..9f770aac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,33 @@ +sudo: required + language: python python: - - "2.7" - - "3.5" + - "3.8" env: - - ADMIN_PARTY=true - - ADMIN_PARTY=false + - ADMIN_PARTY=true COUCHDB_VERSION=2.3.1 + - ADMIN_PARTY=false COUCHDB_VERSION=2.3.1 services: - - couchdb + - docker + +before_install: + - docker pull couchdb:$COUCHDB_VERSION + - docker run -d -p 5984:5984 couchdb:$COUCHDB_VERSION install: "pip install -r requirements.txt && pip install -r test-requirements.txt" + +before_script: + # Make sure CouchDB is up + - while [ $? -ne 0 ]; do sleep 1 && curl -v http://localhost:5984; done + - curl -X PUT http://localhost:5984/_users + - curl -X PUT http://localhost:5984/_replicator + # command to run tests script: - pylint ./src/cloudant - - nosetests -w ./tests/unit + - nosetests -A 'not db or ((db == "couch" or "couch" in db) and (not couchapi or couchapi <='${COUCHDB_VERSION:0:1}'))' -w ./tests/unit notifications: email: false diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 00000000..75402ba5 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,304 @@ +# UNRELEASED +- [DEPRECATED] This library is end-of-life and no longer supported. + +# 2.15.0 (2021-08-26) +- [NEW] Override `dict.get` method for `CouchDatabase` to add `remote` parameter allowing it to + retrieve a remote document if specified. +- [FIXED] Fixed the documentation for `bookmarks`. +- [FIXED] Also exit `follow_replication` for `failed` state. +- [FIXED] Fixed result paging for grouped view queries. +- [FIXED] Incorrect use of username as account name in `Cloudant.bluemix()`. +- [FIXED] Use custom encoder (if provided) for all view `key` params not just `keys`. +- [FIXED] Support boolean type for `key`, `endkey`, and `startkey` in view requests. +- [DEPRECATED] This library is now deprecated and will be EOL on Dec 31 2021. +- [REMOVED] Removed Python 2 compatibility from the supported environments. +- [IMPROVED] Documented use of `None` account name and url override for `Cloudant.iam()`. +- [IMPROVED] - Document IDs and attachment names are now rejected if they could cause an unexpected + Cloudant request. We have seen that some applications pass unsantized document IDs to SDK functions + (e.g. direct from user requests). In response to this we have updated many functions to reject + obviously invalid paths. However, for complete safety applications must still validate that + document IDs and attachment names match expected patterns. + +# 2.14.0 (2020-08-17) + +- [FIXED] Set default value for `partitioned` parameter to false when creating a design document. +- [FIXED] Corrected setting of `partitioned` flag for `create_query_index` requests. +- [FIXED] Added a workaround for installation on Python 2. + +# 2.13.0 (2020-04-16) + +- [FIXED] Correctly raise exceptions from `create_database` calls. +- [FIXED] Fix `DeprecationWarning` from `collections`. + +# 2.12.0 (2019-03-28) + +- [NEW] Added partitioned database support. +- [FIXED] Bug where document context manager performed remote save despite uncaught exceptions being + raised inside `with` block. +- [FIXED] Fixed parameter type of `selector` in docstring. +- [FIXED] Removed internal `Document._document_id` property to allow a safe use of dict's methods. +- [IMPROVED] Performance of `Result` iteration by releasing result objects immediately after they + are returned to the client. +- [IMPROVED] Updated `Getting started` section with a `get_query_result` example. +- [IMPROVED] Updated `Result` iteration by paginating with views' `startkey` and queries' + `bookmark`. + +# 2.11.0 (2019-01-21) + +- [NEW] Added option for client to authenticate with IAM token server. +- [FIXED] Updated the default IAM token server URL. + +# 2.10.2 (2018-12-19) + +- [FIXED] A performance regression deserializing JSON in version 2.10.1. + +# 2.10.1 (2018-11-16) + +- [FIXED] Unexpected keyword argument errors when using the library with the `simplejson` module + present in the environment caused by `requests` preferentially loading it over the system `json` + module. + +# 2.10.0 (2018-09-19) + +- [NEW] Add custom JSON encoder/decoder option to `Document` constructor. +- [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. +- [NEW] Allow arbitrary query parameters to be passed to custom changes filters. +- [FIXED] Case where an exception was raised after successful retry when using `doc.update_field`. +- [FIXED] Removed unnecessary request when retrieving a Result collection that is less than the + `page_size` value. + +# 2.9.0 (2018-06-13) + +- [NEW] Added functionality to test if a key is in a database as in `key in db`, overriding dict + `__contains__` and checking in the remote database. +- [NEW] Moved `create_query_index` and other query related methods to `CouchDatabase` as the + `_index`/`_find` API is available in CouchDB 2.x. +- [NEW] Support IAM authentication in replication documents. +- [FIXED] Case where `Document` context manager would throw instead of creating a new document if no + `_id` was provided. +- [IMPROVED] Added support for IAM API key in `cloudant_bluemix` method. +- [IMPROVED] Shortened length of client URLs by removing username and password. +- [IMPROVED] Verified library operation on Python 3.6.3. + +# 2.8.1 (2018-02-16) + +- [FIXED] Installation failures of 2.8.0 caused by missing VERSION file in distribution. + +# 2.8.0 (2018-02-15) + +- [NEW] Added support for `/_search_disk_size` endpoint which retrieves disk size information for a + specific search index. +- [FIXED] Updated default IBM Cloud Identity and Access Management token URL. +- [REMOVED] Removed broken source and target parameters that constantly threw `AttributeError` when + creating a replication document. + +# 2.7.0 (2017-10-31) + +- [NEW] Added API for upcoming Bluemix Identity and Access Management support for Cloudant on + Bluemix. Note: IAM API key support is not yet enabled in the service. +- [NEW] Added HTTP basic authentication support. +- [NEW] Added `Result.all()` convenience method. +- [NEW] Allow `service_name` to be specified when instantiating from a Bluemix VCAP_SERVICES + environment variable. +- [IMPROVED] Updated `posixpath.join` references to use `'/'.join` when concatenating URL parts. +- [IMPROVED] Updated documentation by replacing deprecated Cloudant links with the latest Bluemix + links. + +# 2.6.0 (2017-08-10) + +- [NEW] Added `Cloudant.bluemix()` class method to the Cloudant client allowing service credentials + to be passed using the CloudFoundry VCAP_SERVICES environment variable. +- [FIXED] Fixed client construction in `cloudant_bluemix` context manager. +- [FIXED] Fixed validation for feed options to accept zero as a valid value. + +# 2.5.0 (2017-07-06) + +- [FIXED] Fixed crash caused by non-UTF8 chars in design documents. +- [FIXED] Fixed `TypeError` when setting revision limits on Python>=3.6. +- [FIXED] Fixed the `exists()` double check on `client.py` and `database.py`. +- [FIXED] Fixed Cloudant exception code 409 with 412 when creating a database that already exists. +- [FIXED] Catch error if `throw_on_exists` flag is `False` for creating a document. +- [FIXED] Fixed /_all_docs call where `keys` is an empty list. +- [FIXED] Issue where docs with IDs that sorted lower than 0 were not returned when iterating + through _all_docs. + +# 2.4.0 (2017-02-14) + +- [NEW] Added `timeout` option to the client constructor for setting a timeout on a HTTP connection + or a response. +- [NEW] Added `cloudant_bluemix` method to the Cloudant client allowing service credentials to be + passed using the CloudFoundry VCAP_SERVICES environment variable. +- [IMPROVED] Updated non-response related errors with additional status code and improved error + message for easier debugging. All non-response error are handled using either CloudantException + or CloudantArgumentError. +- [FIXED] Support `long` type argument when executing in Python 2. + +# 2.3.1 (2016-11-30) + +- [FIXED] Resolved issue where generated UUIDs for replication documents would not be converted to + strings. +- [FIXED] Resolved issue where CouchDatabase.infinite_changes() method can cause a stack overflow. + +# 2.3.0 (2016-11-02) + +- [FIXED] Resolved issue where the custom JSON encoder was at times not used when transforming data. +- [NEW] Added support for managing the database security document through the SecurityDocument class + and CouchDatabase convenience method `get_security_document`. +- [NEW] Added `auto_renewal` option to the client constructor to handle the automatic renewal of an + expired session cookie auth. + +# 2.2.0 (2016-10-20) + +- [NEW] Added auto connect feature to the client constructor. +- [FIXED] Requests session is no longer valid after disconnect. + +# 2.1.1 (2016-10-03) + +- [FIXED] HTTPError is now raised when 4xx or 5xx codes are encountered. + +# 2.1.0 (2016-08-31) + +- [NEW] Added support for Cloudant Search execution. +- [NEW] Added support for Cloudant Search index management. +- [NEW] Added support for managing and querying list functions. +- [NEW] Added support for managing and querying show functions. +- [NEW] Added support for querying update handlers. +- [NEW] Added `rewrites` accessor property for URL rewriting. +- [NEW] Added `st_indexes` accessor property for Cloudant Geospatial indexes. +- [NEW] Added support for DesignDocument `_info` and `_search_info` endpoints. +- [NEW] Added `validate_doc_update` accessor property for update validators. +- [NEW] Added support for a custom `requests.HTTPAdapter` to be configured using an optional + `adapter` arg e.g. `Cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME, + adapter=Replay429Adapter())`. +- [IMPROVED] Made the 429 response code backoff optional and configurable. To enable the backoff add + an `adapter` arg of a `Replay429Adapter` with the desired number of retries and initial + backoff. To replicate the 2.0.0 behaviour use: `adapter=Replay429Adapter(retries=10, + initialBackoff=0.25)`. If `retries` or `initialBackoff` are not specified they will default to 3 + retries and a 0.25 s initial backoff. +- [IMPROVED] Additional error reason details appended to HTTP response message errors. +- [FIX] `415 Client Error: Unsupported Media Type` when using keys with `db.all_docs`. +- [FIX] Allowed strings as well as lists for search `group_sort` arguments. + +# 2.0.3 (2016-06-03) + +- [FIX] Fixed the python-cloudant readthedocs documentation home page to resolve correctly. + +# 2.0.2 (2016-06-02) + +- [IMPROVED] Updated documentation links from python-cloudant.readthedocs.org to + python-cloudant.readthedocs.io. +- [FIX] Fixed issue with Windows platform compatibility,replaced usage of os.uname for the + user-agent string. +- [FIX] Fixed readthedocs link in README.rst to resolve to documentation home page. + +# 2.0.1 (2016-06-02) + +- [IMPROVED] Updated documentation links from python-cloudant.readthedocs.org to + python-cloudant.readthedocs.io. +- [FIX] Fixed issue with Windows platform compatibility,replaced usage of os.uname for the + user-agent string. +- [FIX] Fixed readthedocs link in README.rst to resolve to documentation home page. + +# 2.0.0 (2016-05-02) + +- [BREAKING] Renamed modules account.py, errors.py, indexes.py, views.py, to client.py, error.py, + index.py, and view.py. +- [BREAKING] Removed the `make_result` method from `View` and `Query` classes. If you need to make + a query or view result, use `CloudantDatabase.get_query_result`, `CouchDatabase.get_view_result`, + or the `View.custom_result` context manager. Additionally, the `Result` and `QueryResult` classes + can be called directly to construct a result object. +- [BREAKING] Refactored the `SearchIndex` class to now be the `TextIndex` class. Also renamed the + `CloudantDatabase` convenience methods of `get_all_indexes`, `create_index`, and `delete_index` as + `get_query_indexes`, `create_query_index`, and `delete_query_index` respectively. These changes + were made to clarify that the changed class and the changed methods were specific to query index + processing only. +- [BREAKING] Replace "session" and "url" feed constructor arguments with "source" which can be + either a client or a database object. Changes also made to the client `db_updates` method + signature and the database `changes` method signature. +- [BREAKING] Fixed `CloudantDatabase.share_database` to accept all valid permission roles. Changed + the method signature to accept roles as a list argument. +- [BREAKING] Removed credentials module from the API and moved it to the tests folder since the + functionality is outside of the scope of this library but is still be useful in unit/integration + tests. +- [IMPROVED] Changed the handling of queries using the keys argument to issue a http POST request + instead of a http GET request so that the request is no longer bound by any URL length limitation. +- [IMPROVED] Added support for Result/QueryResult data access via index value and added validation + logic to `Result.__getitem__()`. +- [IMPROVED] Updated feed functionality to process `_changes` and `_db_updates` with their supported + options. Also added an infinite feed option. +- [NEW] Handled HTTP status code `429 Too Many Requests` with blocking backoff and retries. +- [NEW] Added support for CouchDB Admin Party mode. This library can now be used with CouchDB + instances where everyone is Admin. +- [FIX] Fixed `Document.get_attachment` method to successfully create text and binary files based on + http response Content-Type. The method also returns text, binary, and json content based on http + response Content-Type. +- [FIX] Added validation to `Cloudant.bill`, `Cloudant.volume_usage`, and `Cloudant.requests_usage` + methods to ensure that a valid year/month combination or neither are used as arguments. +- [FIX] Fixed the handling of empty views in the DesignDocument. +- [FIX] The `CouchDatabase.create_document` method now handles documents and design documents + correctly. If the document created is a design document then the locally cached object will be a + DesignDocument otherwise it will be a Document. +- [CHANGE] Moved internal `Code` class, functions like `python_to_couch` and `type_or_none`, and + constants into a _common_util module. +- [CHANGE] Updated User-Agent header format to be `python-cloudant//Python///`. +- [CHANGE] Completed the addition of unit tests that target a database server. Removed all mocked + unit tests. + +# 2.0.0b2 (2016-02-24) + +- [FIX] Remove the fields parameter from required Query parameters. +- [NEW] Add Python 3 support. + +# 2.0.0b1 (2016-01-11) + + +- [NEW] Added support for Cloudant Query execution. +- [NEW] Added support for Cloudant Query index management. +- [FIX] DesignDocument content is no longer limited to just views. +- [FIX] Document url encoding is now enforced. +- [FIX] Database iterator now yields Document/DesignDocument objects with valid document urls. + +# 2.0.0a4 (2015-12-03) + + +- [FIX] Fixed incorrect readme reference to current library being Alpha 2. + +# 2.0.0a3 (2015-12-03) + + +- [NEW] Added API documentation hosted on readthedocs.org. + +# 2.0.0a2 (2015-11-19) + + +- [NEW] Added unit tests targeting CouchDB and Cloudant databases. +- [FIX] Fixed bug in database create validation check to work if response code is either 201 + (created) or 202 (accepted). +- [FIX] Fixed database iterator infinite loop problem and to now yield a Document object. +- [BREAKING] Removed previous bulk_docs method from the CouchDatabase class and renamed the previous + bulk_insert method as bulk_docs. The previous bulk_docs functionality is available through the + all_docs method using the "keys" parameter. +- [FIX] Made missing_revisions, revisions_diff, get_revision_limit, set_revision_limit, and + view_cleanup API methods available for CouchDB as well as Cloudant. +- [BREAKING] Moved the db_update method to the account module. +- [FIX] Fixed missing_revisions to key on 'missing_revs'. +- [FIX] Fixed set_revision_limit to encode the request data payload correctly. +- [FIX] `Document.create()` will no longer update an existing document. +- [BREAKING] Renamed Document `field_append` method to `list_field_append`. +- [BREAKING] Renamed Document `field_remove` method to `list_field_remove`. +- [BREAKING] Renamed Document `field_replace` method to `field_set`. +- [FIX] The Document local dictionary `_id` key is now synched with `_document_id` private + attribute. +- [FIX] The Document local dictionary is now refreshed after an add/update/delete of an attachment. +- [FIX] The Document `fetch()` method now refreshes the Document local dictionary content correctly. +- [BREAKING] Replace the ReplicatorDatabase class with the Replicator class. A Replicator object + has a database attribute that represents the _replicator database. This allows the Replicator to + work for both a CloudantDatabase and a CouchDatabase. +- [REMOVED] Removed "not implemented" methods from the DesignDocument. +- [FIX] Add implicit "_design/" prefix for DesignDocument document ids. + +# 2.0.0a1 (2015-10-13) + + +- Initial release (2.0.0a1). diff --git a/CHANGES.rst b/CHANGES.rst deleted file mode 100644 index 305ad920..00000000 --- a/CHANGES.rst +++ /dev/null @@ -1,142 +0,0 @@ -2.5.0 (Unreleased) -================== -- [FIXED] Fixed ``TypeError`` when setting revision limits on Python>=3.6. -- [FIXED] Fixed the ``exists()`` double check on ``client.py`` and ``database.py``. -- [FIXED] Fixed Cloudant exception code 409 with 412 when creating a database that already exists. - -2.4.0 (2017-02-14) -================== -- [NEW] Added ``timeout`` option to the client constructor for setting a timeout on a HTTP connection or a response. -- [NEW] Added ``cloudant_bluemix`` method to the Cloudant client allowing service credentials to be passed using the CloudFoundry VCAP_SERVICES environment variable. -- [IMPROVED] Updated non-response related errors with additional status code and improved error message for easier debugging. - All non-response error are handled using either CloudantException or CloudantArgumentError. -- [FIXED] Support ``long`` type argument when executing in Python 2. - -2.3.1 (2016-11-30) -================== -- [FIXED] Resolved issue where generated UUIDs for replication documents would not be converted to strings. -- [FIXED] Resolved issue where CouchDatabase.infinite_changes() method can cause a stack overflow. - -2.3.0 (2016-11-02) -================== -- [FIXED] Resolved issue where the custom JSON encoder was at times not used when transforming data. -- [NEW] Added support for managing the database security document through the SecurityDocument class and CouchDatabase convenience method ``get_security_document``. -- [NEW] Added ``auto_renewal`` option to the client constructor to handle the automatic renewal of an expired session cookie auth. - -2.2.0 (2016-10-20) -================== -- [NEW] Added auto connect feature to the client constructor.
 -- [FIXED] Requests session is no longer valid after disconnect. - -2.1.1 (2016-10-03) -================== -- [FIXED] HTTPError is now raised when 4xx or 5xx codes are encountered. - -2.1.0 (2016-08-31) -================== -- [NEW] Added support for Cloudant Search execution. -- [NEW] Added support for Cloudant Search index management. -- [NEW] Added support for managing and querying list functions. -- [NEW] Added support for managing and querying show functions. -- [NEW] Added support for querying update handlers. -- [NEW] Added ``rewrites`` accessor property for URL rewriting. -- [NEW] Added ``st_indexes`` accessor property for Cloudant Geospatial indexes. -- [NEW] Added support for DesignDocument ``_info`` and ``_search_info`` endpoints. -- [NEW] Added ``validate_doc_update`` accessor property for update validators. -- [NEW] Added support for a custom ``requests.HTTPAdapter`` to be configured using an optional ``adapter`` arg e.g. - ``Cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME, adapter=Replay429Adapter())``. -- [IMPROVED] Made the 429 response code backoff optional and configurable. To enable the backoff add - an ``adapter`` arg of a ``Replay429Adapter`` with the desired number of retries and initial backoff. To replicate - the 2.0.0 behaviour use: ``adapter=Replay429Adapter(retries=10, initialBackoff=0.25)``. If ``retries`` or - ``initialBackoff`` are not specified they will default to 3 retries and a 0.25 s initial backoff. -- [IMPROVED] Additional error reason details appended to HTTP response message errors. -- [FIX] ``415 Client Error: Unsupported Media Type`` when using keys with ``db.all_docs``. -- [FIX] Allowed strings as well as lists for search ``group_sort`` arguments. - -2.0.3 (2016-06-03) -================== -- [FIX] Fixed the python-cloudant readthedocs documentation home page to resolve correctly. - -2.0.2 (2016-06-02) -================== -- [IMPROVED] Updated documentation links from python-cloudant.readthedocs.org to python-cloudant.readthedocs.io. -- [FIX] Fixed issue with Windows platform compatibility,replaced usage of os.uname for the user-agent string. -- [FIX] Fixed readthedocs link in README.rst to resolve to documentation home page. - -2.0.1 (2016-06-02) -================== -- [IMPROVED] Updated documentation links from python-cloudant.readthedocs.org to python-cloudant.readthedocs.io. -- [FIX] Fixed issue with Windows platform compatibility,replaced usage of os.uname for the user-agent string. -- [FIX] Fixed readthedocs link in README.rst to resolve to documentation home page. - -2.0.0 (2016-05-02) -================== -- [BREAKING] Renamed modules account.py, errors.py, indexes.py, views.py, to client.py, error.py, index.py, and view.py. -- [BREAKING] Removed the ``make_result`` method from ``View`` and ``Query`` classes. If you need to make a query or view result, use ``CloudantDatabase.get_query_result``, ``CouchDatabase.get_view_result``, or the ``View.custom_result`` context manager. Additionally, the ``Result`` and ``QueryResult`` classes can be called directly to construct a result object. -- [BREAKING] Refactored the ``SearchIndex`` class to now be the ``TextIndex`` class. Also renamed the ``CloudantDatabase`` convenience methods of ``get_all_indexes``, ``create_index``, and ``delete_index`` as ``get_query_indexes``, ``create_query_index``, and ``delete_query_index`` respectively. These changes were made to clarify that the changed class and the changed methods were specific to query index processing only. -- [BREAKING] Replace "session" and "url" feed constructor arguments with "source" which can be either a client or a database object. Changes also made to the client ``db_updates`` method signature and the database ``changes`` method signature. -- [BREAKING] Fixed ``CloudantDatabase.share_database`` to accept all valid permission roles. Changed the method signature to accept roles as a list argument. -- [BREAKING] Removed credentials module from the API and moved it to the tests folder since the functionality is outside of the scope of this library but is still be useful in unit/integration tests. -- [IMPROVED] Changed the handling of queries using the keys argument to issue a http POST request instead of a http GET request so that the request is no longer bound by any URL length limitation. -- [IMPROVED] Added support for Result/QueryResult data access via index value and added validation logic to ``Result.__getitem__()``. -- [IMPROVED] Updated feed functionality to process ``_changes`` and ``_db_updates`` with their supported options. Also added an infinite feed option. -- [NEW] Handled HTTP status code ``429 Too Many Requests`` with blocking backoff and retries. -- [NEW] Added support for CouchDB Admin Party mode. This library can now be used with CouchDB instances where everyone is Admin. -- [FIX] Fixed ``Document.get_attachment`` method to successfully create text and binary files based on http response Content-Type. The method also returns text, binary, and json content based on http response Content-Type. -- [FIX] Added validation to ``Cloudant.bill``, ``Cloudant.volume_usage``, and ``Cloudant.requests_usage`` methods to ensure that a valid year/month combination or neither are used as arguments. -- [FIX] Fixed the handling of empty views in the DesignDocument. -- [FIX] The ``CouchDatabase.create_document`` method now handles documents and design documents correctly. If the document created is a design document then the locally cached object will be a DesignDocument otherwise it will be a Document. -- [CHANGE] Moved internal ``Code`` class, functions like ``python_to_couch`` and ``type_or_none``, and constants into a _common_util module. -- [CHANGE] Updated User-Agent header format to be ``python-cloudant//Python///``. -- [CHANGE] Completed the addition of unit tests that target a database server. Removed all mocked unit tests. - -2.0.0b2 (2016-02-24) -==================== -- [FIX] Remove the fields parameter from required Query parameters. -- [NEW] Add Python 3 support. - -2.0.0b1 (2016-01-11) -==================== - -- [NEW] Added support for Cloudant Query execution. -- [NEW] Added support for Cloudant Query index management. -- [FIX] DesignDocument content is no longer limited to just views. -- [FIX] Document url encoding is now enforced. -- [FIX] Database iterator now yields Document/DesignDocument objects with valid document urls. - -2.0.0a4 (2015-12-03) -==================== - -- [FIX] Fixed incorrect readme reference to current library being Alpha 2. - -2.0.0a3 (2015-12-03) -==================== - -- [NEW] Added API documentation hosted on readthedocs.org. - -2.0.0a2 (2015-11-19) -==================== - -- [NEW] Added unit tests targeting CouchDB and Cloudant databases. -- [FIX] Fixed bug in database create validation check to work if response code is either 201 (created) or 202 (accepted). -- [FIX] Fixed database iterator infinite loop problem and to now yield a Document object. -- [BREAKING] Removed previous bulk_docs method from the CouchDatabase class and renamed the previous bulk_insert method as bulk_docs. The previous bulk_docs functionality is available through the all_docs method using the "keys" parameter. -- [FIX] Made missing_revisions, revisions_diff, get_revision_limit, set_revision_limit, and view_cleanup API methods available for CouchDB as well as Cloudant. -- [BREAKING] Moved the db_update method to the account module. -- [FIX] Fixed missing_revisions to key on 'missing_revs'. -- [FIX] Fixed set_revision_limit to encode the request data payload correctly. -- [FIX] ``Document.create()`` will no longer update an existing document. -- [BREAKING] Renamed Document ``field_append`` method to ``list_field_append``. -- [BREAKING] Renamed Document ``field_remove`` method to ``list_field_remove``. -- [BREAKING] Renamed Document ``field_replace`` method to ``field_set``. -- [FIX] The Document local dictionary ``_id`` key is now synched with ``_document_id`` private attribute. -- [FIX] The Document local dictionary is now refreshed after an add/update/delete of an attachment. -- [FIX] The Document ``fetch()`` method now refreshes the Document local dictionary content correctly. -- [BREAKING] Replace the ReplicatorDatabase class with the Replicator class. A Replicator object has a database attribute that represents the _replicator database. This allows the Replicator to work for both a CloudantDatabase and a CouchDatabase. -- [REMOVED] Removed "not implemented" methods from the DesignDocument. -- [FIX] Add implicit "_design/" prefix for DesignDocument document ids. - -2.0.0a1 (2015-10-13) -==================== - -- Initial release (2.0.0a1). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..7f9e5300 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,93 @@ +# Contributing + +## Issues + +Please [read these guidelines](http://ibm.biz/cdt-issue-guide) before opening an issue. +If you still need to open an issue then we ask that you complete the template as +fully as possible. + +## Pull requests + +We welcome pull requests, but ask contributors to keep in mind the following: + +* Only PRs with the template completed will be accepted +* We will not accept PRs for user specific functionality + +### Developer Certificate of Origin + +In order for us to accept pull-requests, the contributor must sign-off a +[Developer Certificate of Origin (DCO)](DCO1.1.txt). This clarifies the +intellectual property license granted with any contribution. It is for your +protection as a Contributor as well as the protection of IBM and its customers; +it does not change your rights to use your own Contributions for any other purpose. + +Please read the agreement and acknowledge it by ticking the appropriate box in the PR + text, for example: + +- [x] Tick to sign-off your agreement to the Developer Certificate of Origin (DCO) 1.1 + +## General information + +Python-Cloudant Client Library is written in Python. + +## Requirements + +- Python +- pip + +It is recommended to use a [virtual environment](https://virtualenv.pypa.io/en/latest) during development. The +python-cloudant dependencies can be installed via the `requirements.txt` file using pip. + +For example to create a virtualenv and install requirements: + +```sh +virtualenv . +./bin/activate +pip install -r requirements.txt +pip install -r test-requirements.txt +``` + +## Testing + +The tests need an Apache CouchDB or Cloudant service to run against. + +The tests create databases in your CouchDB instance, these are `db-`. +They also create and delete documents in the `_replicator` database. + +The tests are run with the `nosetests` runner. In this example the `ADMIN_PARTY` environment variable is used to tell + the tests not to use any authentication. See below for the full set of variables that can be used. + +```sh +$ ADMIN_PARTY=true nosetests -w ./tests/unit +``` + +There are several environment variables which affect +test behaviour: + +- `RUN_CLOUDANT_TESTS`: set this to run the tests that use Cloudant-specific features. If + you set this, you must set one of the following combinations of other variables: + - `DB_URL`, `DB_USER` and `DB_PASSWORD`. + - `CLOUDANT_ACCOUNT`, `DB_USER` and `DB_PASSWORD`. + - If you set both `DB_URL` and `CLOUDANT_ACCOUNT`, `DB_URL` is used as the + URL to make requests to and `CLOUDANT_ACCOUNT` is inserted into the `X-Cloudant-User` + header. +- Without `RUN_CLOUDANT_TESTS`, the following environment variables have an effect: + - Set `DB_URL` to set the root URL of the CouchDB/Cloudant instance. It defaults + to `http://localhost:5984`. + - Set `ADMIN_PARTY` to `true` to not use any authentication details. + - Without `ADMIN_PARTY`, set `DB_USER` and `DB_PASSWORD` to use those + credentials to access the database. + - Without `ADMIN_PARTY` and `DB_USER`, the tests assume CouchDB is in + admin party mode, but create a user via `_config` to run tests as. + This user is deleted at the end of the test run, but beware it'll + break other applications using the CouchDB instance that rely on + admin party mode being in effect while the tests are running. + +### Test attributes + +Database tests also have node attributes. Currently there are these attributes: +`db` - `cloudant` and/or `couch` +`couchapi` - Apache CouchDB major version number (i.e. API level) e.g. `2` + +Example to run database tests that require CouchDB version 1 API and no Cloudant features: +`nosetests -A 'db and ((db is "couch" or "couch" in db) and (not couchapi or couchapi <=1))' -w ./tests/unit` diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst deleted file mode 100644 index 7c0106b5..00000000 --- a/CONTRIBUTING.rst +++ /dev/null @@ -1,80 +0,0 @@ -Developing this library -======================= - -Python-Cloudant Client Library is written in Python. - -============================= -Contributor License Agreement -============================= - -In order for us to accept pull-requests, the contributor must first complete -a Contributor License Agreement (CLA). This clarifies the intellectual -property license granted with any contribution. It is for your protection as a -Contributor as well as the protection of IBM and its customers; it does not -change your rights to use your own Contributions for any other purpose. - -This is a quick process: one option is signing using Preview on a Mac, -then sending a copy to us via email. - -You can download the CLAs here: - -- `Individual `_ -- `Corporate `_ - -If you are an IBMer, please contact us directly as the contribution process is -slightly different. - -====================== -Development Quickstart -====================== - -Clone the repo into a folder, set up a `virtual environment `_, -install the requirements: - -.. code-block:: bash - - $ git clone git clone git@github.com:cloudant/python-cloudant.git - $ cd python-cloudant - $ virtualenv . - $ ./bin/activate - $ pip install -r requirements.txt - $ pip install -r test-requirements.txt - -Before running the tests, start CouchDB: - -.. code-block:: bash - - $ couchdb - -The tests create databases in your CouchDB instance, these are `db-`. -They also create and delete documents in the `_replicator` database. - -Now, run the tests. Here, I use the ``ADMIN_PARTY`` environment variable to -tell the tests not to use any authentication. See below for the full set of -variables that can be used. - -.. code-block:: bash - - $ ADMIN_PARTY=true nosetests -w ./tests/unit - -There are several environment variables which affect -test behaviour: - -- ``RUN_CLOUDANT_TESTS``: set this to run the tests that use Cloudant-specific features. If - you set this, you must set one of the following combinations of other variables: - - ``DB_URL``, ``DB_USER`` and ``DB_PASSWORD``. - - ``CLOUDANT_ACCOUNT``, ``DB_USER`` and ``DB_PASSWORD``. - - If you set both ``DB_URL`` and ``CLOUDANT_ACCOUNT``, ``DB_URL`` is used as the - URL to make requests to and ``CLOUDANT_ACCOUNT`` is inserted into the ``X-Cloudant-User`` - header. -- Without ``RUN_CLOUDANT_TESTS``, the following environment variables have an effect: - - Set ``DB_URL`` to set the root URL of the CouchDB/Cloudant instance. It defaults - to ``http://localhost:5984``. - - Set ``ADMIN_PARTY`` to ``true`` to not use any authentication details. - - Without ``ADMIN_PARTY``, set ``DB_USER`` and ``DB_PASSWORD`` to use those - credentials to access the database. - - Without ``ADMIN_PARTY`` and ``DB_USER``, the tests assume CouchDB is in - admin party mode, but create a user via ``_config`` to run tests as. - This user is deleted at the end of the test run, but beware it'll - break other applications using the CouchDB instance that rely on - admin party mode being in effect while the tests are running. diff --git a/DCO1.1.txt b/DCO1.1.txt new file mode 100644 index 00000000..d5646050 --- /dev/null +++ b/DCO1.1.txt @@ -0,0 +1,37 @@ +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +1 Letterman Drive +Suite D4700 +San Francisco, CA, 94129 + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + + By making a contribution to this project, I certify that: + + (a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + + (b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + + (c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + + (d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. diff --git a/Jenkinsfile b/Jenkinsfile index b89cb5c7..cc2d0ad4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,26 +1,52 @@ -// Define the test routine for different python versions -def test_python(pythonVersion) -{ - node { +def getEnvForSuite(suiteName) { + // Base environment variables + def envVars = [ + "DB_URL=${SDKS_TEST_SERVER_URL}", + "RUN_CLOUDANT_TESTS=1", + "SKIP_DB_UPDATES=1" // Disable pending resolution of case 71610 + ] + // Add test suite specific environment variables + switch(suiteName) { + case 'basic': + envVars.add("RUN_BASIC_AUTH_TESTS=1") + break + case 'iam': + // Setting IAM_API_KEY forces tests to run using an IAM enabled client. + envVars.add("IAM_API_KEY=$DB_IAM_API_KEY") + envVars.add("IAM_TOKEN_URL=$SDKS_TEST_IAM_URL") + break + case 'cookie': + case 'simplejson': + break + default: + error("Unknown test suite environment ${suiteName}") + } + return envVars +} + +def setupPythonAndTest(pythonVersion, testSuite) { + node('sdks-executor') { // Unstash the source on this node unstash name: 'source' // Set up the environment and test - withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'clientlibs-test', usernameVariable: 'DB_USER', passwordVariable: 'DB_PASSWORD']]) { - try { - sh """ virtualenv tmp -p /usr/local/lib/python${pythonVersion}/bin/${pythonVersion.startsWith('3') ? "python3" : "python"} - . ./tmp/bin/activate - echo \$DB_USER - export RUN_CLOUDANT_TESTS=1 - export CLOUDANT_ACCOUNT=\$DB_USER - # Temporarily disable the _db_updates tests pending resolution of case 71610 - export SKIP_DB_UPDATES=1 - pip install -r requirements.txt - pip install -r test-requirements.txt - pylint ./src/cloudant - nosetests -w ./tests/unit --with-xunit""" - } finally { - // Load the test results - junit 'nosetests.xml' + withCredentials([usernamePassword(credentialsId: 'testServerLegacy', usernameVariable: 'DB_USER', passwordVariable: 'DB_PASSWORD'), + string(credentialsId: 'testServerIamApiKey', variable: 'DB_IAM_API_KEY')]) { + withEnv(getEnvForSuite("${testSuite}")) { + try { + sh """ + virtualenv tmp -p ${pythonVersion.startsWith('3') ? "python3" : "python"} + . ./tmp/bin/activate + python --version + pip install -r requirements.txt + pip install -r test-requirements.txt + ${'simplejson'.equals(testSuite) ? 'pip install simplejson' : ''} + pylint ./src/cloudant + nosetests -A 'not db or (db == "cloudant" or "cloudant" in db)' -w ./tests/unit --with-xunit + """ + } finally { + // Load the test results + junit 'nosetests.xml' + } } } } @@ -34,10 +60,22 @@ stage('Checkout'){ stash name: 'source' } } + stage('Test'){ - // Run tests in parallel for multiple python versions - parallel( - Python2: {test_python('2.7.12')}, - Python3: {test_python('3.5.2')} - ) + def py3 = '3' + def axes = [:] + [py3].each { version -> + ['basic','cookie','iam'].each { auth -> + axes.put("Python${version}-${auth}", {setupPythonAndTest(version, auth)}) + } + } + axes.put("Python${py3}-simplejson", {setupPythonAndTest(py3, 'simplejson')}) + parallel(axes) +} + +stage('Publish') { + gitTagAndPublish { + isDraft=true + releaseApiUrl='https://api.github.com/repos/cloudant/python-cloudant/releases' + } } diff --git a/MANIFEST.in b/MANIFEST.in index f9bd1455..7c050d91 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include requirements.txt +include requirements.txt VERSION LICENSE diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 00000000..3323e1d8 --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,114 @@ +# Migrating to the `cloudant-python-sdk` library +This document is to assist in migrating from the `python-cloudant` (module: `cloudant`) to the newly supported [`cloudant-python-sdk`](https://github.com/IBM/cloudant-python-sdk) (module: `ibmcloudant`). + +## Initializing the client connection +There are several ways to create a client connection in `cloudant-python-sdk`: +1. [Environment variables](https://github.com/IBM/cloudant-python-sdk#authentication-with-environment-variables) +2. [External configuration file](https://github.com/IBM/cloudant-python-sdk#authentication-with-external-configuration) +3. [Programmatically](https://github.com/IBM/cloudant-python-sdk#programmatic-authentication) + +[See the README](https://github.com/IBM/cloudant-python-sdk#code-examples) for code examples on using environment variables. + +## Other differences +1. The `cloudant-python-sdk` library does not support local dictionary caching of database and document objects. +1. There are no context managers in `cloudant-python-sdk`. To reproduce the behaviour of the `python-cloudant` +context managers in `cloudant-python-sdk` users need to explicitly call the specific operations against the +remote HTTP API. For example, in the case of the document context manager, this would mean doing both a `get_document` +to fetch and a `put_document` to save. +1. In `cloudant-python-sdk` View, Search, and Query (aka `_find` endpoint) operation responses contain raw JSON +content like using `raw_result=True` in `python-cloudant`. +1. Replay adapters are replaced by the [automatic retries](https://github.com/IBM/ibm-cloud-sdk-common/#automatic-retries) feature for failed requests. +1. Error handling is not transferable from `python-cloudant` to `cloudant-python-sdk`. For more information go to the [Error handling section](https://cloud.ibm.com/apidocs/cloudant?code=python#error-handling) in our API docs. +1. Custom HTTP client configurations in `python-cloudant` can be set differently in + `cloudant-python-sdk`. For more information go to the + [Configuring the HTTP client section](https://github.com/IBM/ibm-cloud-sdk-common/#configuring-the-http-client) + in the IBM Cloud SDK Common README. + +### Troubleshooting +1. Authentication errors occur during service instantiation. For example, the code `service = + CloudantV1.new_instance(service_name="EXAMPLE")` will fail with `ValueError: At least one of + iam_profile_name or iam_profile_id must be specified.` if required environment variables + prefixed with `EXAMPLE` are not set. +1. Server errors occur when running a request against the service. We suggest to + check server errors with + [`getServerInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getserverinformation) + which is the new alternative of `metadata()`. + +## Request mapping +Here's a list of the top 5 most frequently used `python-cloudant` operations and the `cloudant-python-sdk` equivalent API operation documentation link: + +| `python-cloudant` operation | `cloudant-python-sdk` API operation documentation link | +|---------------------------------------|---------------------------------| +|`Document('db_name', 'docid').fetch()` |[`getDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdocument)| +|`db.get_view_result()` |[`postView`](https://cloud.ibm.com/apidocs/cloudant?code=python#postview)| +|`db.get_query_result()` |[`postFind`](https://cloud.ibm.com/apidocs/cloudant?code=python#postfind)| +| `doc.exists()` |[`headDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#headdocument)| +|`Document('db_name', 'docid').save()` |[`putDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#putdocument)| + + +[A table](#reference-table) with the whole list of operations is provided at the end of this guide. + +The `cloudant-python-sdk` library is generated from a more complete API spec and provides a significant number of operations that do not exist in `python-cloudant`. See [the IBM Cloud API Documentation](https://cloud.ibm.com/apidocs/cloudant) to review request parameter and body options, code examples, and additional details for every endpoint. + +## Known Issues +There's an [outline of known issues](https://github.com/IBM/cloudant-python-sdk/blob/master/KNOWN_ISSUES.md) in the `cloudant-python-sdk` repository. + +## Reference table +The table below contains a list of `python-cloudant` functions and the `cloudant-python-sdk` equivalent API operation documentation link. The `cloudant-python-sdk` operation documentation link will contain the new function in a code sample e.g. `getServerInformation` link will contain a code example with `get_server_information()`. + +**Note:** There are many API operations included in the new `cloudant-python-sdk` that are not available in the `python-cloudant` library. The [API documentation](https://cloud.ibm.com/apidocs/cloudant?code=python) contains the full list of operations. + +| `python-cloudant` function | `cloudant-python-sdk` API operation documentation link | +|-----------------|---------------------| +|`metadata()`|[`getServerInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getserverinformation)| +|`all_dbs()`|[`getAllDbs`](https://cloud.ibm.com/apidocs/cloudant?code=python#getalldbs)| +|`db_updates()/infinite_db_updates()`|[`getDbUpdates`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdbupdates)| +|`Replicator.stop_replication()`|[`deleteReplicationDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#deletereplicationdocument)| +|`Replicator.replication_state()`|[`getReplicationDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getreplicationdocument)| +|`Replicator.create_replication()`|[`putReplicationDocument`](https://cloud.ibm.com/apidocs/cloudant?code=#putreplicationdocument)| +|`Scheduler.get_doc()`|[`getSchedulerDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerdocument)| +|`Scheduler.list_docs()`|[`getSchedulerDocs`](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerdocs)| +|`Scheduler.list_jobs()`|[`getSchedulerJobs`](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerjobs)| +|`session()`|[`getSessionInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getsessioninformation)| +|`uuids()`|[`getUuids`](https://cloud.ibm.com/apidocs/cloudant?code=python#getuuids)| +|`db.delete()`|[`deleteDatabase`](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedatabase)| +|`db.metadata()`|[`getDatabaseInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdatabaseinformation)| +|`db.create_document()`|[`postDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#postdocument)| +|`db.create()`|[`putDatabase`](https://cloud.ibm.com/apidocs/cloudant?code=python#putdatabase)| +|`db.all_docs()/db.keys()`|[`postAllDocs`](https://cloud.ibm.com/apidocs/cloudant?code=python#postalldocs)| +|`db.bulk_docs()`|[`postBulkDocs`](https://cloud.ibm.com/apidocs/cloudant?code=python#postbulkdocs)| +|`db.changes()/db.infinite_changes()`|[`postChanges`](https://cloud.ibm.com/apidocs/cloudant?code=python#postchanges-databases)| +|`DesignDocument(db, '_design/doc').delete()`|[`deleteDesignDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedesigndocument)| +|`db.get_design_document()/DesignDocument(db, '_design/doc').fetch()`|[`getDesignDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdesigndocument)| +|`DesignDocument(db, '_design/doc').save()`|[`putDesignDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#putdesigndocument)| +|`DesignDocument(db, '_design/doc').info()`|[`getDesignDocumentInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdesigndocumentinformation)| +|`db.get_search_result()`|[`postSearch`](https://cloud.ibm.com/apidocs/cloudant?code=python#postsearch)| +|`db.get_view_result()`|[`postView`](https://cloud.ibm.com/apidocs/cloudant?code=python#postview)| +|`db.list_design_documents()`|[`postDesignDocs`](https://cloud.ibm.com/apidocs/cloudant?code=python#postdesigndocs)| +|`db.get_query_result()`|[`postFind`](https://cloud.ibm.com/apidocs/cloudant?code=python#postfind)| +|`db.get_query_indexes()`|[`getIndexesInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getindexesinformation)| +|`db.create_query_index()`|[`postIndex`](https://cloud.ibm.com/apidocs/cloudant?code=python#postindex)| +|`db.delete_query_index()`|[`deleteIndex`](https://cloud.ibm.com/apidocs/cloudant?code=python#deleteindex)| +|`Document(db, '_local/docid').fetch()`|[`getLocalDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getlocaldocument)| +|`Document(db, '_local/docid').save()`|[`putLocalDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#putlocaldocument)| +|`Document(db, '_local/docid').delete()`|[`deleteLocalDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#deletelocaldocument)| +|`db.missing_revisions()/db.revisions_diff()`|[`postRevsDiff`](https://cloud.ibm.com/apidocs/cloudant?code=python#postrevsdiff)| +|`db.partition_metadata()`|[`getPartitionInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getpartitioninformation)| +|`db.partitioned_all_docs()`|[`postPartitionAllDocs`](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionalldocs)| +|`db.get_partitioned_search_result()`|[`postPartitionSearch`](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionsearch)| +|`db.get_partitioned_view_result()`|[`postPartitionView`](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionview)| +|`db.get_partitioned_query_result()`|[`postPartitionFind`](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionfind-partitioned-databases)| +|`db.get_security_document()/db.security_document()`|[`getSecurity`](https://cloud.ibm.com/apidocs/cloudant?code=python#getsecurity)| +|`db.share_database()`|[`putSecurity`](https://cloud.ibm.com/apidocs/cloudant?code=python#putsecurity)| +|`db.shards()`|[`getShardsInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getshardsinformation)| +|`Document(db, 'docid').delete()`|[`deleteDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedocument)| +|`Document(db, 'docid').fetch()`|[`getDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdocument)| +|`Document(db, 'docid').exists()`|[`headDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#headdocument)| +|`Document(db, 'docid').save()`|[`putDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#putdocument)| +|`Document(db, 'docid').delete_attachment()`|[`deleteAttachment`](https://cloud.ibm.com/apidocs/cloudant?code=python#deleteattachment)| +|`Document(db, 'docid').get_attachment()`|[`getAttachment`](https://cloud.ibm.com/apidocs/cloudant?code=python#getattachment)| +|`Document(db, 'docid').put_attachment()`|[`putAttachment`](https://cloud.ibm.com/apidocs/cloudant?code=python#putattachment)| +|`generate_api_key()`|[`postApiKeys`](https://cloud.ibm.com/apidocs/cloudant?code=python#postapikeys)| +|`SecurityDocument().save()`|[`putCloudantSecurityConfiguration`](https://cloud.ibm.com/apidocs/cloudant?code=python#putcloudantsecurity)| +|`cors_configuration()/cors_origin()`|[`getCorsInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getcorsinformation)| +|`update_cors_configuration()`|[`putCorsConfiguration`](https://cloud.ibm.com/apidocs/cloudant?code=python#putcorsconfiguration)| diff --git a/README.md b/README.md new file mode 100644 index 00000000..467c9dd3 --- /dev/null +++ b/README.md @@ -0,0 +1,107 @@ +# :warning: NO LONGER MAINTAINED :warning: + +**This library is end-of-life and no longer supported.** + +This repository will not be updated. The repository will be kept available in read-only mode. + +Please see the [Migration Guide](./MIGRATION.md) for advice +about migrating to our replacement library +[cloudant-python-sdk](https://github.com/IBM/cloudant-python-sdk). + +For FAQs and additional information please refer to the +[Cloudant blog](https://blog.cloudant.com/2021/06/30/Cloudant-SDK-Transition.html). + +# Cloudant Python Client + +[![Build Status](https://travis-ci.org/cloudant/python-cloudant.svg?branch=master)](https://travis-ci.org/cloudant/python-cloudant) +[![Readthedocs](https://readthedocs.org/projects/pip/badge/)](http://python-cloudant.readthedocs.io) +[![Compatibility](https://img.shields.io/badge/python-3.5-blue.svg)](http://python-cloudant.readthedocs.io/en/latest/compatibility.html) +[![pypi](https://img.shields.io/pypi/v/cloudant.svg)](https://pypi.python.org/pypi/cloudant) + +This is the official Cloudant library for Python. + +* [Installation and Usage](#installation-and-usage) +* [Getting Started](#getting-started) +* [API Reference](http://python-cloudant.readthedocs.io/en/latest/cloudant.html) +* [Related Documentation](#related-documentation) +* [Development](#development) + * [Contributing](CONTRIBUTING.md) + * [Test Suite](CONTRIBUTING.md#running-the-tests) + * [Using in Other Projects](#using-in-other-projects) + * [License](#license) + * [Issues](#issues) +* [Migrating to `cloudant-python-sdk` library](#migrating-to-cloudant-python-sdk-library) + +## Installation and Usage + + +Released versions of this library are [hosted on PyPI](https://pypi.python.org/pypi/cloudant) and can be installed with `pip`. + +In order to install the latest version, execute + + pip install cloudant + +## Getting started + +See [Getting started (readthedocs.io)](http://python-cloudant.readthedocs.io/en/latest/getting_started.html) + +## API Reference + +See [API reference docs (readthedocs.io)](http://python-cloudant.readthedocs.io/en/latest/cloudant.html) + +## Related Documentation + +* [Cloudant Python client library docs (readthedocs.io)](http://python-cloudant.readthedocs.io) +* [Cloudant documentation](https://console.bluemix.net/docs/services/Cloudant/cloudant.html#overview) +* [Cloudant Learning Center](https://developer.ibm.com/clouddataservices/cloudant-learning-center/) +* [Tutorial for creating and populating a database on IBM Cloud](https://console.bluemix.net/docs/services/Cloudant/tutorials/create_database.html#creating-and-populating-a-simple-cloudant-nosql-db-database-on-ibm-cloud) + +## Development + +See [CONTRIBUTING.md](https://github.com/cloudant/python-cloudant/blob/master/CONTRIBUTING.md) + +## Using in other projects + +The preferred approach for using `python-cloudant` in other projects is to use the PyPI as described above. + +### Examples in open source projects + +[Getting Started with Python Flask on IBM Cloud](https://github.com/IBM-Cloud/get-started-python) + +[Movie Recommender Demo](https://github.com/snowch/movie-recommender-demo): +- [Update and check if documents exist](https://github.com/snowch/movie-recommender-demo/blob/master/web_app/app/dao.py#L162-L168) +- [Connect to Cloudant using 429 backoff with 10 retries](https://github.com/snowch/movie-recommender-demo/blob/master/web_app/app/cloudant_db.py#L17-L18) + +[Watson Recipe Bot](https://github.com/ibm-watson-data-lab/watson-recipe-bot-python-cloudant): +- [Use Cloudant Query to find design docs](https://github.com/ibm-watson-data-lab/watson-recipe-bot-python-cloudant/blob/master/souschef/cloudant_recipe_store.py#L33-L77) + +## License + +Copyright © 2015 IBM. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Issues + +Before opening a new issue please consider the following: +* Only the latest release is supported. If at all possible please try to reproduce the issue using +the latest version. +* Please check the [existing issues](https://github.com/cloudant/python-cloudant/issues) +to see if the problem has already been reported. Note that the default search +includes only open issues, but it may already have been closed. +* Cloudant customers should contact Cloudant support for urgent issues. +* When opening a new issue [here in github](../../issues) please complete the template fully. + +## Migrating to `cloudant-python-sdk` library +We have a newly supported Cloudant Python SDK named [cloudant-python-sdk](https://github.com/IBM/cloudant-python-sdk). +For advice on migrating from this module see [MIGRATION.md](MIGRATION.md). diff --git a/README.rst b/README.rst deleted file mode 100644 index 52736488..00000000 --- a/README.rst +++ /dev/null @@ -1,95 +0,0 @@ -Cloudant Python Client -====================== - -|build-status| |docs| |compatibility| - -.. |build-status| image:: https://travis-ci.org/cloudant/python-cloudant.png - :alt: build status - :scale: 100% - :target: https://travis-ci.org/cloudant/python-cloudant - -.. |docs| image:: https://readthedocs.org/projects/pip/badge/ - :alt: docs - :scale: 100% - :target: http://python-cloudant.readthedocs.io - -.. |compatibility| image:: https://img.shields.io/badge/python-2.7%2C%203.5-blue.svg - :alt: compatibility - :scale: 100% - :target: http://python-cloudant.readthedocs.io/en/latest/compatibility.html - -This is the official Cloudant library for Python. - -.. contents:: - :local: - :depth: 2 - :backlinks: none - -====================== -Installation and Usage -====================== - -Released versions of this library are `hosted on PyPI `_ -and can be installed with ``pip``. - -In order to install the latest version, execute - -.. code-block:: bash - - pip install cloudant - -=============== -Getting started -=============== - -See `Getting started (readthedocs.io) `_ - -============= -API Reference -============= - -See `API reference docs (readthedocs.io) `_ - -===================== -Related Documentation -===================== - -* `Cloudant Python client library docs (readthedocs.io) `_ -* `Cloudant documentation `_ -* `Cloudant for developers `_ - -=========== -Development -=========== - -See `CONTRIBUTING.rst `_ - -********** -Test Suite -********** - -Content coming soon... - -*********************** -Using in other projects -*********************** - -Content coming soon... - -******* -License -******* - -Copyright © 2015 IBM. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/VERSION b/VERSION new file mode 100644 index 00000000..3a7d90b0 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +2.15.1-SNAPSHOT diff --git a/docs/compatibility.rst b/docs/compatibility.rst index 1bb98cce..b756a65e 100644 --- a/docs/compatibility.rst +++ b/docs/compatibility.rst @@ -11,5 +11,4 @@ Note that some features are Cloudant specific. This library has been tested with the following versions of Python -* `Python™ 2.7 `_ * `Python™ 3.5 `_ diff --git a/docs/conf.py b/docs/conf.py index 6099c07d..c8d4ad26 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -21,6 +21,8 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath('../src')) + # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. @@ -58,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '2.5.0.dev' +version = '2.15.1-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.5.0.dev' +release = '2.15.1-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 815d0452..37c5b92c 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -1,6 +1,6 @@ -=============== +############### Getting started -=============== +############### Now it's time to begin doing some work with Cloudant and Python. For working code samples of any of the API's please go to our test suite. @@ -32,13 +32,13 @@ a HTTP connection or a response on all requests. A timeout can be set using the ``timeout`` argument when constructing a client. Connecting with a client -^^^^^^^^^^^^^^^^^^^^^^^^ +======================== .. code-block:: python # Use CouchDB to create a CouchDB client # from cloudant.client import CouchDB - # client = CouchDB(USERNAME, PASSWORD, url='http://127.0.0.1:5984') + # client = CouchDB(USERNAME, PASSWORD, url='http://127.0.0.1:5984', connect=True) # Use Cloudant to create a Cloudant client using account from cloudant.client import Cloudant @@ -56,8 +56,8 @@ Connecting with a client # Perform client tasks... session = client.session() - print 'Username: {0}'.format(session['userCtx']['name']) - print 'Databases: {0}'.format(client.all_dbs()) + print('Username: {0}'.format(session['userCtx']['name'])) + print('Databases: {0}'.format(client.all_dbs())) # Disconnect from the server client.disconnect() @@ -88,6 +88,37 @@ following statements hold true: connect=True, auto_renew=True) + +************************************ +Identity and Access Management (IAM) +************************************ + +IBM Cloud Identity & Access Management enables you to securely authenticate +users and control access to all cloud resources consistently in the IBM Bluemix +Cloud Platform. + +See `IBM Cloud Identity and Access Management `_ +for more information. + +The production IAM token service at *https://iam.cloud.ibm.com/identity/token* is used +by default. You can set an ``IAM_TOKEN_URL`` environment variable to override +this. + +You can easily connect to your Cloudant account using an IAM API key: + +.. code-block:: python + + # Authenticate using an IAM API key + client = Cloudant.iam(ACCOUNT_NAME, API_KEY, connect=True) + +If you need to authenticate to a server outside of the `cloudant.com` domain, you can use the `url` parameter: + +.. code-block:: python + + # Authenticate using an IAM API key to an account outside of the cloudant.com domain + client = Cloudant.iam(None, API_KEY, url='https://private.endpoint.example', connect=True) + + **************** Resource sharing **************** @@ -120,7 +151,7 @@ Note: Idle connections within the pool may be terminated by the server, so will indefinitely meaning that this will not completely remove the overhead of creating new connections. Using library in app server environment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +======================================= This library can be used in an app server, and the example below shows how to use ``client`` in a ``flask`` app server. @@ -166,7 +197,7 @@ existing database, or delete a database. The following examples assume a client connection has already been established. Creating a database -^^^^^^^^^^^^^^^^^^^ +=================== .. code-block:: python @@ -176,10 +207,10 @@ Creating a database # You can check that the database exists if my_database.exists(): - print 'SUCCESS!!' + print('SUCCESS!!') Opening a database -^^^^^^^^^^^^^^^^^^ +================== Opening an existing database is done by supplying the name of an existing database to the client. Since the ``Cloudant`` and ``CouchDB`` classes are @@ -192,13 +223,138 @@ sub-classes of ``dict``, this can be accomplished through standard Python my_database = client['my_database'] Deleting a database -^^^^^^^^^^^^^^^^^^^ +=================== .. code-block:: python # Delete a database using an initialized client client.delete_database('my_database') + +Partitioned Databases +===================== + +Partitioned databases introduce the ability for a user to create logical groups +of documents called partitions by providing a partition key with each document. + +.. warning:: Your Cloudant cluster must have the ``partitions`` feature enabled. + A full list of enabled features can be retrieved by calling the + client :func:`~cloudant.client.CouchDB.metadata` method. + +Creating a partitioned database +------------------------------- + +.. code-block:: python + + db = client.create_database('mydb', partitioned=True) + +Handling documents +------------------ + +The document ID contains both the partition key and document key in the form +``:`` where: + +- Partition Key *(string)*. Must be non-empty. Must not contain colons (as this + is the partition key delimiter) or begin with an underscore. +- Document Key *(string)*. Must be non-empty. Must not begin with an underscore. + +Be aware that ``_design`` documents and ``_local`` documents must not contain a +partition key as they are global definitions. + +**Create a document** + +.. code-block:: python + + partition_key = 'Year2' + document_key = 'julia30' + db.create_document({ + '_id': ':'.join((partition_key, document_key)), + 'name': 'Jules', + 'age': 6 + }) + +**Get a document** + +.. code-block:: python + + doc = db[':'.join((partition_key, document_key))] + +Creating design documents +------------------------- + +To define partitioned indexes you must set the ``partitioned=True`` optional +when constructing the new ``DesignDocument`` class. + +.. code-block:: python + + ddoc = DesignDocument(db, document_id='view', partitioned=True) + ddoc.add_view('myview','function(doc) { emit(doc.foo, doc.bar); }') + ddoc.save() + + +To define a partitioned Cloudant Query index you may set the +``partitioned=True`` optional, but it is not required as the index will be +partitioned by default in a partitioned database. Conversely, you must +set the ``partitioned=False`` optional if you wish to create a global +(non-partitioned) index in a partitioned database. + +.. code-block:: python + + index = db.create_query_index( + design_document_id='query', + index_name='foo-index', + fields=['foo'], + partitioned=True + ) + index.create() + +Querying data +------------- + +A partition key can be specified when querying data so that results can be +constrained to a specific database partition. + +.. warning:: To run partitioned queries the database itself must be partitioned. + +**Query** + +.. code-block:: python + + results = self.db.get_partitioned_query_result( + partition_key, selector={'foo': {'$eq': 'bar'}}) + + for result in results: + ... + +See :func:`~cloudant.database.CouchDatabase.get_partitioned_query_result` for a +full list of supported parameters. + +**Search** + +.. code-block:: python + + results = self.db.get_partitioned_search_result( + partition_key, search_ddoc['_id'], 'search1', query='*:*') + + for result in results['rows']: + .... + +See :func:`~cloudant.database.CloudantDatabase.get_partitioned_search_result` +for a full list of supported parameters. + +**Views (MapReduce)** + +.. code-block:: python + + results = self.db.get_partitioned_view_result( + partition_key, view_ddoc['_id'], 'view1') + + for result in results: + .... + +See :func:`~cloudant.database.CouchDatabase.get_partitioned_view_result` for a +full list of supported parameters. + ********* Documents ********* @@ -211,7 +367,7 @@ create, read, update, and delete a document. These examples assume that either a CloudantDatabase or a CouchDatabase object already exists. Creating a document -^^^^^^^^^^^^^^^^^^^ +=================== .. code-block:: python @@ -228,10 +384,10 @@ Creating a document # Check that the document exists in the database if my_document.exists(): - print 'SUCCESS!!' + print('SUCCESS!!') Retrieving a document -^^^^^^^^^^^^^^^^^^^^^ +===================== Accessing a document from a database is done by supplying the document identifier of an existing document to either a ``CloudantDatabase`` or a @@ -244,10 +400,23 @@ classes are sub-classes of ``dict``, this is accomplished through standard my_document = my_database['julia30'] # Display the document - print my_document + print(my_document) + +Checking if a document exists +============================= + +You can check if a document exists in a database the same way you would check +if a ``dict`` has a key-value pair by key. + +.. code-block:: python + + doc_exists = 'julia30' in my_database + + if doc_exists: + print('document with _id julia30 exists') Retrieve all documents -^^^^^^^^^^^^^^^^^^^^^^ +====================== You can also iterate over a ``CloudantDatabase`` or a ``CouchDatabase`` object to retrieve all documents in a database. @@ -256,10 +425,10 @@ to retrieve all documents in a database. # Get all of the documents from my_database for document in my_database: - print document + print(document) Update a document -^^^^^^^^^^^^^^^^^ +================= .. code-block:: python @@ -275,7 +444,7 @@ Update a document my_document.save() Delete a document -^^^^^^^^^^^^^^^^^ +================= .. code-block:: python @@ -333,7 +502,27 @@ object already exists. # Iterate over the result collection for result in result_collection: - print result + print(result) + +This example retrieves the query result from the specified database based on the query parameters provided, updates the +document, and saves the document in the remote database. +By default, the result is returned as a ``QueryResult`` which uses the skip and limit query parameters internally to +handle slicing and iteration through the query result collection. For more detail on slicing and iteration, refer +to the :class:`~cloudant.result.QueryResult` documentation. + +.. code-block:: python + + # Retrieve documents where the name field is 'foo' + selector = {'name': {'$eq': 'foo'}} + docs = my_database.get_query_result(selector) + for doc in docs: + # Create Document object from dict + updated_doc = Document(my_database, doc['_id']) + updated_doc.update(doc) + # Update document field + updated_doc['name'] = 'new_name' + # Save document + updated_doc.save() **************** Context managers @@ -342,57 +531,95 @@ Context managers Now that we've gone through the basics, let's take a look at how to simplify the process of connection, database acquisition, and document management through the use of Python *with* blocks and this library's context managers. -Handling your business using *with* blocks saves you from having to connect and + +Handling your business using *with* blocks saves you from having to connect and disconnect your client as well as saves you from having to perform a lot of fetch and save operations as the context managers handle these operations for -you. This example uses the ``cloudant`` context helper to illustrate the +you. + +This example uses the ``cloudant`` context helper to illustrate the process but identical functionality exists for CouchDB through the ``couchdb`` and ``couchdb_admin_party`` context helpers. .. code-block:: python - # cloudant context helper from cloudant import cloudant - # couchdb context helper + # ...or use CouchDB variant # from cloudant import couchdb - from cloudant.document import Document - # Perform a connect upon entry and a disconnect upon exit of the block with cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME) as client: - # CouchDB variant + # ...or use CouchDB variant # with couchdb(USERNAME, PASSWORD, url=COUCHDB_URL) as client: # Perform client tasks... session = client.session() - print 'Username: {0}'.format(session['userCtx']['name']) - print 'Databases: {0}'.format(client.all_dbs()) + print('Username: {0}'.format(session['userCtx']['name'])) + print('Databases: {0}'.format(client.all_dbs())) # Create a database my_database = client.create_database('my_database') if my_database.exists(): - print 'SUCCESS!!' + print('SUCCESS!!') # You can open an existing database del my_database my_database = client['my_database'] - - # Performs a fetch upon entry and a save upon exit of this block - # Use this context manager to create or update a Document - with Document(my_database, 'julia30') as doc: - doc['name'] = 'Julia' - doc['age'] = 30 - doc['pets'] = ['cat', 'dog', 'frog'] - - # Display a Document - print my_database['julia30'] - - # Delete the database - client.delete_database('my_database') - print 'Databases: {0}'.format(client.all_dbs()) +The following example uses the ``Document`` context manager. Here we make +multiple updates to a single document. Note that we don't save to the server +after each update. We only save once to the server upon exiting the ``Document`` +context manager. + +.. warning:: Uncaught exceptions inside the ``with`` block will prevent your + document changes being saved to the remote server. However, changes + will still be applied to your local document object. + +.. code-block:: python + + from cloudant import cloudant + from cloudant.document import Document + + with cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME) as client: + + my_database = client.create_database('my_database') + + # Upon entry into the document context, fetches the document from the + # remote database, if it exists. Upon exit from the context, saves the + # document to the remote database with changes made within the context + # or creates a new document. + with Document(database, 'julia006') as document: + # If document exists, it's fetched from the remote database + # Changes are made locally + document['name'] = 'Julia' + document['age'] = 6 + # The document is saved to the remote database + + # Display a Document + print(my_database['julia30']) + + # Delete the database + client.delete_database('my_database') + + print('Databases: {0}'.format(client.all_dbs())) + +Always use the ``_deleted`` document property to delete a document from within +a ``Document`` context manager. For example: + +.. code-block:: python + + with Document(my_database, 'julia30') as doc: + doc['_deleted'] = True + +*You can also delete non underscore prefixed document keys to reduce the size of the request.* + +.. warning:: Don't use the ``doc.delete()`` method inside your ``Document`` + context manager. This method immediately deletes the document on + the server and clears the local document dictionary. A new, empty + document is still saved to the server upon exiting the context + manager. **************** Endpoint access @@ -422,4 +649,16 @@ Cloudant/CouchDB server. This example assumes that either a ``Cloudant`` or a response = client.r_session.get(end_point, params=params) # Display the response content - print response.json() + print(response.json()) + +*************** +TLS 1.2 Support +*************** + +The TLS protocol is used to encrypt communications across a network to ensure +that transmitted data remains private. There are three released versions of TLS: +1.0, 1.1, and 1.2. All HTTPS connections use TLS. + +If your server enforces the use of TLS 1.2 then the python-cloudant client will +continue to work as expected (assuming you're running a version of +Python/OpenSSL that supports TLS 1.2). diff --git a/docs/index.rst b/docs/index.rst index 0f642eed..a26ea611 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,14 +1 @@ -python-cloudant documentation -============================= - -This is the official Cloudant client library for Python. - -.. toctree:: - :maxdepth: 4 - - compatibility - getting_started - cloudant - -* :ref:`genindex` - +This library is end-of-life and no longer supported. diff --git a/pylintrc b/pylintrc index 0e625d99..0b4f402d 100644 --- a/pylintrc +++ b/pylintrc @@ -66,7 +66,8 @@ confidence= # Disable "redefined-variable-type" refactor warning messages # Disable "too-many-..." and "too-few-..." refactor warning messages # Disable "locally-disabled" message -disable=R0204,R0901,R0902,R0903,R0904,R0913,R0914,R0915,locally-disabled +# Disable Python 3 "useless-object-inheritance" message +disable=R0204,R0901,R0902,R0903,R0904,R0913,R0914,R0915,locally-disabled,keyword-arg-before-vararg,useless-object-inheritance [REPORTS] diff --git a/setup.py b/setup.py index beeb376f..a8ac4ef8 100644 --- a/setup.py +++ b/setup.py @@ -19,17 +19,29 @@ """ +from io import open +from os import path from setuptools import setup, find_packages requirements_file = open('requirements.txt') requirements = requirements_file.read().strip().split('\n') +requirements_file.close() +version_file = open('VERSION') +version = version_file.read().strip() +version_file.close() + +this_directory = path.abspath(path.dirname(__file__)) +with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: + long_description = f.read() setup_args = { 'description': 'Cloudant / CouchDB Client Library', + 'long_description': long_description, + 'long_description_content_type': 'text/markdown', 'include_package_data': True, 'install_requires': requirements, 'name': 'cloudant', - 'version': '2.5.0.dev', + 'version': version, 'author': 'IBM', 'author_email': 'alfinkel@us.ibm.com', 'url': 'https://github.com/cloudant/python-cloudant', @@ -43,8 +55,6 @@ 'Topic :: Software Development :: Libraries', 'Development Status :: 5 - Production/Stable', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5' ] diff --git a/src/cloudant/_2to3.py b/src/cloudant/_2to3.py index 595c0fc3..5c7d412b 100644 --- a/src/cloudant/_2to3.py +++ b/src/cloudant/_2to3.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (c) 2016, 2017 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -32,11 +32,16 @@ # pylint: disable=undefined-variable LONGTYPE = long if PY2 else int +# pylint: disable=undefined-variable +UNICHR = unichr if PY2 else chr + if PY2: # pylint: disable=wrong-import-position,no-name-in-module,import-error,unused-import from urllib import quote as url_quote, quote_plus as url_quote_plus from urlparse import urlparse as url_parse + from urlparse import urljoin as url_join from ConfigParser import RawConfigParser + from cookielib import Cookie def iteritems_(adict): """ @@ -57,9 +62,11 @@ def next_(itr): return itr.next() else: from urllib.parse import urlparse as url_parse # pylint: disable=wrong-import-position,no-name-in-module,import-error,ungrouped-imports + from urllib.parse import urljoin as url_join # pylint: disable=wrong-import-position,no-name-in-module,import-error,ungrouped-imports from urllib.parse import quote as url_quote # pylint: disable=wrong-import-position,no-name-in-module,import-error,ungrouped-imports from urllib.parse import quote_plus as url_quote_plus # pylint: disable=wrong-import-position,no-name-in-module,import-error,ungrouped-imports from configparser import RawConfigParser # pylint: disable=wrong-import-position,no-name-in-module,import-error + from http.cookiejar import Cookie # pylint: disable=wrong-import-position,no-name-in-module,import-error def iteritems_(adict): """ diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 9400ab2a..8e4d7e33 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (c) 2015, 2018 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,14 +15,18 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.5.0.dev' +__version__ = '2.15.1-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib +import warnings # pylint: disable=wrong-import-position from .client import Cloudant, CouchDB from ._common_util import CloudFoundryService +warnings.warn('The module cloudant is now end-of-life. The replacement is ibmcloudant.', + DeprecationWarning) + @contextlib.contextmanager def cloudant(user, passwd, **kwargs): """ @@ -63,7 +67,36 @@ def cloudant(user, passwd, **kwargs): cloudant_session.disconnect() @contextlib.contextmanager -def cloudant_bluemix(vcap_services, instance_name=None, **kwargs): +def cloudant_iam(account_name, api_key, **kwargs): + """ + Provides a context manager to create a Cloudant session using IAM + authentication and provide access to databases, docs etc. + + :param account_name: Cloudant account name. + :param api_key: IAM authentication API key. + + For example: + + .. code-block:: python + + # cloudant context manager + from cloudant import cloudant_iam + + with cloudant_iam(ACCOUNT_NAME, API_KEY) as client: + # Context handles connect() and disconnect() for you. + # Perform library operations within this context. Such as: + print client.all_dbs() + # ... + + """ + cloudant_session = Cloudant.iam(account_name, api_key, **kwargs) + + cloudant_session.connect() + yield cloudant_session + cloudant_session.disconnect() + +@contextlib.contextmanager +def cloudant_bluemix(vcap_services, instance_name=None, service_name=None, **kwargs): """ Provides a context manager to create a Cloudant session and provide access to databases, docs etc. @@ -72,6 +105,7 @@ def cloudant_bluemix(vcap_services, instance_name=None, **kwargs): :type vcap_services: dict or str :param str instance_name: Optional Bluemix instance name. Only required if multiple Cloudant instances are available. + :param str service_name: Optional Bluemix service name. :param str encoder: Optional json Encoder object used to encode documents for storage. Defaults to json.JSONEncoder. @@ -86,6 +120,7 @@ def cloudant_bluemix(vcap_services, instance_name=None, **kwargs): "cloudantNoSQLDB": [ { "credentials": { + "apikey": "some123api456key" "username": "example", "password": "xxxxxxx", "host": "example.cloudant.com", @@ -120,11 +155,10 @@ def cloudant_bluemix(vcap_services, instance_name=None, **kwargs): print client.all_dbs() # ... """ - service = CloudFoundryService(vcap_services, instance_name) - cloudant_session = Cloudant( - username=service.username, - password=service.password, - url=service.url, + cloudant_session = Cloudant.bluemix( + vcap_services, + instance_name=instance_name, + service_name=service_name, **kwargs ) cloudant_session.connect() diff --git a/src/cloudant/_client_session.py b/src/cloudant/_client_session.py new file mode 100644 index 00000000..f6339ed2 --- /dev/null +++ b/src/cloudant/_client_session.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python +# Copyright (c) 2015, 2019 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Module containing client session classes. +""" +import base64 +import json +import os + +from requests import RequestException, Session + +from ._2to3 import bytes_, unicode_, url_join +from ._common_util import response_to_json_dict +from .error import CloudantException + + +class ClientSession(Session): + """ + This class extends Session and provides a default timeout. + """ + + def __init__(self, username=None, password=None, session_url=None, **kwargs): + super(ClientSession, self).__init__() + + self._username = username + self._password = password + self._session_url = session_url + + self._auto_renew = kwargs.get('auto_renew', False) + self._timeout = kwargs.get('timeout', None) + + def base64_user_pass(self): + """ + Composes a basic http auth string, suitable for use with the + _replicator database, and other places that need it. + + :returns: Basic http authentication string + """ + if self._username is None or self._password is None: + return None + + hash_ = base64.urlsafe_b64encode(bytes_("{username}:{password}".format( + username=self._username, + password=self._password + ))) + return "Basic {0}".format(unicode_(hash_)) + + # pylint: disable=arguments-differ + def request(self, method, url, **kwargs): + """ + Overrides ``requests.Session.request`` to set the timeout. + """ + resp = super(ClientSession, self).request( + method, url, timeout=self._timeout, **kwargs) + + return resp + + def info(self): + """ + Get session information. + """ + if self._session_url is None: + return None + + resp = self.get(self._session_url) + resp.raise_for_status() + return response_to_json_dict(resp) + + def set_credentials(self, username, password): + """ + Set a new username and password. + + :param str username: New username. + :param str password: New password. + """ + if username is not None: + self._username = username + + if password is not None: + self._password = password + + def login(self): + """ + No-op method - not implemented here. + """ + # pylint: disable=unnecessary-pass + pass + + def logout(self): + """ + No-op method - not implemented here. + """ + # pylint: disable=unnecessary-pass + pass + + +class BasicSession(ClientSession): + """ + This class extends ClientSession to provide basic access authentication. + """ + + def __init__(self, username, password, server_url, **kwargs): + super(BasicSession, self).__init__( + username=username, + password=password, + session_url=url_join(server_url, '_session'), + **kwargs) + + def request(self, method, url, **kwargs): + """ + Overrides ``requests.Session.request`` to provide basic access + authentication. + """ + auth = None + if self._username is not None and self._password is not None: + auth = (self._username, self._password) + + return super(BasicSession, self).request( + method, url, auth=auth, **kwargs) + + +class CookieSession(ClientSession): + """ + This class extends ClientSession and provides cookie authentication. + """ + + def __init__(self, username, password, server_url, **kwargs): + super(CookieSession, self).__init__( + username=username, + password=password, + session_url=url_join(server_url, '_session'), + **kwargs) + + def login(self): + """ + Perform cookie based user login. + """ + resp = super(CookieSession, self).request( + 'POST', + self._session_url, + data={'name': self._username, 'password': self._password}, + ) + resp.raise_for_status() + + def logout(self): + """ + Logout cookie based user. + """ + resp = super(CookieSession, self).request('DELETE', self._session_url) + resp.raise_for_status() + + def request(self, method, url, **kwargs): + """ + Overrides ``requests.Session.request`` to renew the cookie and then + retry the original request (if required). + """ + resp = super(CookieSession, self).request(method, url, **kwargs) + + if not self._auto_renew: + return resp + + is_expired = any(( + resp.status_code == 403 and + response_to_json_dict(resp).get('error') == 'credentials_expired', + resp.status_code == 401 + )) + + if is_expired: + self.login() + resp = super(CookieSession, self).request(method, url, **kwargs) + + return resp + + +class IAMSession(ClientSession): + """ + This class extends ClientSession and provides IAM authentication. + """ + + def __init__(self, api_key, server_url, client_id=None, client_secret=None, + **kwargs): + super(IAMSession, self).__init__( + session_url=url_join(server_url, '_iam_session'), + **kwargs) + + self._api_key = api_key + self._token_url = os.environ.get( + 'IAM_TOKEN_URL', 'https://iam.cloud.ibm.com/identity/token') + self._token_auth = None + if client_id and client_secret: + self._token_auth = (client_id, client_secret) + + @property + def get_api_key(self): + """ + Get IAM API key. + + :return: IAM API key. + """ + return self._api_key + + def login(self): + """ + Perform IAM cookie based user login. + """ + access_token = self._get_access_token() + try: + super(IAMSession, self).request( + 'POST', + self._session_url, + headers={'Content-Type': 'application/json'}, + data=json.dumps({'access_token': access_token}) + ).raise_for_status() + + except RequestException: + raise CloudantException( + 'Failed to exchange IAM token with Cloudant') + + def logout(self): + """ + Logout IAM cookie based user. + """ + self.cookies.clear() + + def request(self, method, url, **kwargs): + """ + Overrides ``requests.Session.request`` to renew the IAM cookie + and then retry the original request (if required). + """ + # The CookieJar API prevents callers from getting an individual Cookie + # object by name. + # We are forced to use the only exposed method of discarding expired + # cookies from the CookieJar. Internally this involves iterating over + # the entire CookieJar and calling `.is_expired()` on each Cookie + # object. + self.cookies.clear_expired_cookies() + + if self._auto_renew and 'IAMSession' not in self.cookies.keys(): + self.login() + + resp = super(IAMSession, self).request(method, url, **kwargs) + + if not self._auto_renew: + return resp + + if resp.status_code == 401: + self.login() + resp = super(IAMSession, self).request(method, url, **kwargs) + + return resp + + # pylint: disable=arguments-differ, unused-argument + def set_credentials(self, username, api_key): + """ + Set a new IAM API key. + + :param str username: Username parameter is unused. + :param str api_key: New IAM API key. + """ + if api_key is not None: + self._api_key = api_key + + def _get_access_token(self): + """ + Get IAM access token using API key. + """ + err = 'Failed to contact IAM token service' + try: + resp = super(IAMSession, self).request( + 'POST', + self._token_url, + auth=self._token_auth, + headers={'Accepts': 'application/json'}, + data={ + 'grant_type': 'urn:ibm:params:oauth:grant-type:apikey', + 'response_type': 'cloud_iam', + 'apikey': self._api_key + } + ) + err = response_to_json_dict(resp).get('errorMessage', err) + resp.raise_for_status() + + return response_to_json_dict(resp)['access_token'] + + except KeyError: + raise CloudantException('Invalid response from IAM token service') + + except RequestException: + raise CloudantException(err) diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 6acab04a..433e09a6 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2016, 2017 IBM Corp. All rights reserved. +# Copyright © 2015, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,15 +19,20 @@ import sys import platform -from collections import Sequence import json -from requests import Session -from ._2to3 import LONGTYPE, STRTYPE, NONETYPE, UNITYPE, iteritems_, url_parse -from .error import CloudantArgumentError, CloudantException +from ._2to3 import LONGTYPE, STRTYPE, NONETYPE, UNITYPE, iteritems_ +from .error import CloudantArgumentError, CloudantException, CloudantClientException + +try: + from collections.abc import Sequence +except ImportError: + from collections import Sequence # Library Constants +DESIGN_PREFIX = '_design/' +LOCAL_PREFIX = '_local/' USER_AGENT = '/'.join([ 'python-cloudant', sys.modules['cloudant'].__version__, @@ -48,22 +53,27 @@ # Argument Types +ANY_ARG = object() +ANY_TYPE = object() + RESULT_ARG_TYPES = { 'descending': (bool,), - 'endkey': (int, LONGTYPE, STRTYPE, Sequence,), + 'endkey': (int, LONGTYPE, STRTYPE, Sequence, bool,), 'endkey_docid': (STRTYPE,), 'group': (bool,), 'group_level': (int, LONGTYPE, NONETYPE,), 'include_docs': (bool,), 'inclusive_end': (bool,), - 'key': (int, LONGTYPE, STRTYPE, Sequence,), + 'key': (int, LONGTYPE, STRTYPE, Sequence, bool,), 'keys': (list,), 'limit': (int, LONGTYPE, NONETYPE,), 'reduce': (bool,), 'skip': (int, LONGTYPE, NONETYPE,), + 'stable': (bool,), 'stale': (STRTYPE,), - 'startkey': (int, LONGTYPE, STRTYPE, Sequence,), + 'startkey': (int, LONGTYPE, STRTYPE, Sequence, bool,), 'startkey_docid': (STRTYPE,), + 'update': (STRTYPE,), } # pylint: disable=unnecessary-lambda @@ -100,6 +110,7 @@ 'filter': (STRTYPE,), 'include_docs': (bool,), 'style': (STRTYPE,), + ANY_ARG: ANY_TYPE # pass arbitrary query parameters to a custom filter } _CHANGES_ARG_TYPES.update(_DB_UPDATES_ARG_TYPES) @@ -135,7 +146,8 @@ 'highlight_post_tag': STRTYPE, 'highlight_number': (int, LONGTYPE, NONETYPE), 'highlight_size': (int, LONGTYPE, NONETYPE), - 'include_fields': list + 'include_fields': list, + 'partition': STRTYPE } # Functions @@ -146,11 +158,11 @@ def feed_arg_types(feed_type): """ if feed_type == 'Cloudant': return _DB_UPDATES_ARG_TYPES - elif feed_type == 'CouchDB': + if feed_type == 'CouchDB': return _COUCH_DB_UPDATES_ARG_TYPES return _CHANGES_ARG_TYPES -def python_to_couch(options): +def python_to_couch(options, encoder=None): """ Translates query options from python style options into CouchDB/Cloudant query options. For example ``{'include_docs': True}`` will @@ -161,13 +173,14 @@ def python_to_couch(options): :func:`~cloudant.view.View.__call__` callable, both used to retrieve data. :param dict options: Python style parameters to be translated. + :param encoder: Custom encoder, defaults to None :returns: Dictionary of translated CouchDB/Cloudant query parameters """ translation = dict() for key, val in iteritems_(options): py_to_couch_validate(key, val) - translation.update(_py_to_couch_translate(key, val)) + translation.update(_py_to_couch_translate(key, val, encoder)) return translation def py_to_couch_validate(key, val): @@ -180,30 +193,32 @@ def py_to_couch_validate(key, val): # Validate argument values and ensure that a boolean is not passed in # if an integer is expected if (not isinstance(val, RESULT_ARG_TYPES[key]) or - (type(val) is bool and int in RESULT_ARG_TYPES[key])): + (type(val) is bool and bool not in RESULT_ARG_TYPES[key] and + int in RESULT_ARG_TYPES[key])): raise CloudantArgumentError(117, key, RESULT_ARG_TYPES[key]) if key == 'keys': for key_list_val in val: if (not isinstance(key_list_val, RESULT_ARG_TYPES['key']) or - type(key_list_val) is bool): + isinstance(key_list_val, bool)): raise CloudantArgumentError(134, RESULT_ARG_TYPES['key']) if key == 'stale': if val not in ('ok', 'update_after'): raise CloudantArgumentError(135, val) -def _py_to_couch_translate(key, val): +def _py_to_couch_translate(key, val, encoder=None): """ Performs the conversion of the Python parameter value to its CouchDB equivalent. """ try: - if key in ['keys', 'endkey_docid', 'startkey_docid', 'stale']: + if key in ['keys', 'endkey_docid', 'startkey_docid', 'stale', 'update']: return {key: val} - elif val is None: + if key in ['endkey', 'key', 'startkey']: + return {key: json.dumps(val, cls=encoder)} + if val is None: return {key: None} - else: - arg_converter = TYPE_CONVERTERS.get(type(val)) - return {key: arg_converter(val)} + arg_converter = TYPE_CONVERTERS.get(type(val)) + return {key: arg_converter(val)} except Exception as ex: raise CloudantArgumentError(136, key, ex) @@ -238,11 +253,11 @@ def get_docs(r_session, url, encoder=None, headers=None, **params): """ keys_list = params.pop('keys', None) keys = None - if keys_list: + if keys_list is not None: keys = json.dumps({'keys': keys_list}, cls=encoder) - f_params = python_to_couch(params) + f_params = python_to_couch(params, encoder) resp = None - if keys: + if keys is not None: # If we're using POST we are sending JSON so add the header if headers is None: headers = {} @@ -266,7 +281,7 @@ def append_response_error_content(response, **kwargs): """ if response.status_code >= 400: try: - resp_dict = response.json() + resp_dict = response_to_json_dict(response) error = resp_dict.get('error', '') reason = resp_dict.get('reason', '') # Append to the existing response's reason @@ -275,98 +290,84 @@ def append_response_error_content(response, **kwargs): pass return response -# Classes - -class _Code(str): +def response_to_json_dict(response, **kwargs): """ - Wraps a ``str`` object as a _Code object providing the means to handle - Javascript blob content. Used internally by the View object when - codifying map and reduce Javascript content. + Standard place to convert responses to JSON. + + :param response: requests response object + :param **kwargs: arguments accepted by json.loads + + :returns: dict of JSON response """ - def __new__(cls, code): - return str.__new__(cls, code) + if response.encoding is None: + response.encoding = 'utf-8' + return json.loads(response.text, **kwargs) -class InfiniteSession(Session): +def assert_document_type_id(docid): """ - This class provides for the ability to automatically renew session login - information in the event of expired session authentication. + Validate the document ID. Raises an error if the ID is an `_` prefixed name + that isn't either `_design` or `_local`. + :return: """ + invalid = False + if docid.startswith('_'): + if docid.startswith(DESIGN_PREFIX) and DESIGN_PREFIX != docid: + invalid = False + elif docid.startswith(LOCAL_PREFIX) and LOCAL_PREFIX != docid: + invalid = False + else: + invalid = True + if invalid: + raise CloudantArgumentError(137, docid) - def __init__(self, username, password, server_url, **kwargs): - super(InfiniteSession, self).__init__() - self._username = username - self._password = password - self._server_url = server_url - self._timeout = kwargs.get('timeout', None) - - def request(self, method, url, **kwargs): - """ - Overrides ``requests.Session.request`` to perform a POST to the - _session endpoint to renew Session cookie authentication settings and - then retry the original request, if necessary. - """ - resp = super(InfiniteSession, self).request( - method, url, timeout=self._timeout, **kwargs) - path = url_parse(url).path.lower() - post_to_session = method.upper() == 'POST' and path == '/_session' - is_expired = any(( - resp.status_code == 403 and - resp.json().get('error') == 'credentials_expired', - resp.status_code == 401 - )) - if not post_to_session and is_expired: - super(InfiniteSession, self).request( - 'POST', - '/'.join([self._server_url, '_session']), - data={'name': self._username, 'password': self._password}, - headers={'Content-Type': 'application/x-www-form-urlencoded'} - ) - resp = super(InfiniteSession, self).request( - method, url, timeout=self._timeout, **kwargs) +def assert_attachment_name(attname): + """ + Validate the document attachment's name. Raises an error if `_` prefixed name exists. + :return: + """ + if attname.startswith('_'): + raise CloudantArgumentError(138, attname) +# Classes - return resp -class ClientSession(Session): +class _Code(str): """ - This class extends Session and provides a default timeout. + Wraps a ``str`` object as a _Code object providing the means to handle + Javascript blob content. Used internally by the View object when + codifying map and reduce Javascript content. """ + def __new__(cls, code): + if type(code).__name__ == 'unicode': + return str.__new__(cls, code.encode('utf8')) + return str.__new__(cls, code) - def __init__(self, username, password, server_url, **kwargs): - super(ClientSession, self).__init__() - self._username = username - self._password = password - self._server_url = server_url - self._timeout = kwargs.get('timeout', None) - - def request(self, method, url, **kwargs): - """ - Overrides ``requests.Session.request`` to set the timeout. - """ - resp = super(ClientSession, self).request( - method, url, timeout=self._timeout, **kwargs) - return resp class CloudFoundryService(object): """ Manages Cloud Foundry service configuration. """ - def __init__(self, vcap_services, name=None): + def __init__(self, vcap_services, instance_name=None, service_name=None): try: services = vcap_services if not isinstance(vcap_services, dict): services = json.loads(vcap_services) - cloudant_services = services.get('cloudantNoSQLDB', []) + cloudant_services = services.get(service_name, []) # use first service if no name given and only one service present - use_first = name is None and len(cloudant_services) == 1 + use_first = instance_name is None and len(cloudant_services) == 1 for service in cloudant_services: - if use_first or service.get('name') == name: + if use_first or service.get('name') == instance_name: credentials = service['credentials'] self._host = credentials['host'] self._name = service.get('name') - self._password = credentials['password'] self._port = credentials.get('port', 443) self._username = credentials['username'] + if 'apikey' in credentials: + self._iam_api_key = credentials['apikey'] + elif 'username' in credentials and 'password' in credentials: + self._password = credentials['password'] + else: + raise CloudantClientException(103) break else: raise CloudantException('Missing service in VCAP_SERVICES') @@ -413,3 +414,8 @@ def url(self): def username(self): """ Return service username. """ return self._username + + @property + def iam_api_key(self): + """ Return service IAM API key. """ + return self._iam_api_key diff --git a/src/cloudant/_messages.py b/src/cloudant/_messages.py index 2b6e9257..b428998c 100644 --- a/src/cloudant/_messages.py +++ b/src/cloudant/_messages.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright © 2016, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -65,13 +65,16 @@ # Common_util 134: 'Key list element not of expected type: {0}', 135: 'Invalid value for stale option {0} must be ok or update_after.', - 136: 'Error converting argument {0}: {1}' + 136: 'Error converting argument {0}: {1}', + 137: 'Invalid document ID: {0}', + 138: 'Invalid attachment name: {0}' } CLIENT = { 100: 'A general Cloudant client exception was raised.', 101: 'Value must be set to a Database object. Found type: {0}', 102: 'You must provide a url or an account.', + 103: 'Invalid service: IAM API key or username/password credentials are required.', 404: 'Database {0} does not exist. Verify that the client is valid and try again.', 412: 'Database {0} already exists.' } diff --git a/src/cloudant/adapters.py b/src/cloudant/adapters.py index 7428bbe0..b5bd6964 100644 --- a/src/cloudant/adapters.py +++ b/src/cloudant/adapters.py @@ -18,7 +18,7 @@ """ from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util import Retry +from requests.packages import urllib3 class Replay429Adapter(HTTPAdapter): """ @@ -33,7 +33,7 @@ class Replay429Adapter(HTTPAdapter): :param float initialBackoff: time in seconds for the first backoff. """ def __init__(self, retries=3, initialBackoff=0.25): - super(Replay429Adapter, self).__init__(max_retries=Retry( + super(Replay429Adapter, self).__init__(max_retries=urllib3.util.Retry( # Configure the number of retries for status codes total=retries, # No retries for connect|read errors diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 1cbd428d..47c8555c 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2016, 2017 IBM Corp. All rights reserved. +# Copyright (c) 2015, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,22 +16,27 @@ Top level API module that maps to a Cloudant or CouchDB client connection instance. """ -import base64 import json -import posixpath - -from ._2to3 import bytes_, unicode_ +from ._2to3 import url_parse + +from ._client_session import ( + BasicSession, + ClientSession, + CookieSession, + IAMSession +) from .database import CloudantDatabase, CouchDatabase from .feed import Feed, InfiniteFeed from .error import ( CloudantArgumentError, CloudantClientException, - CloudantDatabaseException) + CloudantDatabaseException, CloudantException) from ._common_util import ( USER_AGENT, append_response_error_content, - InfiniteSession, - ClientSession) + CloudFoundryService, + response_to_json_dict, + ) class CouchDB(dict): @@ -67,6 +72,16 @@ class CouchDB(dict): `Requests library timeout argument `_. but will apply to every request made using this client. + :param bool use_basic_auth: Keyword argument, if set to True performs basic + access authentication with server. Default is False. + :param bool use_iam: Keyword argument, if set to True performs + IAM authentication with server. Default is False. + Use :func:`~cloudant.client.CouchDB.iam` to construct an IAM + authenticated client. + :param string iam_client_id: Keyword argument, client ID to use when + authenticating with the IAM token server. Default is ``None``. + :param string iam_client_secret: Keyword argument, client secret to use when + authenticating with the IAM token server. Default is ``None``. """ _DATABASE_CLASS = CouchDatabase @@ -74,7 +89,6 @@ def __init__(self, user, auth_token, admin_party=False, **kwargs): super(CouchDB, self).__init__() self._user = user self._auth_token = auth_token - self._client_session = None self.server_url = kwargs.get('url') self._client_user_header = None self.admin_party = admin_party @@ -83,39 +97,98 @@ def __init__(self, user, auth_token, admin_party=False, **kwargs): self._timeout = kwargs.get('timeout', None) self.r_session = None self._auto_renew = kwargs.get('auto_renew', False) + self._use_basic_auth = kwargs.get('use_basic_auth', False) + self._use_iam = kwargs.get('use_iam', False) + self._iam_client_id = kwargs.get('iam_client_id', None) + self._iam_client_secret = kwargs.get('iam_client_secret', None) + # If user/pass exist in URL, remove and set variables + if not self._use_basic_auth and self.server_url: + parsed_url = url_parse(kwargs.get('url')) + # Note: To prevent conflicts with field names, the method + # and attribute names of `url_parse` start with an underscore + if parsed_url.port is None: + self.server_url = parsed_url._replace( + netloc="{}".format(parsed_url.hostname)).geturl() + else: + self.server_url = parsed_url._replace( + netloc="{}:{}".format(parsed_url.hostname, parsed_url.port)).geturl() + if (not user and not auth_token) and (parsed_url.username and parsed_url.password): + self._user = parsed_url.username + self._auth_token = parsed_url.password + self._features = None + connect_to_couch = kwargs.get('connect', False) if connect_to_couch and self._DATABASE_CLASS == CouchDatabase: self.connect() + @property + def is_iam_authenticated(self): + """ + Show if a client has authenticated using an IAM API key. + + :return: True if client is IAM authenticated. False otherwise. + """ + return self._use_iam + + def features(self): + """ + lazy fetch and cache features + """ + if self._features is None: + metadata = self.metadata() + if "features" in metadata: + self._features = metadata["features"] + else: + self._features = [] + return self._features + def connect(self): """ Starts up an authentication session for the client using cookie authentication if necessary. """ if self.r_session: - return + self.session_logout() - if self._auto_renew and not self.admin_party: - self.r_session = InfiniteSession( + if self.admin_party: + self._use_iam = False + self.r_session = ClientSession( + timeout=self._timeout + ) + elif self._use_basic_auth: + self._use_iam = False + self.r_session = BasicSession( self._user, self._auth_token, self.server_url, timeout=self._timeout ) + elif self._use_iam: + self.r_session = IAMSession( + self._auth_token, + self.server_url, + auto_renew=self._auto_renew, + client_id=self._iam_client_id, + client_secret=self._iam_client_secret, + timeout=self._timeout + ) else: - self.r_session = ClientSession( + self.r_session = CookieSession( self._user, self._auth_token, self.server_url, + auto_renew=self._auto_renew, timeout=self._timeout ) + # If a Transport Adapter was supplied add it to the session if self.adapter is not None: self.r_session.mount(self.server_url, self.adapter) if self._client_user_header is not None: self.r_session.headers.update(self._client_user_header) - self.session_login(self._user, self._auth_token) - self._client_session = self.session() + + self.session_login() + # Utilize an event hook to append to the response message # using :func:`~cloudant.common_util.append_response_error_content` self.r_session.hooks['response'].append(append_response_error_content) @@ -124,7 +197,9 @@ def disconnect(self): """ Ends a client authentication session, performs a logout and a clean up. """ - self.session_logout() + if self.r_session: + self.session_logout() + self.r_session = None self.clear() @@ -135,13 +210,7 @@ def session(self): :returns: Dictionary of session info for the current session. """ - if self.admin_party: - return None - sess_url = posixpath.join(self.server_url, '_session') - resp = self.r_session.get(sess_url) - resp.raise_for_status() - sess_data = resp.json() - return sess_data + return self.r_session.info() def session_cookie(self): """ @@ -149,41 +218,34 @@ def session_cookie(self): :returns: Session cookie for the current session """ - if self.admin_party: - return None return self.r_session.cookies.get('AuthSession') - def session_login(self, user, passwd): + def session_login(self, user=None, passwd=None): """ Performs a session login by posting the auth information to the _session endpoint. - :param str user: Username used to connect. - :param str passwd: Passcode used to connect. + :param str user: Username used to connect to server. + :param str auth_token: Authentication token used to connect to server. """ - if self.admin_party: - return - sess_url = posixpath.join(self.server_url, '_session') - resp = self.r_session.post( - sess_url, - data={ - 'name': user, - 'password': passwd - }, - headers={'Content-Type': 'application/x-www-form-urlencoded'} - ) - resp.raise_for_status() + self.change_credentials(user=user, auth_token=passwd) + + def change_credentials(self, user=None, auth_token=None): + """ + Change login credentials. + + :param str user: Username used to connect to server. + :param str auth_token: Authentication token used to connect to server. + """ + self.r_session.set_credentials(user, auth_token) + self.r_session.login() def session_logout(self): """ Performs a session logout and clears the current session by sending a delete request to the _session endpoint. """ - if self.admin_party: - return - sess_url = posixpath.join(self.server_url, '_session') - resp = self.r_session.delete(sess_url) - resp.raise_for_status() + self.r_session.logout() def basic_auth_str(self): """ @@ -192,13 +254,7 @@ def basic_auth_str(self): :returns: Basic http authentication string """ - if self.admin_party: - return None - hash_ = base64.urlsafe_b64encode(bytes_("{username}:{password}".format( - username=self._user, - password=self._auth_token - ))) - return "Basic {0}".format(unicode_(hash_)) + return self.r_session.base64_user_pass() def all_dbs(self): """ @@ -206,12 +262,12 @@ def all_dbs(self): :returns: List of database names for the client """ - url = posixpath.join(self.server_url, '_all_dbs') + url = '/'.join((self.server_url, '_all_dbs')) resp = self.r_session.get(url) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) - def create_database(self, dbname, **kwargs): + def create_database(self, dbname, partitioned=False, **kwargs): """ Creates a new database on the remote server with the name provided and adds the new database object to the client's locally cached @@ -223,15 +279,18 @@ def create_database(self, dbname, **kwargs): :param bool throw_on_exists: Boolean flag dictating whether or not to throw a CloudantClientException when attempting to create a database that already exists. + :param bool partitioned: Create as a partitioned database. Defaults to + ``False``. :returns: The newly created database object """ - new_db = self._DATABASE_CLASS(self, dbname) + new_db = self._DATABASE_CLASS(self, dbname, partitioned=partitioned) try: new_db.create(kwargs.get('throw_on_exists', False)) except CloudantDatabaseException as ex: if ex.status_code == 412: raise CloudantClientException(412, dbname) + raise ex super(CouchDB, self).__setitem__(dbname, new_db) return new_db @@ -290,6 +349,16 @@ def db_updates(self, raw_data=False, **kwargs): """ return Feed(self, raw_data, **kwargs) + def metadata(self): + """ + Retrieves the remote server metadata dictionary. + + :returns: Dictionary containing server metadata details + """ + resp = self.r_session.get(self.server_url) + resp.raise_for_status() + return response_to_json_dict(resp) + def keys(self, remote=False): """ Returns the database names for this client. Default is @@ -330,9 +399,9 @@ def __getitem__(self, key): db = self._DATABASE_CLASS(self, key) if db.exists(): super(CouchDB, self).__setitem__(key, db) - return db else: raise KeyError(key) + return db def __delitem__(self, key, remote=False): """ @@ -371,8 +440,8 @@ def get(self, key, default=None, remote=False): if db.exists(): super(CouchDB, self).__setitem__(key, db) return db - else: - return default + + return default def __setitem__(self, key, value, remote=False): """ @@ -431,14 +500,10 @@ def __init__(self, cloudant_user, auth_token, **kwargs): super(Cloudant, self).__init__(cloudant_user, auth_token, **kwargs) self._client_user_header = {'User-Agent': USER_AGENT} account = kwargs.get('account') - url = kwargs.get('url') - x_cloudant_user = kwargs.get('x_cloudant_user') if account is not None: self.server_url = 'https://{0}.cloudant.com'.format(account) - elif kwargs.get('url') is not None: - self.server_url = url - if x_cloudant_user is not None: - self._client_user_header['X-Cloudant-User'] = x_cloudant_user + if kwargs.get('x_cloudant_user') is not None: + self._client_user_header['X-Cloudant-User'] = kwargs.get('x_cloudant_user') if self.server_url is None: raise CloudantClientException(102) @@ -559,9 +624,7 @@ def _usage_endpoint(self, endpoint, year=None, month=None): try: if int(year) > 0 and int(month) in range(1, 13): resp = self.r_session.get( - posixpath.join( - endpoint, str(int(year)), str(int(month))) - ) + '/'.join((endpoint, str(int(year)), str(int(month))))) else: err = True except (ValueError, TypeError): @@ -569,9 +632,9 @@ def _usage_endpoint(self, endpoint, year=None, month=None): if err: raise CloudantArgumentError(101, year, month) - else: - resp.raise_for_status() - return resp.json() + + resp.raise_for_status() + return response_to_json_dict(resp) def bill(self, year=None, month=None): """ @@ -586,7 +649,7 @@ def bill(self, year=None, month=None): :returns: Billing data in JSON format """ - endpoint = posixpath.join(self.server_url, '_api', 'v2', 'bill') + endpoint = '/'.join((self.server_url, '_api', 'v2', 'bill')) return self._usage_endpoint(endpoint, year, month) def volume_usage(self, year=None, month=None): @@ -603,9 +666,8 @@ def volume_usage(self, year=None, month=None): :returns: Volume usage data in JSON format """ - endpoint = posixpath.join( - self.server_url, '_api', 'v2', 'usage', 'data_volume' - ) + endpoint = '/'.join(( + self.server_url, '_api', 'v2', 'usage', 'data_volume')) return self._usage_endpoint(endpoint, year, month) def requests_usage(self, year=None, month=None): @@ -622,9 +684,8 @@ def requests_usage(self, year=None, month=None): :returns: Requests usage data in JSON format """ - endpoint = posixpath.join( - self.server_url, '_api', 'v2', 'usage', 'requests' - ) + endpoint = '/'.join(( + self.server_url, '_api', 'v2', 'usage', 'requests')) return self._usage_endpoint(endpoint, year, month) def shared_databases(self): @@ -634,12 +695,11 @@ def shared_databases(self): :returns: List of database names """ - endpoint = posixpath.join( - self.server_url, '_api', 'v2', 'user', 'shared_databases' - ) + endpoint = '/'.join(( + self.server_url, '_api', 'v2', 'user', 'shared_databases')) resp = self.r_session.get(endpoint) resp.raise_for_status() - data = resp.json() + data = response_to_json_dict(resp) return data.get('shared_databases', []) def generate_api_key(self): @@ -648,12 +708,10 @@ def generate_api_key(self): :returns: API key/pass pair in JSON format """ - endpoint = posixpath.join( - self.server_url, '_api', 'v2', 'api_keys' - ) + endpoint = '/'.join((self.server_url, '_api', 'v2', 'api_keys')) resp = self.r_session.post(endpoint) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def cors_configuration(self): """ @@ -661,13 +719,12 @@ def cors_configuration(self): :returns: CORS data in JSON format """ - endpoint = posixpath.join( - self.server_url, '_api', 'v2', 'user', 'config', 'cors' - ) + endpoint = '/'.join(( + self.server_url, '_api', 'v2', 'user', 'config', 'cors')) resp = self.r_session.get(endpoint) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def disable_cors(self): """ @@ -753,9 +810,8 @@ def _write_cors_configuration(self, config): :returns: CORS configuration update status in JSON format """ - endpoint = posixpath.join( - self.server_url, '_api', 'v2', 'user', 'config', 'cors' - ) + endpoint = '/'.join(( + self.server_url, '_api', 'v2', 'user', 'config', 'cors')) resp = self.r_session.put( endpoint, data=json.dumps(config, cls=self.encoder), @@ -763,4 +819,60 @@ def _write_cors_configuration(self, config): ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) + + @classmethod + def bluemix(cls, vcap_services, instance_name=None, service_name=None, **kwargs): + """ + Create a Cloudant session using a VCAP_SERVICES environment variable. + + :param vcap_services: VCAP_SERVICES environment variable + :type vcap_services: dict or str + :param str instance_name: Optional Bluemix instance name. Only required + if multiple Cloudant instances are available. + :param str service_name: Optional Bluemix service name. + + Example usage: + + .. code-block:: python + + import os + from cloudant.client import Cloudant + + client = Cloudant.bluemix(os.getenv('VCAP_SERVICES'), + 'Cloudant NoSQL DB') + + print client.all_dbs() + """ + service_name = service_name or 'cloudantNoSQLDB' # default service + try: + service = CloudFoundryService(vcap_services, + instance_name=instance_name, + service_name=service_name) + except CloudantException: + raise CloudantClientException(103) + + if hasattr(service, 'iam_api_key'): + return Cloudant.iam(None, + service.iam_api_key, + url=service.url, + **kwargs) + return Cloudant(service.username, + service.password, + url=service.url, + **kwargs) + + @classmethod + def iam(cls, account_name, api_key, **kwargs): + """ + Create a Cloudant client that uses IAM authentication. + + :param account_name: Cloudant account name; or use None and a url kwarg. + :param api_key: IAM authentication API key. + """ + return cls(None, + api_key, + account=account_name, + auto_renew=kwargs.get('auto_renew', True), + use_iam=True, + **kwargs) diff --git a/src/cloudant/database.py b/src/cloudant/database.py index bae7e5c1..4ad93f90 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,7 +17,6 @@ """ import json import contextlib -import posixpath from requests.exceptions import HTTPError @@ -27,7 +26,9 @@ SEARCH_INDEX_ARGS, SPECIAL_INDEX_TYPE, TEXT_INDEX_TYPE, - get_docs) + TYPE_CONVERTERS, + get_docs, + response_to_json_dict) from .document import Document from .design_document import DesignDocument from .security_document import SecurityDocument @@ -49,13 +50,17 @@ class CouchDatabase(dict): :param str database_name: Database name used to reference the database. :param int fetch_limit: Optional fetch limit used to set the max number of documents to fetch per query during iteration cycles. Defaults to 100. + :param bool partitioned: Create as a partitioned database. Defaults to + ``False``. """ - def __init__(self, client, database_name, fetch_limit=100): + def __init__(self, client, database_name, fetch_limit=100, + partitioned=False): super(CouchDatabase, self).__init__() self.client = client self._database_host = client.server_url self.database_name = database_name self._fetch_limit = fetch_limit + self._partitioned = partitioned self.result = Result(self.all_docs) @property @@ -84,10 +89,8 @@ def database_url(self): :returns: Database URL """ - return posixpath.join( - self._database_host, - url_quote_plus(self.database_name) - ) + return '/'.join(( + self._database_host, url_quote_plus(self.database_name))) @property def creds(self): @@ -97,13 +100,27 @@ def creds(self): :returns: Dictionary containing authentication information """ - if self.admin_party: + session = self.client.session() + if session is None: return None + return { "basic_auth": self.client.basic_auth_str(), - "user_ctx": self.client.session()['userCtx'] + "user_ctx": session.get('userCtx') } + def database_partition_url(self, partition_key): + """ + Get the URL of the database partition. + + :param str partition_key: Partition key. + :return: URL of the database partition. + :rtype: str + """ + return '/'.join((self.database_url, + '_partition', + url_quote_plus(partition_key))) + def exists(self): """ Performs an existence check on the remote database. @@ -124,7 +141,19 @@ def metadata(self): """ resp = self.r_session.get(self.database_url) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) + + def partition_metadata(self, partition_key): + """ + Retrieves the metadata dictionary for the remote database partition. + + :param str partition_key: Partition key. + :returns: Metadata dictionary for the database partition. + :rtype: dict + """ + resp = self.r_session.get(self.database_partition_url(partition_key)) + resp.raise_for_status() + return response_to_json_dict(resp) def doc_count(self): """ @@ -157,10 +186,15 @@ def create_document(self, data, throw_on_exists=False): doc = DesignDocument(self, docid) else: doc = Document(self, docid) - if throw_on_exists and doc.exists(): - raise CloudantDatabaseException(409, docid) doc.update(data) - doc.create() + try: + doc.create() + except HTTPError as error: + if error.response.status_code == 409: + if throw_on_exists: + raise CloudantDatabaseException(409, docid) + else: + raise super(CouchDatabase, self).__setitem__(doc['_id'], doc) return doc @@ -184,11 +218,11 @@ def design_documents(self): :returns: All design documents found in this database in JSON format """ - url = posixpath.join(self.database_url, '_all_docs') + url = '/'.join((self.database_url, '_all_docs')) query = "startkey=\"_design\"&endkey=\"_design0\"&include_docs=true" resp = self.r_session.get(url, params=query) resp.raise_for_status() - data = resp.json() + data = response_to_json_dict(resp) return data['rows'] def list_design_documents(self): @@ -198,11 +232,11 @@ def list_design_documents(self): :returns: List of names for all design documents in this database """ - url = posixpath.join(self.database_url, '_all_docs') + url = '/'.join((self.database_url, '_all_docs')) query = "startkey=\"_design\"&endkey=\"_design0\"" resp = self.r_session.get(url, params=query) resp.raise_for_status() - data = resp.json() + data = response_to_json_dict(resp) return [x.get('key') for x in data.get('rows', [])] def get_design_document(self, ddoc_id): @@ -238,6 +272,33 @@ def get_security_document(self): sdoc.fetch() return sdoc + def get_partitioned_view_result(self, partition_key, ddoc_id, view_name, + raw_result=False, **kwargs): + """ + Retrieves the partitioned view result based on the design document and + view name. + + See :func:`~cloudant.database.CouchDatabase.get_view_result` method for + further details. + + :param str partition_key: Partition key. + :param str ddoc_id: Design document id used to get result. + :param str view_name: Name of the view used to get result. + :param bool raw_result: Dictates whether the view result is returned + as a default Result object or a raw JSON response. + Defaults to False. + :param kwargs: See + :func:`~cloudant.database.CouchDatabase.get_view_result` method for + available keyword arguments. + :returns: The result content either wrapped in a QueryResult or + as the raw response JSON content. + :rtype: QueryResult, dict + """ + ddoc = DesignDocument(self, ddoc_id) + view = View(ddoc, view_name, partition_key=partition_key) + + return self._get_view_result(view, raw_result, **kwargs) + def get_view_result(self, ddoc_id, view_name, raw_result=False, **kwargs): """ Retrieves the view result based on the design document and view name. @@ -308,26 +369,42 @@ def get_view_result(self, ddoc_id, view_name, raw_result=False, **kwargs): :param bool reduce: True to use the reduce function, false otherwise. :param int skip: Skip this number of rows from the start. Not valid when used with :class:`~cloudant.result.Result` iteration. + :param bool stable: Whether or not the view results should be returned + from a "stable" set of shards. :param str stale: Allow the results from a stale view to be used. This makes the request return immediately, even if the view has not been completely built yet. If this parameter is not given, a response is - returned only after the view has been built. + returned only after the view has been built. Note that this + parameter is deprecated and the appropriate combination of `stable` + and `update` should be used instead. :param startkey: Return records starting with the specified key. Not valid when used with :class:`~cloudant.result.Result` key access and key slicing. :param str startkey_docid: Return records starting with the specified document ID. + :param str update: Determine whether the view in question should be + updated prior to or after responding to the user. Valid values are: + false: return results before updating the view; true: Return results + after updating the view; lazy: Return the view results without + waiting for an update, but update them immediately after the request. :returns: The result content either wrapped in a QueryResult or as the raw response JSON content """ - view = View(DesignDocument(self, ddoc_id), view_name) + ddoc = DesignDocument(self, ddoc_id) + view = View(ddoc, view_name) + + return self._get_view_result(view, raw_result, **kwargs) + + @staticmethod + def _get_view_result(view, raw_result, **kwargs): + """ Get view results helper. """ if raw_result: return view(**kwargs) - elif kwargs: + if kwargs: return Result(view, **kwargs) - else: - return view.result + + return view.result def create(self, throw_on_exists=False): """ @@ -344,7 +421,9 @@ def create(self, throw_on_exists=False): if not throw_on_exists and self.exists(): return self - resp = self.r_session.put(self.database_url) + resp = self.r_session.put(self.database_url, params={ + 'partitioned': TYPE_CONVERTERS.get(bool)(self._partitioned) + }) if resp.status_code == 201 or resp.status_code == 202: return self @@ -390,7 +469,30 @@ def all_docs(self, **kwargs): '/'.join([self.database_url, '_all_docs']), self.client.encoder, **kwargs) - return resp.json() + return response_to_json_dict(resp) + + def partitioned_all_docs(self, partition_key, **kwargs): + """ + Wraps the _all_docs primary index on the database partition, and returns + the results by value. + + See :func:`~cloudant.database.CouchDatabase.all_docs` method for further + details. + + :param str partition_key: Partition key. + :param kwargs: See :func:`~cloudant.database.CouchDatabase.all_docs` + method for available keyword arguments. + :returns: Raw JSON response content from ``_all_docs`` endpoint. + :rtype: dict + """ + resp = get_docs(self.r_session, + '/'.join([ + self.database_partition_url(partition_key), + '_all_docs' + ]), + self.client.encoder, + **kwargs) + return response_to_json_dict(resp) @contextlib.contextmanager def custom_result(self, **options): @@ -600,9 +702,53 @@ def __getitem__(self, key): if doc.exists(): doc.fetch() super(CouchDatabase, self).__setitem__(key, doc) - return doc else: raise KeyError(key) + return doc + + def get(self, key, remote=False): + """ + Overrides dict's get method. This gets an item from the database or cache + like __getitem__, but instead of throwing an exception if the item is not + found, it simply returns None. + + :param bool remote: Dictates whether a remote request is made to + retrieve the doc, if it is not present in the local cache. + Defaults to False. + """ + if remote: + try: + return self.__getitem__(key) + except KeyError: + return None + else: + return super(CouchDatabase, self).get(key) + + def __contains__(self, key): + """ + Overrides dictionary __contains__ behavior to check if a document + by key exists in the current cached or remote database. + + For example: + + .. code-block:: python + + if key in database: + doc = database[key] + # Do something with doc + + :param str key: Document id used to check if it exists in the database. + + :returns: True if the document exists in the local or remote + database, otherwise False. + """ + if key in list(self.keys()): + return True + if key.startswith('_design/'): + doc = DesignDocument(self, key) + else: + doc = Document(self, key) + return doc.exists() def __iter__(self, remote=True): """ @@ -625,18 +771,20 @@ def __iter__(self, remote=True): if not remote: super(CouchDatabase, self).__iter__() else: - next_startkey = '0' + # Use unicode Null U+0000 as the initial lower bound to ensure any + # document id could exist in the results set. + next_startkey = u'\u0000' while next_startkey is not None: docs = self.all_docs( - limit=self._fetch_limit + 1, # Get one extra doc - # to use as - # next_startkey + limit=self._fetch_limit, include_docs=True, startkey=next_startkey ).get('rows', []) - if len(docs) > self._fetch_limit: - next_startkey = docs.pop()['id'] + if len(docs) >= self._fetch_limit: + # Ensure the next document batch contains ids that sort + # strictly higher than the previous document id fetched. + next_startkey = docs[-1]['id'] + u'\u0000' else: # This is the last batch of docs, so we set # ourselves up to break out of the while loop @@ -654,7 +802,7 @@ def __iter__(self, remote=True): super(CouchDatabase, self).__setitem__(doc['id'], document) yield document - raise StopIteration + return def bulk_docs(self, docs): """ @@ -668,7 +816,7 @@ def bulk_docs(self, docs): :returns: Bulk document creation/update status in JSON format """ - url = posixpath.join(self.database_url, '_bulk_docs') + url = '/'.join((self.database_url, '_bulk_docs')) data = {'docs': docs} headers = {'Content-Type': 'application/json'} resp = self.r_session.post( @@ -677,7 +825,7 @@ def bulk_docs(self, docs): headers=headers ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def missing_revisions(self, doc_id, *revisions): """ @@ -691,7 +839,7 @@ def missing_revisions(self, doc_id, *revisions): :returns: List of missing document revision values """ - url = posixpath.join(self.database_url, '_missing_revs') + url = '/'.join((self.database_url, '_missing_revs')) data = {doc_id: list(revisions)} resp = self.r_session.post( @@ -701,7 +849,7 @@ def missing_revisions(self, doc_id, *revisions): ) resp.raise_for_status() - resp_json = resp.json() + resp_json = response_to_json_dict(resp) missing_revs = resp_json['missing_revs'].get(doc_id) if missing_revs is None: missing_revs = [] @@ -720,7 +868,7 @@ def revisions_diff(self, doc_id, *revisions): :returns: The revision differences in JSON format """ - url = posixpath.join(self.database_url, '_revs_diff') + url = '/'.join((self.database_url, '_revs_diff')) data = {doc_id: list(revisions)} resp = self.r_session.post( @@ -730,7 +878,7 @@ def revisions_diff(self, doc_id, *revisions): ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def get_revision_limit(self): """ @@ -739,14 +887,14 @@ def get_revision_limit(self): :returns: Revision limit value for the current remote database """ - url = posixpath.join(self.database_url, '_revs_limit') + url = '/'.join((self.database_url, '_revs_limit')) resp = self.r_session.get(url) resp.raise_for_status() try: ret = int(resp.text) except ValueError: - raise CloudantDatabaseException(400, resp.json()) + raise CloudantDatabaseException(400, response_to_json_dict(resp)) return ret @@ -760,12 +908,12 @@ def set_revision_limit(self, limit): :returns: Revision limit set operation status in JSON format """ - url = posixpath.join(self.database_url, '_revs_limit') + url = '/'.join((self.database_url, '_revs_limit')) resp = self.r_session.put(url, data=json.dumps(limit, cls=self.client.encoder)) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def view_cleanup(self): """ @@ -774,14 +922,14 @@ def view_cleanup(self): :returns: View cleanup status in JSON format """ - url = posixpath.join(self.database_url, '_view_cleanup') + url = '/'.join((self.database_url, '_view_cleanup')) resp = self.r_session.post( url, headers={'Content-Type': 'application/json'} ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def get_list_function_result(self, ddoc_id, list_name, view_name, **kwargs): """ @@ -802,12 +950,12 @@ def get_list_function_result(self, ddoc_id, list_name, view_name, **kwargs): # Assuming that 'view001' exists as part of the # 'ddoc001' design document in the remote database... # Retrieve documents where the list function is 'list1' - resp = db.get_list_result('ddoc001', 'list1', 'view001', limit=10) + resp = db.get_list_function_result('ddoc001', 'list1', 'view001', limit=10) for row in resp['rows']: # Process data (in text format). For more detail on list functions, refer to the - `Cloudant list documentation `_. :param str ddoc_id: Design document id used to get result. @@ -845,7 +993,7 @@ def get_show_function_result(self, ddoc_id, show_name, doc_id): # Process data (in text format). For more detail on show functions, refer to the - `Cloudant show documentation `_. :param str ddoc_id: Design document id used to get the result. @@ -893,7 +1041,7 @@ def update_handler_result(self, ddoc_id, handler_name, doc_id=None, data=None, * data={'month': 'July'}) For more details, see the `update handlers documentation - `_. + `_. :param str ddoc_id: Design document id used to get result. :param str handler_name: Name used in part to identify the @@ -915,130 +1063,6 @@ def update_handler_result(self, ddoc_id, handler_name, doc_id=None, data=None, * resp.raise_for_status() return resp.text -class CloudantDatabase(CouchDatabase): - """ - Encapsulates a Cloudant database. A CloudantDatabase object is - instantiated with a reference to a client/session. - It supports accessing the documents, and various database - features such as the document indexes, changes feed, design documents, etc. - - :param Cloudant client: Client instance used by the database. - :param str database_name: Database name used to reference the database. - :param int fetch_limit: Optional fetch limit used to set the max number of - documents to fetch per query during iteration cycles. Defaults to 100. - """ - def __init__(self, client, database_name, fetch_limit=100): - super(CloudantDatabase, self).__init__( - client, - database_name, - fetch_limit=fetch_limit - ) - - def security_document(self): - """ - Retrieves the security document for the current database - containing information about the users that the database - is shared with. - - :returns: Security document as a ``dict`` - """ - return dict(self.get_security_document()) - - @property - def security_url(self): - """ - Constructs and returns the security document URL. - - :returns: Security document URL - """ - parts = ['_api', 'v2', 'db', self.database_name, '_security'] - url = posixpath.join(self._database_host, *parts) - return url - - def share_database(self, username, roles=None): - """ - Shares the current remote database with the username provided. - You can grant varying degrees of access rights, - default is to share read-only, but additional - roles can be added by providing the specific roles as a - ``list`` argument. If the user already has this database shared with - them then it will modify/overwrite the existing permissions. - - :param str username: Cloudant user to share the database with. - :param list roles: A list of - `roles `_ - to grant to the named user. - - :returns: Share database status in JSON format - """ - if roles is None: - roles = ['_reader'] - valid_roles = [ - '_reader', - '_writer', - '_admin', - '_replicator', - '_db_updates', - '_design', - '_shards', - '_security' - ] - doc = self.security_document() - data = doc.get('cloudant', {}) - perms = [] - if all(role in valid_roles for role in roles): - perms = list(set(roles)) - - if not perms: - raise CloudantArgumentError(102, roles, valid_roles) - - data[username] = perms - doc['cloudant'] = data - resp = self.r_session.put( - self.security_url, - data=json.dumps(doc, cls=self.client.encoder), - headers={'Content-Type': 'application/json'} - ) - resp.raise_for_status() - return resp.json() - - def unshare_database(self, username): - """ - Removes all sharing with the named user for the current remote database. - This will remove the entry for the user from the security document. - To modify permissions, use the - :func:`~cloudant.database.CloudantDatabase.share_database` method - instead. - - :param str username: Cloudant user to unshare the database from. - - :returns: Unshare database status in JSON format - """ - doc = self.security_document() - data = doc.get('cloudant', {}) - if username in data: - del data[username] - doc['cloudant'] = data - resp = self.r_session.put( - self.security_url, - data=json.dumps(doc, cls=self.client.encoder), - headers={'Content-Type': 'application/json'} - ) - resp.raise_for_status() - return resp.json() - - def shards(self): - """ - Retrieves information about the shards in the current remote database. - - :returns: Shard information retrieval status in JSON format - """ - url = posixpath.join(self.database_url, '_shards') - resp = self.r_session.get(url) - resp.raise_for_status() - - return resp.json() - def get_query_indexes(self, raw_result=False): """ Retrieves query indexes from the remote database. @@ -1052,20 +1076,21 @@ def get_query_indexes(self, raw_result=False): :returns: The query indexes in the database """ - url = posixpath.join(self.database_url, '_index') + url = '/'.join((self.database_url, '_index')) resp = self.r_session.get(url) resp.raise_for_status() if raw_result: - return resp.json() + return response_to_json_dict(resp) indexes = [] - for data in resp.json().get('indexes', []): + for data in response_to_json_dict(resp).get('indexes', []): if data.get('type') == JSON_INDEX_TYPE: indexes.append(Index( self, data.get('ddoc'), data.get('name'), + partitioned=data.get('partitioned', False), **data.get('def', {}) )) elif data.get('type') == TEXT_INDEX_TYPE: @@ -1073,6 +1098,7 @@ def get_query_indexes(self, raw_result=False): self, data.get('ddoc'), data.get('name'), + partitioned=data.get('partitioned', False), **data.get('def', {}) )) elif data.get('type') == SPECIAL_INDEX_TYPE: @@ -1080,6 +1106,7 @@ def get_query_indexes(self, raw_result=False): self, data.get('ddoc'), data.get('name'), + partitioned=data.get('partitioned', False), **data.get('def', {}) )) else: @@ -1091,6 +1118,7 @@ def create_query_index( design_document_id=None, index_name=None, index_type='json', + partitioned=None, **kwargs ): """ @@ -1128,9 +1156,11 @@ def create_query_index( remote database """ if index_type == JSON_INDEX_TYPE: - index = Index(self, design_document_id, index_name, **kwargs) + index = Index(self, design_document_id, index_name, + partitioned=partitioned, **kwargs) elif index_type == TEXT_INDEX_TYPE: - index = TextIndex(self, design_document_id, index_name, **kwargs) + index = TextIndex(self, design_document_id, index_name, + partitioned=partitioned, **kwargs) else: raise CloudantArgumentError(103, index_type) index.create() @@ -1155,6 +1185,36 @@ def delete_query_index(self, design_document_id, index_type, index_name): raise CloudantArgumentError(103, index_type) index.delete() + def get_partitioned_query_result(self, partition_key, selector, fields=None, + raw_result=False, **kwargs): + """ + Retrieves the partitioned query result from the specified database based + on the query parameters provided. + + See :func:`~cloudant.database.CouchDatabase.get_query_result` method for + further details. + + :param str partition_key: Partition key. + :param str selector: Dictionary object describing criteria used to + select documents. + :param list fields: A list of fields to be returned by the query. + :param bool raw_result: Dictates whether the query result is returned + wrapped in a QueryResult or if the response JSON is returned. + Defaults to False. + :param kwargs: See + :func:`~cloudant.database.CouchDatabase.get_query_result` method for + available keyword arguments. + :returns: The result content either wrapped in a QueryResult or + as the raw response JSON content. + :rtype: QueryResult, dict + """ + query = Query(self, + selector=selector, + fields=fields, + partition_key=partition_key) + + return self._get_query_result(query, raw_result, **kwargs) + def get_query_result(self, selector, fields=None, raw_result=False, **kwargs): """ @@ -1193,15 +1253,14 @@ def get_query_result(self, selector, fields=None, raw_result=False, For more detail on slicing and iteration, refer to the :class:`~cloudant.result.QueryResult` documentation. - :param str selector: Dictionary object describing criteria used to + :param dict selector: Dictionary object describing criteria used to select documents. :param list fields: A list of fields to be returned by the query. :param bool raw_result: Dictates whether the query result is returned wrapped in a QueryResult or if the response JSON is returned. Defaults to False. :param str bookmark: A string that enables you to specify which page of - results you require. Only valid for queries using indexes of type - *text*. + results you require. :param int limit: Maximum number of results returned. Only valid if used with ``raw_result=True``. :param int page_size: Sets the page size for result iteration. Default @@ -1224,16 +1283,181 @@ def get_query_result(self, selector, fields=None, raw_result=False, :returns: The result content either wrapped in a QueryResult or as the raw response JSON content """ - if fields: - query = Query(self, selector=selector, fields=fields) - else: - query = Query(self, selector=selector) + query = Query(self, + selector=selector, + fields=fields) + + return self._get_query_result(query, raw_result, **kwargs) + + @staticmethod + def _get_query_result(query, raw_result, **kwargs): + """ Get query results helper. """ if raw_result: return query(**kwargs) if kwargs: return QueryResult(query, **kwargs) - else: - return query.result + + return query.result + + +class CloudantDatabase(CouchDatabase): + """ + Encapsulates a Cloudant database. A CloudantDatabase object is + instantiated with a reference to a client/session. + It supports accessing the documents, and various database + features such as the document indexes, changes feed, design documents, etc. + + :param Cloudant client: Client instance used by the database. + :param str database_name: Database name used to reference the database. + :param int fetch_limit: Optional fetch limit used to set the max number of + documents to fetch per query during iteration cycles. Defaults to 100. + :param bool partitioned: Create as a partitioned database. Defaults to + ``False``. + """ + def __init__(self, client, database_name, fetch_limit=100, + partitioned=False): + super(CloudantDatabase, self).__init__( + client, + database_name, + fetch_limit=fetch_limit, + partitioned=partitioned + ) + + def security_document(self): + """ + Retrieves the security document for the current database + containing information about the users that the database + is shared with. + + :returns: Security document as a ``dict`` + """ + return dict(self.get_security_document()) + + @property + def security_url(self): + """ + Constructs and returns the security document URL. + + :returns: Security document URL + """ + url = '/'.join((self._database_host, '_api', 'v2', 'db', + self.database_name, '_security')) + return url + + def share_database(self, username, roles=None): + """ + Shares the current remote database with the username provided. + You can grant varying degrees of access rights, + default is to share read-only, but additional + roles can be added by providing the specific roles as a + ``list`` argument. If the user already has this database shared with + them then it will modify/overwrite the existing permissions. + + :param str username: Cloudant user to share the database with. + :param list roles: A list of + `roles + `_ + to grant to the named user. + + :returns: Share database status in JSON format + """ + if roles is None: + roles = ['_reader'] + valid_roles = [ + '_reader', + '_writer', + '_admin', + '_replicator', + '_db_updates', + '_design', + '_shards', + '_security' + ] + doc = self.security_document() + data = doc.get('cloudant', {}) + perms = [] + if all(role in valid_roles for role in roles): + perms = list(set(roles)) + + if not perms: + raise CloudantArgumentError(102, roles, valid_roles) + + data[username] = perms + doc['cloudant'] = data + resp = self.r_session.put( + self.security_url, + data=json.dumps(doc, cls=self.client.encoder), + headers={'Content-Type': 'application/json'} + ) + resp.raise_for_status() + return response_to_json_dict(resp) + + def unshare_database(self, username): + """ + Removes all sharing with the named user for the current remote database. + This will remove the entry for the user from the security document. + To modify permissions, use the + :func:`~cloudant.database.CloudantDatabase.share_database` method + instead. + + :param str username: Cloudant user to unshare the database from. + + :returns: Unshare database status in JSON format + """ + doc = self.security_document() + data = doc.get('cloudant', {}) + if username in data: + del data[username] + doc['cloudant'] = data + resp = self.r_session.put( + self.security_url, + data=json.dumps(doc, cls=self.client.encoder), + headers={'Content-Type': 'application/json'} + ) + resp.raise_for_status() + return response_to_json_dict(resp) + + def shards(self): + """ + Retrieves information about the shards in the current remote database. + + :returns: Shard information retrieval status in JSON format + """ + url = '/'.join((self.database_url, '_shards')) + resp = self.r_session.get(url) + resp.raise_for_status() + + return response_to_json_dict(resp) + + def get_partitioned_search_result(self, partition_key, ddoc_id, index_name, + **query_params): + """ + Retrieves the raw JSON content from the remote database based on the + partitioned search index on the server, using the query_params provided + as query parameters. + + See :func:`~cloudant.database.CouchDatabase.get_search_result` method + for further details. + + :param str partition_key: Partition key. + :param str ddoc_id: Design document id used to get the search result. + :param str index_name: Name used in part to identify the index. + :param query_params: See + :func:`~cloudant.database.CloudantDatabase.get_search_result` method + for available keyword arguments. + :returns: Search query result data in JSON format. + :rtype: dict + """ + ddoc = DesignDocument(self, ddoc_id) + + return self._get_search_result( + '/'.join(( + ddoc.document_partition_url(partition_key), + '_search', + index_name + )), + **query_params + ) def get_search_result(self, ddoc_id, index_name, **query_params): """ @@ -1335,6 +1559,14 @@ def get_search_result(self, ddoc_id, index_name, **query_params): :returns: Search query result data in JSON format """ + ddoc = DesignDocument(self, ddoc_id) + return self._get_search_result( + '/'.join((ddoc.document_url, '_search', index_name)), + **query_params + ) + + def _get_search_result(self, query_url, **query_params): + """ Get search results helper. """ param_q = query_params.get('q') param_query = query_params.get('query') # Either q or query parameter is required @@ -1349,11 +1581,10 @@ def get_search_result(self, ddoc_id, index_name, **query_params): raise CloudantArgumentError(106, key, SEARCH_INDEX_ARGS[key]) # Execute query search headers = {'Content-Type': 'application/json'} - ddoc = DesignDocument(self, ddoc_id) resp = self.r_session.post( - '/'.join([ddoc.document_url, '_search', index_name]), + query_url, headers=headers, data=json.dumps(query_params, cls=self.client.encoder) ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) diff --git a/src/cloudant/design_document.py b/src/cloudant/design_document.py index b3f5a187..df9fb44d 100644 --- a/src/cloudant/design_document.py +++ b/src/cloudant/design_document.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2019 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,8 +15,9 @@ """ API module/class for interacting with a design document in a database. """ -from ._2to3 import iteritems_, STRTYPE -from ._common_util import QUERY_LANGUAGE, codify +from ._2to3 import iteritems_, url_quote_plus, STRTYPE +from ._common_util import QUERY_LANGUAGE, codify, response_to_json_dict, \ + assert_document_type_id, DESIGN_PREFIX from .document import Document from .view import View, QueryIndexView from .error import CloudantArgumentError, CloudantDesignDocumentException @@ -39,11 +40,22 @@ class DesignDocument(Document): either a ``CouchDatabase`` or ``CloudantDatabase`` instance. :param str document_id: Optional document id. If provided and does not start with ``_design/``, it will be prepended with ``_design/``. + :param bool partitioned: Optional. Create as a partitioned design document. + Defaults to ``False`` for both partitioned and non-partitioned + databases. """ - def __init__(self, database, document_id=None): - if document_id and not document_id.startswith('_design/'): - document_id = '_design/{0}'.format(document_id) + def __init__(self, database, document_id=None, partitioned=False): + if document_id: + assert_document_type_id(document_id) + if document_id and not document_id.startswith(DESIGN_PREFIX): + document_id = '{0}{1}'.format(DESIGN_PREFIX, document_id) super(DesignDocument, self).__init__(database, document_id) + + if partitioned: + self.setdefault('options', {'partitioned': True}) + else: + self.setdefault('options', {'partitioned': False}) + self._nested_object_names = frozenset(['views', 'indexes', 'lists', 'shows']) for prop in self._nested_object_names: self.setdefault(prop, dict()) @@ -68,7 +80,7 @@ def validate_doc_update(self): ddoc.save() For more details, see the `Update Validators documentation - `_. + `_. :returns: Dictionary containing update validator functions """ @@ -104,7 +116,7 @@ def filters(self): :func:`~cloudant.database.CouchDatabase.changes` For more details, see the `Filter functions documentation - `_. + `_. :returns: Dictionary containing filter function names and functions as key/value @@ -179,10 +191,10 @@ def st_indexes(self): Once the Cloudant Geo index is saved to the remote database, you can query the index with a GET request. To issue a request against the ``_geo`` endpoint, see the steps outlined in the `endpoint access - documentation `_. + `_ section. For more details, see the `Cloudant Geospatial - documentation `_. + documentation `_. :return: Dictionary containing Cloudant Geo names and index objects as key/value @@ -241,7 +253,7 @@ def rewrites(self): rewritten as ``/$DATABASE/_design/doc/_rewrite/new?k=v``. For more details on URL rewriting, see the `rewrite rules - documentation `_. :returns: List of dictionaries containing rewrite rules as key/value @@ -269,6 +281,20 @@ def indexes(self): """ return self.get('indexes') + def document_partition_url(self, partition_key): + """ + Retrieve the design document partition URL. + + :param str partition_key: Partition key. + :return: Design document partition URL. + :rtype: str + """ + return '/'.join(( + self._database.database_partition_url(partition_key), + '_design', + url_quote_plus(self['_id'][8:], safe='') + )) + def add_view(self, view_name, map_func, reduce_func=None, **kwargs): """ Appends a MapReduce view to the locally cached DesignDocument View @@ -686,7 +712,7 @@ def info(self): ddoc_info = self.r_session.get( '/'.join([self.document_url, '_info'])) ddoc_info.raise_for_status() - return ddoc_info.json() + return response_to_json_dict(ddoc_info) def search_info(self, search_index): """ @@ -698,4 +724,16 @@ def search_info(self, search_index): ddoc_search_info = self.r_session.get( '/'.join([self.document_url, '_search_info', search_index])) ddoc_search_info.raise_for_status() - return ddoc_search_info.json() + return response_to_json_dict(ddoc_search_info) + + def search_disk_size(self, search_index): + """ + Retrieves disk size information about a specified search index within + the design document, returns dictionary + + GET databasename/_design/{ddoc}/_search_disk_size/{search_index} + """ + ddoc_search_disk_size = self.r_session.get( + '/'.join([self.document_url, '_search_disk_size', search_index])) + ddoc_search_disk_size.raise_for_status() + return response_to_json_dict(ddoc_search_disk_size) diff --git a/src/cloudant/document.py b/src/cloudant/document.py index 500acc18..96267bc9 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright © 2015, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,11 +16,11 @@ API module/class for interacting with a document in a database. """ import json -import posixpath import requests from requests.exceptions import HTTPError from ._2to3 import url_quote, url_quote_plus +from ._common_util import response_to_json_dict, assert_document_type_id, assert_attachment_name from .error import CloudantDocumentException @@ -54,17 +54,19 @@ class Document(dict): :param database: A database instance used by the Document. Can be either a ``CouchDatabase`` or ``CloudantDatabase`` instance. :param str document_id: Optional document id used to identify the document. + :param str encoder: Optional JSON encoder object (extending json.JSONEncoder). + :param str decoder: Optional JSON decoder object (extending json.JSONDecoder). """ - def __init__(self, database, document_id=None): + def __init__(self, database, document_id=None, **kwargs): super(Document, self).__init__() self._client = database.client self._database = database self._database_host = self._client.server_url self._database_name = database.database_name - self._document_id = document_id - if self._document_id is not None: - self['_id'] = self._document_id - self.encoder = self._client.encoder + if document_id: + self['_id'] = document_id + self.encoder = kwargs.get('encoder') or self._client.encoder + self.decoder = kwargs.get('decoder') or json.JSONDecoder @property def r_session(self): @@ -82,24 +84,33 @@ def document_url(self): :returns: Document URL """ - if self._document_id is None: + if '_id' not in self or self['_id'] is None: return None # handle design document url - if self._document_id.startswith('_design/'): - return posixpath.join( + if self['_id'].startswith('_design/'): + return '/'.join(( self._database_host, url_quote_plus(self._database_name), '_design', - url_quote(self._document_id[8:], safe='') - ) + url_quote(self['_id'][8:], safe='') + )) + + # handle _local document url + if self['_id'].startswith('_local/'): + return '/'.join(( + self._database_host, + url_quote_plus(self._database_name), + '_local', + url_quote(self['_id'][7:], safe='') + )) # handle document url - return posixpath.join( + return '/'.join(( self._database_host, url_quote_plus(self._database_name), - url_quote(self._document_id, safe='') - ) + url_quote(self['_id'], safe='') + )) def exists(self): """ @@ -108,12 +119,14 @@ def exists(self): :returns: True if the document exists in the remote database, otherwise False """ - if self._document_id is None: + if '_id' not in self or self['_id'] is None: return False - else: - resp = self.r_session.head(self.document_url) - if resp.status_code not in [200, 404]: - resp.raise_for_status() + + assert_document_type_id(self['_id']) + + resp = self.r_session.head(self.document_url) + if resp.status_code not in [200, 404]: + resp.raise_for_status() return resp.status_code == 200 @@ -133,8 +146,6 @@ def create(self): updates the locally cached Document object with the ``_id`` and ``_rev`` returned as part of the successful response. """ - if self._document_id is not None: - self['_id'] = self._document_id # Ensure that an existing document will not be "updated" doc = dict(self) @@ -148,11 +159,9 @@ def create(self): data=json.dumps(doc, cls=self.encoder) ) resp.raise_for_status() - data = resp.json() - self._document_id = data['id'] + data = response_to_json_dict(resp) super(Document, self).__setitem__('_id', data['id']) super(Document, self).__setitem__('_rev', data['rev']) - return def fetch(self): """ @@ -163,10 +172,12 @@ def fetch(self): """ if self.document_url is None: raise CloudantDocumentException(101) + if '_id' in self: + assert_document_type_id(self['_id']) resp = self.r_session.get(self.document_url) resp.raise_for_status() self.clear() - self.update(resp.json()) + self.update(response_to_json_dict(resp, cls=self.decoder)) def save(self): """ @@ -188,7 +199,7 @@ def save(self): headers=headers ) put_resp.raise_for_status() - data = put_resp.json() + data = response_to_json_dict(put_resp) super(Document, self).__setitem__('_rev', data['rev']) return @@ -258,9 +269,10 @@ def _update_field(self, action, field, value, max_tries, tries=0): self.save() except requests.HTTPError as ex: if tries < max_tries and ex.response.status_code == 409: - return self._update_field( + self._update_field( action, field, value, max_tries, tries=tries+1) - raise + else: + raise def update_field(self, action, field, value, max_tries=10): """ @@ -310,14 +322,16 @@ def delete(self): if not self.get("_rev"): raise CloudantDocumentException(103) + assert_document_type_id(self['_id']) + del_resp = self.r_session.delete( self.document_url, params={"rev": self["_rev"]}, ) del_resp.raise_for_status() + _id = self['_id'] self.clear() - self.__setitem__('_id', self._document_id) - return + self['_id'] = _id def __enter__(self): """ @@ -333,32 +347,19 @@ def __enter__(self): except HTTPError as error: if error.response.status_code != 404: raise + except CloudantDocumentException as error: + if error.status_code != 101: + raise return self - def __exit__(self, *args): - """ - Support context like editing of document fields. Handles context exit - logic. Executes a Document.save() upon exit. - """ - self.save() - - def __setitem__(self, key, value): - """ - Sets the _document_id when setting the '_id' field. - The _document_id is used to construct the document url. + def __exit__(self, exc_type, exc_value, traceback): """ - if key == '_id': - self._document_id = value - super(Document, self).__setitem__(key, value) - - def __delitem__(self, key): + Support context like editing of document fields. Handles context exit + logic. Executes a `Document.save()` upon exit if no exception occurred. """ - Sets the _document_id to None when deleting the '_id' field. - """ - if key == '_id': - self._document_id = None - super(Document, self).__delitem__(key) + if exc_type is None: + self.save() def get_attachment( self, @@ -389,7 +390,8 @@ def get_attachment( """ # need latest rev self.fetch() - attachment_url = posixpath.join(self.document_url, attachment) + assert_attachment_name(attachment) + attachment_url = '/'.join((self.document_url, url_quote(attachment, safe=''))) if headers is None: headers = {'If-Match': self['_rev']} else: @@ -407,14 +409,14 @@ def get_attachment( attachment_type = 'binary' if write_to is not None: - if attachment_type == 'text' or attachment_type == 'json': + if attachment_type in ('text', 'json'): write_to.write(resp.text) else: write_to.write(resp.content) if attachment_type == 'text': return resp.text - elif attachment_type == 'json': - return resp.json() + if attachment_type == 'json': + return response_to_json_dict(resp) return resp.content @@ -432,7 +434,8 @@ def delete_attachment(self, attachment, headers=None): """ # need latest rev self.fetch() - attachment_url = posixpath.join(self.document_url, attachment) + assert_attachment_name(attachment) + attachment_url = '/'.join((self.document_url, attachment)) if headers is None: headers = {'If-Match': self['_rev']} else: @@ -443,7 +446,7 @@ def delete_attachment(self, attachment, headers=None): headers=headers ) resp.raise_for_status() - super(Document, self).__setitem__('_rev', resp.json()['rev']) + super(Document, self).__setitem__('_rev', response_to_json_dict(resp)['rev']) # Execute logic only if attachment metadata exists locally if self.get('_attachments'): # Remove the attachment metadata for the specified attachment @@ -453,7 +456,7 @@ def delete_attachment(self, attachment, headers=None): if not self['_attachments']: super(Document, self).__delitem__('_attachments') - return resp.json() + return response_to_json_dict(resp) def put_attachment(self, attachment, content_type, data, headers=None): """ @@ -473,7 +476,8 @@ def put_attachment(self, attachment, content_type, data, headers=None): """ # need latest rev self.fetch() - attachment_url = posixpath.join(self.document_url, attachment) + assert_attachment_name(attachment) + attachment_url = '/'.join((self.document_url, attachment)) if headers is None: headers = { 'If-Match': self['_rev'], @@ -490,4 +494,4 @@ def put_attachment(self, attachment, content_type, data, headers=None): ) resp.raise_for_status() self.fetch() - return resp.json() + return response_to_json_dict(resp) diff --git a/src/cloudant/error.py b/src/cloudant/error.py index a35345f2..348e1bf6 100644 --- a/src/cloudant/error.py +++ b/src/cloudant/error.py @@ -107,7 +107,13 @@ class CloudantDatabaseException(CloudantException): """ def __init__(self, code=100, *args): try: - msg = DATABASE[code].format(*args) + if code in DATABASE: + msg = DATABASE[code].format(*args) + elif isinstance(code, int): + msg = ' '.join(args) + else: + code = 100 + msg = DATABASE[code] except (KeyError, IndexError): code = 100 msg = DATABASE[code] diff --git a/src/cloudant/feed.py b/src/cloudant/feed.py index f4797a38..f038ebfe 100644 --- a/src/cloudant/feed.py +++ b/src/cloudant/feed.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2016 IBM. All rights reserved. +# Copyright (c) 2015, 2018 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ from ._2to3 import iteritems_, next_, unicode_, STRTYPE, NONETYPE from .error import CloudantArgumentError, CloudantFeedException -from ._common_util import feed_arg_types, TYPE_CONVERTERS +from ._common_util import ANY_ARG, ANY_TYPE, feed_arg_types, TYPE_CONVERTERS class Feed(object): """ @@ -100,7 +100,7 @@ def _translate(self, options): if isinstance(val, STRTYPE): translation[key] = val elif not isinstance(val, NONETYPE): - arg_converter = TYPE_CONVERTERS.get(type(val)) + arg_converter = TYPE_CONVERTERS.get(type(val), json.dumps) translation[key] = arg_converter(val) except Exception as ex: raise CloudantArgumentError(115, key, ex) @@ -111,12 +111,19 @@ def _validate(self, key, val, arg_types): Ensures that the key and the value are valid arguments to be used with the feed. """ - if key not in arg_types: - raise CloudantArgumentError(116, key) - if (not isinstance(val, arg_types[key]) or - (isinstance(val, bool) and int in arg_types[key])): - raise CloudantArgumentError(117, key, arg_types[key]) - if isinstance(val, int) and val <= 0 and not isinstance(val, bool): + if key in arg_types: + arg_type = arg_types[key] + else: + if ANY_ARG not in arg_types: + raise CloudantArgumentError(116, key) + arg_type = arg_types[ANY_ARG] + + if arg_type == ANY_TYPE: + return + if (not isinstance(val, arg_type) or + (isinstance(val, bool) and int in arg_type)): + raise CloudantArgumentError(117, key, arg_type) + if isinstance(val, int) and val < 0 and not isinstance(val, bool): raise CloudantArgumentError(118, key, val) if key == 'feed': valid_vals = ('continuous', 'normal', 'longpoll') @@ -137,7 +144,7 @@ def __next__(self): """ Provides Python3 compatibility. """ - return self.next() + return self.next() # pylint: disable=not-callable def next(self): """ @@ -164,8 +171,7 @@ def _process_data(self, line): skip = False if self._raw_data: return skip, line - else: - line = unicode_(line) + line = unicode_(line) if not line: if (self._options.get('heartbeat', False) and self._options.get('feed') in ('continuous', 'longpoll') and diff --git a/src/cloudant/index.py b/src/cloudant/index.py index e44fd258..c66c7ac2 100644 --- a/src/cloudant/index.py +++ b/src/cloudant/index.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2019 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,7 +16,6 @@ API module for managing/viewing query indexes. """ -import posixpath import json from ._2to3 import STRTYPE, iteritems_ @@ -24,6 +23,7 @@ from ._common_util import TEXT_INDEX_TYPE from ._common_util import SPECIAL_INDEX_TYPE from ._common_util import TEXT_INDEX_ARGS +from ._common_util import response_to_json_dict from .error import CloudantArgumentError, CloudantIndexException class Index(object): @@ -40,18 +40,21 @@ class Index(object): Index. :param str design_document_id: Optional identifier of the design document. :param str name: Optional name of the index. + :param bool partitioned: Optional. Create as a partitioned index. Defaults + to ``False`` for both partitioned and non-partitioned databases. :param kwargs: Options used to construct the index definition for the purposes of index creation. For more details on valid options See :func:`~cloudant.database.CloudantDatabase.create_query_index`. """ - def __init__(self, database, design_document_id=None, name=None, **kwargs): + def __init__(self, database, design_document_id=None, name=None, partitioned=None, **kwargs): self._database = database self._r_session = self._database.r_session self._ddoc_id = design_document_id self._name = name self._type = JSON_INDEX_TYPE self._def = kwargs + self._partitioned = partitioned @property def index_url(self): @@ -60,7 +63,7 @@ def index_url(self): :returns: Index URL """ - return posixpath.join(self._database.database_url, '_index') + return '/'.join((self._database.database_url, '_index')) @property def design_document_id(self): @@ -100,6 +103,17 @@ def definition(self): """ return self._def + @property + def partitioned(self): + """ + Check if this index is partitioned. + + :return: ``True`` if index is partitioned, else ``False``. + :rtype: bool + """ + + return self._partitioned + def as_a_dict(self): """ Displays the index as a dictionary. This includes the design document @@ -114,6 +128,9 @@ def as_a_dict(self): 'def': self._def } + if self._partitioned: + index_dict['partitioned'] = True + return index_dict def create(self): @@ -137,6 +154,9 @@ def create(self): self._def_check() payload['index'] = self._def + if self._partitioned is not None: + payload['partitioned'] = bool(self._partitioned) + headers = {'Content-Type': 'application/json'} resp = self._r_session.post( self.index_url, @@ -144,9 +164,8 @@ def create(self): headers=headers ) resp.raise_for_status() - self._ddoc_id = resp.json()['id'] - self._name = resp.json()['name'] - return + self._ddoc_id = response_to_json_dict(resp)['id'] + self._name = response_to_json_dict(resp)['name'] def _def_check(self): """ @@ -166,10 +185,9 @@ def delete(self): ddoc_id = self._ddoc_id if ddoc_id.startswith('_design/'): ddoc_id = ddoc_id[8:] - url = posixpath.join(self.index_url, ddoc_id, self._type, self._name) + url = '/'.join((self.index_url, ddoc_id, self._type, self._name)) resp = self._r_session.delete(url) resp.raise_for_status() - return class TextIndex(Index): """ diff --git a/src/cloudant/query.py b/src/cloudant/query.py index a72ec6a7..2362f80d 100644 --- a/src/cloudant/query.py +++ b/src/cloudant/query.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2019 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,7 +16,6 @@ API module for composing and executing Cloudant queries. """ -import posixpath import json import contextlib @@ -24,6 +23,7 @@ from .result import QueryResult from .error import CloudantArgumentError from ._common_util import QUERY_ARG_TYPES +from ._common_util import response_to_json_dict class Query(dict): """ @@ -69,13 +69,12 @@ class Query(dict): :param CloudantDatabase database: A Cloudant database instance used by the Query. :param str bookmark: A string that enables you to specify which page of - results you require. Only valid for queries using indexes of type - *text*. + results you require. :param list fields: A list of fields to be returned by the query. :param int limit: Maximum number of results returned. :param int r: Read quorum needed for the result. Each document is read from at least 'r' number of replicas before it is returned in the results. - :param str selector: Dictionary object describing criteria used to select + :param dict selector: Dictionary object describing criteria used to select documents. :param int skip: Skip the first 'n' results, where 'n' is the value specified. @@ -87,13 +86,18 @@ class Query(dict): :param str use_index: Identifies a specific index for the query to run against, rather than using the Cloudant Query algorithm which finds what it believes to be the best index. + :param str partition_key: Optional. Specify a query partition key. Defaults + to ``None`` resulting in global queries. """ def __init__(self, database, **kwargs): super(Query, self).__init__() self._database = database + self._partition_key = kwargs.pop('partition_key', None) self._r_session = self._database.r_session self._encoder = self._database.client.encoder + if kwargs.get('fields', True) is None: + del kwargs['fields'] # delete `None` fields kwarg if kwargs: super(Query, self).update(kwargs) self.result = QueryResult(self) @@ -105,7 +109,13 @@ def url(self): :returns: Query URL """ - return posixpath.join(self._database.database_url, '_find') + if self._partition_key: + base_url = self._database.database_partition_url( + self._partition_key) + else: + base_url = self._database.database_url + + return base_url + '/_find' def __call__(self, **kwargs): """ @@ -130,14 +140,13 @@ def __call__(self, **kwargs): and set ``raw_result=True`` instead. :param str bookmark: A string that enables you to specify which page of - results you require. Only valid for queries using indexes of type - *text*. + results you require. :param list fields: A list of fields to be returned by the query. :param int limit: Maximum number of results returned. :param int r: Read quorum needed for the result. Each document is read from at least 'r' number of replicas before it is returned in the results. - :param str selector: Dictionary object describing criteria used to + :param dict selector: Dictionary object describing criteria used to select documents. :param int skip: Skip the first 'n' results, where 'n' is the value specified. @@ -173,7 +182,7 @@ def __call__(self, **kwargs): data=json.dumps(data, cls=self._encoder) ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) @contextlib.contextmanager def custom_result(self, **options): @@ -193,15 +202,14 @@ def custom_result(self, **options): data = rslt[100:200] :param str bookmark: A string that enables you to specify which page of - results you require. Only valid for queries using indexes of type - *text*. + results you require. :param list fields: A list of fields to be returned by the query. :param int page_size: Sets the page size for result iteration. Default is 100. :param int r: Read quorum needed for the result. Each document is read from at least 'r' number of replicas before it is returned in the results. - :param str selector: Dictionary object describing criteria used to + :param dict selector: Dictionary object describing criteria used to select documents. :param list sort: A list of fields to sort by. Optionally the list can contain elements that are single member dictionary structures that diff --git a/src/cloudant/replicator.py b/src/cloudant/replicator.py index 06e91b0a..f1b44643 100644 --- a/src/cloudant/replicator.py +++ b/src/cloudant/replicator.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,8 +18,11 @@ import uuid +from requests.exceptions import HTTPError + from .error import CloudantReplicatorException, CloudantClientException from .document import Document +from .scheduler import Scheduler class Replicator(object): """ @@ -34,6 +37,7 @@ class Replicator(object): def __init__(self, client): repl_db = '_replicator' + self.client = client try: self.database = client[repl_db] except Exception: @@ -50,17 +54,8 @@ def create_replication(self, source_db=None, target_db=None, ``CouchDatabase`` or ``CloudantDatabase`` instance. :param str repl_id: Optional replication id. Generated internally if not explicitly set. - :param source: Optional ``str`` or ``dict`` representing the source - database, along with authentication info, if any. Composed - internally if not explicitly set and not in CouchDB Admin Party - mode. - :param target: Optional ``str`` or ``dict`` representing the - target database, possibly including authentication info. Composed - internally if not explicitly set and not in CouchDB Admin Party - mode. :param dict user_ctx: Optional user to act as. Composed internally - if not explicitly set and not in CouchDB Admin Party - mode. + if not explicitly set. :param bool create_target: Specifies whether or not to create the target, if it does not already exist. :param bool continuous: If set to True then the replication will be @@ -68,33 +63,50 @@ def create_replication(self, source_db=None, target_db=None, :returns: Replication document as a Document instance """ + if source_db is None: + raise CloudantReplicatorException(101) + + if target_db is None: + raise CloudantReplicatorException(102) data = dict( _id=repl_id if repl_id else str(uuid.uuid4()), **kwargs ) - if not data.get('source'): - if source_db is None: - raise CloudantReplicatorException(101) - data['source'] = {'url': source_db.database_url} - if not source_db.admin_party: - data['source'].update( - {'headers': {'Authorization': source_db.creds['basic_auth']}} - ) - - if not data.get('target'): - if target_db is None: - raise CloudantReplicatorException(102) - data['target'] = {'url': target_db.database_url} - if not target_db.admin_party: - data['target'].update( - {'headers': {'Authorization': target_db.creds['basic_auth']}} - ) - - if not data.get('user_ctx'): - if not target_db.admin_party: - data['user_ctx'] = self.database.creds['user_ctx'] + # replication source + + data['source'] = {'url': source_db.database_url} + if source_db.admin_party: + pass # no credentials required + elif source_db.client.is_iam_authenticated: + data['source'].update({'auth': { + 'iam': {'api_key': source_db.client.r_session.get_api_key} + }}) + else: + data['source'].update({'headers': { + 'Authorization': source_db.creds['basic_auth'] + }}) + + # replication target + + data['target'] = {'url': target_db.database_url} + if target_db.admin_party: + pass # no credentials required + elif target_db.client.is_iam_authenticated: + data['target'].update({'auth': { + 'iam': {'api_key': target_db.client.r_session.get_api_key} + }}) + else: + data['target'].update({'headers': { + 'Authorization': target_db.creds['basic_auth'] + }}) + + # add user context delegation + + if not data.get('user_ctx') and self.database.creds and \ + self.database.creds.get('user_ctx'): + data['user_ctx'] = self.database.creds['user_ctx'] return self.database.create_document(data, throw_on_exists=True) @@ -125,12 +137,20 @@ def replication_state(self, repl_id): :returns: Replication state as a ``str`` """ - try: - repl_doc = self.database[repl_id] - except KeyError: - raise CloudantReplicatorException(404, repl_id) - repl_doc.fetch() - return repl_doc.get('_replication_state') + if "scheduler" in self.client.features(): + try: + repl_doc = Scheduler(self.client).get_doc(repl_id) + except HTTPError as err: + raise CloudantReplicatorException(err.response.status_code, repl_id) + state = repl_doc['state'] + else: + try: + repl_doc = self.database[repl_id] + except KeyError: + raise CloudantReplicatorException(404, repl_id) + repl_doc.fetch() + state = repl_doc.get('_replication_state') + return state def follow_replication(self, repl_id): """ @@ -153,12 +173,19 @@ def update_state(): """ Retrieves the replication state. """ - try: - arepl_doc = self.database[repl_id] - arepl_doc.fetch() - return arepl_doc, arepl_doc.get('_replication_state') - except KeyError: - return None, None + if "scheduler" in self.client.features(): + try: + arepl_doc = Scheduler(self.client).get_doc(repl_id) + return arepl_doc, arepl_doc['state'] + except HTTPError: + return None, None + else: + try: + arepl_doc = self.database[repl_id] + arepl_doc.fetch() + return arepl_doc, arepl_doc.get('_replication_state') + except KeyError: + return None, None while True: # Make sure we fetch the state up front, just in case it moves @@ -166,8 +193,13 @@ def update_state(): repl_doc, state = update_state() if repl_doc: yield repl_doc - if state is not None and state in ['error', 'completed']: - raise StopIteration + # This is a little awkward, since 2.1 the terminal states are + # "failed" and "completed", so those should be the exit states, but + # for backwards compatibility with older versions "error" is also + # needed. The code has always exited for "error" state even long + # after 2.1 was available so that behaviour is retained. + if state is not None and state in ['error', 'failed', 'completed']: + return # Now listen on changes feed for the state for change in self.database.changes(): @@ -175,8 +207,9 @@ def update_state(): repl_doc, state = update_state() if repl_doc is not None: yield repl_doc - if state is not None and state in ['error', 'completed']: - raise StopIteration + # See note about these states + if state is not None and state in ['error', 'failed', 'completed']: + return def stop_replication(self, repl_id): """ diff --git a/src/cloudant/result.py b/src/cloudant/result.py index 58aa743f..319d7268 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,8 @@ """ API module for interacting with result collections. """ +from collections import deque +from functools import partial from ._2to3 import STRTYPE from .error import ResultException from ._common_util import py_to_couch_validate, type_or_none @@ -150,14 +152,24 @@ class Result(object): :param bool reduce: True to use the reduce function, false otherwise. :param int skip: Skip this number of rows from the start. Not valid when used with key iteration. - :param str stale: Allow the results from a stale view to be used. This - makes the request return immediately, even if the view has not been - completely built yet. If this parameter is not given, a response is - returned only after the view has been built. + :param bool stable: Whether or not the view results should be returned from + a "stable" set of shards. + :param str stale: Allow the results from a stale view to be used. This makes + the request return immediately, even if the view has not been completely + built yet. If this parameter is not given, a response is returned only + after the view has been built. Note that this parameter is deprecated + and the appropriate combination of `stable` and `update` should be used + instead. :param startkey: Return records starting with the specified key. Not valid when used with key access and key slicing. :param str startkey_docid: Return records starting with the specified document ID. + :param str update: Determine whether the view in question should be + updated prior to or after responding to the user. Valid values are: + false: return results before updating the view; true: Return results + after updating the view; lazy: Return the view results without + waiting for an update, but update them immediately after the request. + """ def __init__(self, method_ref, **options): self.options = options @@ -201,7 +213,7 @@ def __getitem__(self, arg): data = None if isinstance(arg, int): data = self._handle_result_by_index(arg) - elif isinstance(arg, STRTYPE) or isinstance(arg, list): + elif isinstance(arg, (STRTYPE, list)): data = self._handle_result_by_key(arg) elif isinstance(arg, ResultByKey): data = self._handle_result_by_key(arg()) @@ -258,13 +270,13 @@ def _handle_result_by_idx_slice(self, idx_slice): start = idx_slice.start stop = idx_slice.stop data = None - if (start is not None and stop is not None and - start >= 0 and stop >= 0 and start < stop): + # start and stop cannot be None and both must be greater than 0 + if all(i is not None and i >= 0 for i in [start, stop]) and start < stop: if limit is not None: if start >= limit: # Result is out of range return dict() - elif stop > limit: + if stop > limit: # Ensure that slice does not extend past original limit return self._ref(skip=skip+start, limit=limit-start, **opts) data = self._ref(skip=skip+start, limit=stop-start, **opts) @@ -317,38 +329,82 @@ def _handle_result_by_key_slice(self, key_slice): def __iter__(self): """ Provides iteration support, primarily for large data collections. - The iterator uses the ``skip`` and ``limit`` options to consume - data in chunks controlled by the ``page_size`` option. It retrieves - a batch of data from the result collection and then yields each - element. + The iterator uses the ``startkey``, ``startkey_docid``, and ``limit`` + options to consume data in chunks controlled by the ``page_size`` + option. It retrieves a batch of data from the result collection + and then yields each element. See :class:`~cloudant.result.Result` for Result iteration examples. :returns: Iterable data sequence """ - invalid_options = ('skip', 'limit') + invalid_options = ('limit', ) if any(x in invalid_options for x in self.options): raise ResultException(103, invalid_options, self.options) try: - if int(self._page_size) <= 0: + self._page_size = int(self._page_size) + if self._page_size <= 0: raise ResultException(104, self._page_size) except ValueError: raise ResultException(104, self._page_size) - skip = 0 + init_opts = { + 'skip': self.options.pop('skip', None), + 'startkey': self.options.pop('startkey', None) + } + + self._call = partial(self._ref, #pylint: disable=attribute-defined-outside-init + limit=self._real_page_size, + **self.options) + + response = self._call(**{k: v + for k, v + in init_opts.items() + if v is not None}) + + return self._iterator(response) + + @property + def _real_page_size(self): + ''' + In views we paginate with N+1 items per page. + https://docs.couchdb.org/en/stable/ddocs/views/pagination.html#paging-alternate-method + ''' + return self._page_size + 1 + + def _iterator(self, response): + ''' + Iterate through view data. + ''' + while True: - response = self._ref( - limit=int(self._page_size), - skip=skip, - **self.options - ) - result = self._parse_data(response) - skip += int(self._page_size) - if len(result) > 0: - for row in result: - yield row + result = deque(self._parse_data(response)) + del response + if result: + doc_count = len(result) + last = result.pop() + while result: + yield result.popleft() + + # We expect doc_count = self._page_size + 1 results, if + # we have self._page_size or less it means we are on the + # last page and need to return the last result. + if doc_count < self._real_page_size: + yield last + break del result + + # if we are in a view, keys could be duplicate so we + # need to start from the right docid + last_doc_id = last.get('id') + if last_doc_id is not None: + response = self._call(startkey=last['key'], + startkey_docid=last_doc_id) + # reduce result keys are unique by definition + else: + response = self._call(startkey=last['key']) + else: break @@ -359,6 +415,18 @@ def _parse_data(self, data): """ return data.get('rows', []) + def all(self): + """ + Retrieve all results. + + Specifying a ``limit`` parameter in the ``Result`` constructor will + limit the number of documents returned. Be aware that the ``page_size`` + parameter is not honoured. + + :return: results data as list in JSON format. + """ + return self[:] + class QueryResult(Result): """ Provides a index key accessible, sliceable and iterable interface to query @@ -421,15 +489,14 @@ class QueryResult(Result): :param query: A reference to the query callable that returns the JSON content result to be wrapped. :param str bookmark: A string that enables you to specify which page of - results you require. Only valid for queries using indexes of type - *text*. + results you require. :param list fields: A list of fields to be returned by the query. :param int page_size: Sets the page size for result iteration. Default is 100. :param int r: Read quorum needed for the result. Each document is read from at least 'r' number of replicas before it is returned in the results. - :param str selector: Dictionary object describing criteria used to + :param dict selector: Dictionary object describing criteria used to select documents. :param list sort: A list of fields to sort by. Optionally the list can contain elements that are single member dictionary structures that @@ -476,8 +543,8 @@ def __getitem__(self, arg): type_or_none(int, arg.start) and type_or_none(int, arg.stop))): return super(QueryResult, self).__getitem__(arg) - else: - raise ResultException(101, arg) + + raise ResultException(101, arg) def _parse_data(self, data): """ @@ -485,3 +552,32 @@ def _parse_data(self, data): query result JSON response content """ return data.get('docs', []) + + @property + def _real_page_size(self): + ''' + During queries iteration page size is user-specified + ''' + return self._page_size + + def _iterator(self, response): + ''' + Iterate through query data. + ''' + + while True: + result = self._parse_data(response) + bookmark = response.get('bookmark') + if result: + for row in result: + yield row + + del result + + if not bookmark: + break + + response = self._call(bookmark=bookmark) + + else: + break diff --git a/src/cloudant/scheduler.py b/src/cloudant/scheduler.py new file mode 100644 index 00000000..012272fe --- /dev/null +++ b/src/cloudant/scheduler.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# Copyright (C) 2018 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +API module for interacting with scheduler endpoints +""" + +from ._common_util import response_to_json_dict + +class Scheduler(object): + """ + API for retrieving scheduler jobs and documents. + + :param client: Client instance used by the database. Can either be a + ``CouchDB`` or ``Cloudant`` client instance. + """ + + def __init__(self, client): + self._client = client + self._r_session = client.r_session + self._scheduler = '/'.join([self._client.server_url, '_scheduler']) + + def list_docs(self, limit=None, skip=None): + """ + Lists replication documents. Includes information + about all the documents, even in completed and failed + states. For each document it returns the document ID, the + database, the replication ID, source and target, and other + information. + + :param limit: How many results to return. + :param skip: How many result to skip starting at the beginning, if ordered by document ID. + """ + params = dict() + if limit is not None: + params["limit"] = limit + if skip is not None: + params["skip"] = skip + resp = self._r_session.get('/'.join([self._scheduler, 'docs']), params=params) + resp.raise_for_status() + return response_to_json_dict(resp) + + def get_doc(self, doc_id): + """ + Get replication document state for a given replication document ID. + """ + resp = self._r_session.get('/'.join([self._scheduler, 'docs', '_replicator', doc_id])) + resp.raise_for_status() + return response_to_json_dict(resp) + + + def list_jobs(self, limit=None, skip=None): + """ + Lists replication jobs. Includes replications created via + /_replicate endpoint as well as those created from replication + documents. Does not include replications which have completed + or have failed to start because replication documents were + malformed. Each job description will include source and target + information, replication id, a history of recent event, and a + few other things. + + :param limit: How many results to return. + :param skip: How many result to skip starting at the beginning, if ordered by document ID. + """ + params = dict() + if limit is not None: + params["limit"] = limit + if skip is not None: + params["skip"] = skip + resp = self._r_session.get('/'.join([self._scheduler, 'jobs']), params=params) + resp.raise_for_status() + return response_to_json_dict(resp) diff --git a/src/cloudant/security_document.py b/src/cloudant/security_document.py index 22b0f79c..87b77ca3 100644 --- a/src/cloudant/security_document.py +++ b/src/cloudant/security_document.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import json from ._2to3 import url_quote_plus +from ._common_util import response_to_json_dict class SecurityDocument(dict): """ @@ -101,7 +102,7 @@ def fetch(self): resp = self.r_session.get(self.document_url) resp.raise_for_status() self.clear() - self.update(resp.json()) + self.update(response_to_json_dict(resp)) def save(self): """ diff --git a/src/cloudant/view.py b/src/cloudant/view.py index 37449e8d..7b76f2a5 100644 --- a/src/cloudant/view.py +++ b/src/cloudant/view.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2019 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,10 +16,9 @@ API module for interacting with a view in a design document. """ import contextlib -import posixpath from ._2to3 import STRTYPE -from ._common_util import codify, get_docs +from ._common_util import codify, get_docs, response_to_json_dict from .result import Result from .error import CloudantArgumentError, CloudantViewException @@ -88,6 +87,8 @@ class View(dict): :param str view_name: Name used in part to identify the view. :param str map_func: Optional Javascript map function. :param str reduce_func: Optional Javascript reduce function. + :param str partition_key: Optional. Specify a view partition key. Defaults + to ``None`` resulting in global queries. """ def __init__( self, @@ -95,6 +96,7 @@ def __init__( view_name, map_func=None, reduce_func=None, + partition_key=None, **kwargs ): super(View, self).__init__() @@ -105,6 +107,7 @@ def __init__( self['map'] = codify(map_func) if reduce_func is not None: self['reduce'] = codify(reduce_func) + self._partition_key = partition_key self.update(kwargs) self.result = Result(self) @@ -168,11 +171,17 @@ def url(self): :returns: View URL """ - return posixpath.join( - self.design_doc.document_url, + if self._partition_key: + base_url = self.design_doc.document_partition_url( + self._partition_key) + else: + base_url = self.design_doc.document_url + + return '/'.join(( + base_url, '_view', self.view_name - ) + )) def __call__(self, **kwargs): """ @@ -228,7 +237,7 @@ def __call__(self, **kwargs): self.url, self.design_doc.encoder, **kwargs) - return resp.json() + return response_to_json_dict(resp) @contextlib.contextmanager def custom_result(self, **options): diff --git a/test-requirements.txt b/test-requirements.txt index 098c70e5..8d95c01d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,6 @@ mock==1.3.0 nose sphinx -pylint +sphinx_rtd_theme +pylint==2.5.2 flaky diff --git a/tests/integration/replicator_test.py b/tests/integration/replicator_test.py index c0542e0f..4fc3e759 100644 --- a/tests/integration/replicator_test.py +++ b/tests/integration/replicator_test.py @@ -190,57 +190,6 @@ def test_follow_replication(self): self.assertTrue(len(updates) > 0) self.assertEqual(updates[-1]['_replication_state'], 'completed') - @unittest.skip("Doesn't reliably get into error state on couch side.") - def test_follow_replication_with_errors(self): - """ - _test_follow_replication_with_errors_ - - Test to make sure that we exit the follow loop when we submit - a bad replication. - - """ - dbsource = unicode_("test_follow_replication_source_error_{}".format( - unicode_(uuid.uuid4()))) - dbtarget = unicode_("test_follow_replication_target_error_{}".format( - unicode_(uuid.uuid4()))) - - self.dbs = [dbsource, dbtarget] - - with cloudant(self.user, self.passwd, account=self.user) as c: - dbs = c.create_database(dbsource) - dbt = c.create_database(dbtarget) - - doc1 = dbs.create_document( - {"_id": "doc1", "testing": "document 1"} - ) - doc2 = dbs.create_document( - {"_id": "doc2", "testing": "document 1"} - ) - doc3 = dbs.create_document( - {"_id": "doc3", "testing": "document 1"} - ) - - replicator = Replicator(c) - repl_id = unicode_("test_follow_replication_{}".format( - unicode_(uuid.uuid4()))) - self.replication_ids.append(repl_id) - - ret = replicator.create_replication( - source_db=dbs, - target_db=dbt, - # Deliberately override these good params with bad params - source=dbsource + "foo", - target=dbtarget + "foo", - repl_id=repl_id, - continuous=False, - ) - updates = [ - update for update in replicator.follow_replication(repl_id) - ] - self.assertTrue(len(updates) > 0) - self.assertEqual(updates[-1]['_replication_state'], 'error') - - def test_replication_state(self): """ _test_replication_state_ diff --git a/tests/unit/auth_renewal_tests.py b/tests/unit/auth_renewal_tests.py index 799783d3..e32f7459 100644 --- a/tests/unit/auth_renewal_tests.py +++ b/tests/unit/auth_renewal_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,19 +18,22 @@ See configuration options for environment variables in unit_t_db_base module docstring. """ -import unittest import os -import requests import time +import unittest -from cloudant._common_util import InfiniteSession +import requests +from cloudant._client_session import CookieSession +from nose.plugins.attrib import attr + +from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase -from .unit_t_db_base import UnitTestDbBase +@attr(db=['cloudant','couch']) @unittest.skipIf(os.environ.get('ADMIN_PARTY') == 'true', 'Skipping - Admin Party mode') class AuthRenewalTests(UnitTestDbBase): """ - Auto renewal tests primarily testing the InfiniteSession functionality + Auto renewal tests primarily testing the CookieSession functionality """ def setUp(self): @@ -44,7 +47,8 @@ def tearDown(self): Override UnitTestDbBase.tearDown() with no tear down """ pass - + + @skip_if_not_cookie_auth def test_client_db_doc_stack_success(self): """ Ensure that auto renewal of cookie auth happens as expected and applies @@ -62,10 +66,10 @@ def test_client_db_doc_stack_success(self): db_2_auth_session = db_2.r_session.cookies.get('AuthSession') doc_auth_session = doc.r_session.cookies.get('AuthSession') - self.assertIsInstance(self.client.r_session, InfiniteSession) - self.assertIsInstance(db.r_session, InfiniteSession) - self.assertIsInstance(db_2.r_session, InfiniteSession) - self.assertIsInstance(doc.r_session, InfiniteSession) + self.assertIsInstance(self.client.r_session, CookieSession) + self.assertIsInstance(db.r_session, CookieSession) + self.assertIsInstance(db_2.r_session, CookieSession) + self.assertIsInstance(doc.r_session, CookieSession) self.assertIsNotNone(auth_session) self.assertTrue( auth_session == @@ -109,6 +113,7 @@ def test_client_db_doc_stack_success(self): self.client.disconnect() del self.client + @skip_if_not_cookie_auth def test_client_db_doc_stack_failure(self): """ Ensure that when the regular requests.Session is used that diff --git a/tests/unit/changes_tests.py b/tests/unit/changes_tests.py index be78c520..3df0cef3 100644 --- a/tests/unit/changes_tests.py +++ b/tests/unit/changes_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,20 +16,23 @@ Unit tests for _changes feed """ -import unittest -from requests import Session import json import os +import unittest -from cloudant.feed import Feed -from cloudant.document import Document -from cloudant.design_document import DesignDocument -from cloudant.error import CloudantArgumentError, CloudantException from cloudant._2to3 import unicode_ +from cloudant.design_document import DesignDocument +from cloudant.document import Document +from cloudant.error import CloudantArgumentError +from cloudant.feed import Feed +from nose.plugins.attrib import attr +from requests import Session from .unit_t_db_base import UnitTestDbBase from .. import BYTETYPE + +@attr(db=['cloudant','couch']) class ChangesTests(UnitTestDbBase): """ _changes feed unit tests @@ -41,7 +44,6 @@ def setUp(self): """ super(ChangesTests, self).setUp() self.db_set_up() - self.cloudant_test = os.environ.get('RUN_CLOUDANT_TESTS') is not None def tearDown(self): """ @@ -100,12 +102,11 @@ def test_get_raw_content(self): self.assertIsInstance(raw_line, BYTETYPE) raw_content.append(raw_line) changes = json.loads(''.join([unicode_(x) for x in raw_content])) - if self.cloudant_test: - self.assertSetEqual( - set(changes.keys()), set(['results', 'last_seq', 'pending'])) - else: + if self.is_couchdb_1x_version() is True: self.assertSetEqual( set(changes.keys()), set(['results', 'last_seq'])) + else: + self.assertSetEqual(set(changes.keys()), set(['results', 'last_seq', 'pending'])) results = list() for result in changes['results']: self.assertSetEqual(set(result.keys()), set(['seq', 'changes', 'id'])) @@ -223,12 +224,10 @@ def test_get_raw_feed_with_heartbeat(self): def test_get_feed_descending(self): """ - Test getting content back for a descending feed. When testing with - Cloudant the sequence identifier is in the form of - -. Often times the number prefix sorts - as expected when using descending but sometimes the number prefix is - repeated. In these cases the check is to see if the following random - character sequence suffix is longer than its predecessor. + Test getting content back for a descending feed. When testing, the sequence + identifier is in the form of -. Often times + the number prefix sorts as expected when using descending but sometimes the + number prefix is repeated. """ self.populate_db_with_documents(50) feed = Feed(self.db, descending=True) @@ -236,16 +235,15 @@ def test_get_feed_descending(self): last_seq = None for change in feed: if last_seq: - if self.cloudant_test: + if self.is_couchdb_1x_version() is True: + self.assertTrue(change['seq'] < last_seq) + else: current = int(change['seq'][0: change['seq'].find('-')]) last = int(last_seq[0:last_seq.find('-')]) try: self.assertTrue(current < last) except AssertionError: self.assertEqual(current, last) - self.assertTrue(len(change['seq']) > len(last_seq)) - else: - self.assertTrue(change['seq'] < last_seq) seq_list.append(change['seq']) last_seq = change['seq'] self.assertEqual(len(seq_list), 50) @@ -303,8 +301,6 @@ def test_get_feed_using_style_all_docs(self): changes = list() for change in feed: self.assertSetEqual(set(change.keys()), set(['seq', 'changes', 'id'])) - if not self.cloudant_test: - self.assertEqual(len(change['changes']), 2) changes.append(change) expected = set(['julia000', 'julia001', 'julia002']) self.assertSetEqual(set([x['id'] for x in changes]), expected) @@ -353,6 +349,20 @@ def test_get_feed_using_since_now(self): expected = set(['julia003', 'julia004', 'julia005']) self.assertSetEqual(set([x['id'] for x in changes]), expected) + def test_get_feed_using_since_zero(self): + """ + Test getting content back for a feed using since set to zero + """ + self.populate_db_with_documents(3) + feed = Feed(self.db, since=0) + changes = list() + for change in feed: + self.assertSetEqual(set(change.keys()), {'seq', 'changes', 'id'}) + changes.append(change) + expected = set(['julia{0:03d}'.format(i) for i in range(3)]) + self.assertSetEqual(set([x['id'] for x in changes]), expected) + self.assertTrue(str(feed.last_seq).startswith('3')) + def test_get_feed_using_timeout(self): """ Test getting content back for a feed using timeout @@ -438,8 +448,6 @@ def test_get_feed_using_conflicts_false(self): self.assertSetEqual(set([x['id'] for x in changes]), expected) self.assertTrue(str(feed.last_seq).startswith('3')) - @unittest.skipIf(os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping since _doc_ids filter is not supported on all Cloudant clusters') def test_get_feed_using_doc_ids(self): """ Test getting content back for a feed using doc_ids @@ -455,14 +463,20 @@ def test_get_feed_using_doc_ids(self): self.assertSetEqual(set([x['id'] for x in changes]), expected) self.assertTrue(str(feed.last_seq).startswith('100')) - def test_invalid_argument(self): + def test_get_feed_with_custom_filter_query_params(self): """ - Test that an invalid argument is caught and an exception is raised + Test using feed with custom filter query parameters. """ - feed = Feed(self.db, foo='bar') - with self.assertRaises(CloudantArgumentError) as cm: - invalid_feed = [x for x in feed] - self.assertEqual(str(cm.exception), 'Invalid argument foo') + feed = Feed( + self.db, + filter='mailbox/new_mail', + foo='bar', # query parameters to a custom filter + include_docs=False + ) + params = feed._translate(feed._options) + self.assertEqual(params['filter'], 'mailbox/new_mail') + self.assertEqual(params['foo'], 'bar') + self.assertEqual(params['include_docs'], 'false') def test_invalid_argument_type(self): """ @@ -506,7 +520,7 @@ def test_invalid_style_value(self): with self.assertRaises(CloudantArgumentError) as cm: invalid_feed = [x for x in feed] self.assertEqual( - str(cm.exception), + str(cm.exception), 'Invalid value (foo) for style option. Must be main_only, or all_docs.') if __name__ == '__main__': diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index 797c90b9..35139168 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2016, 2017 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,25 +20,31 @@ """ -import unittest -import requests -import json import base64 -import sys -import os import datetime +import json +import os +import sys +import unittest +from time import sleep +from urllib.parse import urlparse -from requests import ConnectTimeout - -from cloudant import cloudant, couchdb, couchdb_admin_party +import mock +import requests +from cloudant import cloudant, cloudant_bluemix, couchdb, couchdb_admin_party +from cloudant._client_session import BasicSession, CookieSession from cloudant.client import Cloudant, CouchDB -from cloudant.error import CloudantArgumentError, CloudantClientException +from cloudant.database import CloudantDatabase +from cloudant.error import (CloudantArgumentError, CloudantClientException, + CloudantDatabaseException) from cloudant.feed import Feed, InfiniteFeed -from cloudant._common_util import InfiniteSession +from nose.plugins.attrib import attr +from requests import ConnectTimeout, HTTPError -from .unit_t_db_base import UnitTestDbBase +from .unit_t_db_base import skip_if_iam, skip_if_not_cookie_auth, UnitTestDbBase from .. import bytes_, str_ + class CloudantClientExceptionTests(unittest.TestCase): """ Ensure CloudantClientException functions as expected. @@ -83,10 +89,10 @@ class ClientTests(UnitTestDbBase): """ @unittest.skipIf( - (os.environ.get('RUN_CLOUDANT_TESTS') is not None or - (os.environ.get('ADMIN_PARTY') and os.environ.get('ADMIN_PARTY') == 'true')), + ((os.environ.get('ADMIN_PARTY') and os.environ.get('ADMIN_PARTY') == 'true')), 'Skipping couchdb context manager test' ) + @attr(db='couch') def test_couchdb_context_helper(self): """ Test that the couchdb context helper works as expected. @@ -99,10 +105,10 @@ def test_couchdb_context_helper(self): self.fail('Exception {0} was raised.'.format(str(err))) @unittest.skipUnless( - (os.environ.get('RUN_CLOUDANT_TESTS') is None and - (os.environ.get('ADMIN_PARTY') and os.environ.get('ADMIN_PARTY') == 'true')), + ((os.environ.get('ADMIN_PARTY') and os.environ.get('ADMIN_PARTY') == 'true')), 'Skipping couchdb_admin_party context manager test' ) + @attr(db='couch') def test_couchdb_admin_party_context_helper(self): """ Test that the couchdb_admin_party context helper works as expected. @@ -125,6 +131,20 @@ def test_constructor_with_url(self): self.assertEqual(self.client.encoder, json.JSONEncoder) self.assertIsNone(self.client.r_session) + def test_constructor_with_creds_removed_from_url(self): + """ + Test instantiating a client object using a URL + """ + client = CouchDB(None, None, url='http://a9a9a9a9-a9a9-a9a9-a9a9-a9a9a9a9a9a9-bluemix' + ':a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9' + 'a9a9a9a9a9a9@d8a01891-e4d2-4102-b5f8-751fb735ce31-' + 'bluemix.couchdb.local:5984') + self.assertEqual(client.server_url, 'http://d8a01891-e4d2-4102-b5f8-751fb735ce31-' + 'bluemix.couchdb.local:5984') + self.assertEqual(client._user, 'a9a9a9a9-a9a9-a9a9-a9a9-a9a9a9a9a9a9-bluemix') + self.assertEqual(client._auth_token, 'a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a' + '9a9a9a9a9a9a9a9a9a9a9a9a9') + def test_connect(self): """ Test connect and disconnect functionality. @@ -161,9 +181,10 @@ def test_multiple_connect(self): self.client.disconnect() self.assertIsNone(self.client.r_session) + @skip_if_not_cookie_auth def test_auto_renew_enabled(self): """ - Test that InfiniteSession is used when auto_renew is enabled. + Test that CookieSession is used when auto_renew is enabled. """ try: self.set_up_client(auto_renew=True) @@ -171,13 +192,14 @@ def test_auto_renew_enabled(self): if os.environ.get('ADMIN_PARTY') == 'true': self.assertIsInstance(self.client.r_session, requests.Session) else: - self.assertIsInstance(self.client.r_session, InfiniteSession) + self.assertIsInstance(self.client.r_session, CookieSession) finally: self.client.disconnect() + @skip_if_not_cookie_auth def test_auto_renew_enabled_with_auto_connect(self): """ - Test that InfiniteSession is used when auto_renew is enabled along with + Test that CookieSession is used when auto_renew is enabled along with an auto_connect. """ try: @@ -185,13 +207,14 @@ def test_auto_renew_enabled_with_auto_connect(self): if os.environ.get('ADMIN_PARTY') == 'true': self.assertIsInstance(self.client.r_session, requests.Session) else: - self.assertIsInstance(self.client.r_session, InfiniteSession) + self.assertIsInstance(self.client.r_session, CookieSession) finally: self.client.disconnect() + @skip_if_not_cookie_auth def test_session(self): """ - Test getting session information. + Test getting session information. Session info is None if CouchDB Admin Party mode was selected. """ try: @@ -204,6 +227,7 @@ def test_session(self): finally: self.client.disconnect() + @skip_if_not_cookie_auth def test_session_cookie(self): """ Test getting the session cookie. @@ -218,6 +242,100 @@ def test_session_cookie(self): finally: self.client.disconnect() + @mock.patch('cloudant._client_session.Session.request') + def test_session_basic(self, m_req): + """ + Test using basic access authentication. + """ + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + type(m_response_ok).text = mock.PropertyMock(return_value='["animaldb"]') + m_req.return_value = m_response_ok + + client = Cloudant('foo', 'bar', url=self.url, use_basic_auth=True) + client.connect() + self.assertIsInstance(client.r_session, BasicSession) + + all_dbs = client.all_dbs() + + m_req.assert_called_once_with( + 'GET', + self.url + '/_all_dbs', + allow_redirects=True, + auth=('foo', 'bar'), # uses HTTP Basic Auth + timeout=None + ) + + self.assertEqual(all_dbs, ['animaldb']) + + @mock.patch('cloudant._client_session.Session.request') + def test_session_basic_with_no_credentials(self, m_req): + """ + Test using basic access authentication with no credentials. + """ + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + m_req.return_value = m_response_ok + + client = Cloudant(None, None, url=self.url, use_basic_auth=True) + client.connect() + self.assertIsInstance(client.r_session, BasicSession) + + db = client['animaldb'] + + m_req.assert_called_once_with( + 'HEAD', + self.url + '/animaldb', + allow_redirects=False, + auth=None, # ensure no authentication specified + timeout=None + ) + + self.assertIsInstance(db, CloudantDatabase) + + @mock.patch('cloudant._client_session.Session.request') + def test_change_credentials_basic(self, m_req): + """ + Test changing credentials when using basic access authentication. + """ + # mock 200 + m_response_ok = mock.MagicMock() + type(m_response_ok).text = mock.PropertyMock(return_value='["animaldb"]') + # mock 401 + m_response_bad = mock.MagicMock() + m_response_bad.raise_for_status.side_effect = HTTPError('401 Unauthorized') + + m_req.side_effect = [m_response_bad, m_response_ok] + + client = Cloudant('foo', 'bar', url=self.url, use_basic_auth=True) + client.connect() + self.assertIsInstance(client.r_session, BasicSession) + + with self.assertRaises(HTTPError): + client.all_dbs() # expected 401 + + m_req.assert_called_with( + 'GET', + self.url + '/_all_dbs', + allow_redirects=True, + auth=('foo', 'bar'), # uses HTTP Basic Auth + timeout=None + ) + + # use valid credentials + client.change_credentials('baz', 'qux') + all_dbs = client.all_dbs() + + m_req.assert_called_with( + 'GET', + self.url + '/_all_dbs', + allow_redirects=True, + auth=('baz', 'qux'), # uses HTTP Basic Auth + timeout=None + ) + self.assertEqual(all_dbs, ['animaldb']) + + @skip_if_not_cookie_auth def test_basic_auth_str(self): """ Test getting the basic authentication string. @@ -282,6 +400,57 @@ def test_create_existing_database(self): self.client.delete_database(dbname) self.client.disconnect() + def test_create_invalid_database_name(self): + """ + Test creation of database with an invalid name + """ + dbname = 'invalidDbName_' + self.client.connect() + with self.assertRaises((CloudantDatabaseException, HTTPError)) as cm: + self.client.create_database(dbname) + code = cm.exception.status_code if hasattr(cm.exception, 'status_code') else cm.exception.response.status_code + self.assertEqual(code, 400) + self.client.disconnect() + + @skip_if_not_cookie_auth + @mock.patch('cloudant._client_session.Session.request') + def test_create_with_server_error(self, m_req): + """ + Test creation of database with a server error + """ + dbname = self.dbname() + # mock 200 for authentication + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + + # mock 404 for head request when verifying if database exists + m_response_bad = mock.MagicMock() + type(m_response_bad).status_code = mock.PropertyMock(return_value=404) + + # mock 500 when trying to create the database + m_resp_service_error = mock.MagicMock() + type(m_resp_service_error).status_code = mock.PropertyMock( + return_value=500) + type(m_resp_service_error).text = mock.PropertyMock( + return_value='Internal Server Error') + + m_req.side_effect = [m_response_ok, m_response_bad, m_resp_service_error] + + self.client.connect() + with self.assertRaises(CloudantDatabaseException) as cm: + self.client.create_database(dbname) + + self.assertEqual(cm.exception.status_code, 500) + + self.assertEqual(m_req.call_count, 3) + m_req.assert_called_with( + 'PUT', + '/'.join([self.url, dbname]), + data=None, + params={'partitioned': 'false'}, + timeout=(30, 300) + ) + def test_delete_non_existing_database(self): """ Test deletion of non-existing database @@ -395,7 +564,7 @@ def test_get_cached_db_object_via_get(self): self.client.connect() # Default returns None self.assertIsNone(self.client.get('no_such_db')) - # Creates the database remotely and adds it to the + # Creates the database remotely and adds it to the # client database cache db = self.client.create_database(dbname) # Locally cached database object is returned @@ -483,48 +652,333 @@ def test_db_updates_feed_call(self): finally: self.client.disconnect() -@unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant client specific tests' -) +@attr(db='cloudant') class CloudantClientTests(UnitTestDbBase): """ Cloudant specific client unit tests """ + def test_constructor_with_creds_removed_from_url(self): + """ + Test instantiating a client object using a URL + """ + client = Cloudant(None, None, url='https://a9a9a9a9-a9a9-a9a9-a9a9-a9a9a9a9a9a9-bluemix' + ':a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9' + 'a9a9a9a9a9a9@d8a01891-e4d2-4102-b5f8-751fb735ce31-' + 'bluemix.cloudant.com') + self.assertEqual(client.server_url, 'https://d8a01891-e4d2-4102-b5f8-751fb735ce31-' + 'bluemix.cloudant.com') + self.assertEqual(client._user, 'a9a9a9a9-a9a9-a9a9-a9a9-a9a9a9a9a9a9-bluemix') + self.assertEqual(client._auth_token, 'a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a' + '9a9a9a9a9a9a9a9a9a9a9a9a9') + @skip_if_not_cookie_auth + def test_cloudant_session_login(self): + """ + Test that the Cloudant client session successfully authenticates. + """ + self.client.connect() + old_cookie = self.client.session_cookie() + + sleep(5) # ensure we get a different cookie back + + self.client.session_login() + self.assertNotEqual(self.client.session_cookie(), old_cookie) + + @skip_if_not_cookie_auth + def test_cloudant_session_login_with_new_credentials(self): + """ + Test that the Cloudant client session fails to authenticate when + passed incorrect credentials. + """ + self.client.connect() + + with self.assertRaises(HTTPError) as cm: + self.client.session_login('invalid-user-123', 'pa$$w0rd01') + + self.assertTrue(str(cm.exception).find('Name or password is incorrect')) + + @skip_if_not_cookie_auth def test_cloudant_context_helper(self): """ Test that the cloudant context helper works as expected. """ try: - with cloudant(self.user, self.pwd, account=self.account) as c: + with cloudant(self.user, self.pwd, url=self.url) as c: self.assertIsInstance(c, Cloudant) self.assertIsInstance(c.r_session, requests.Session) except Exception as err: self.fail('Exception {0} was raised.'.format(str(err))) - + + @skip_if_not_cookie_auth + def test_cloudant_bluemix_context_helper_with_legacy_creds(self): + """ + Test that the cloudant_bluemix context helper with legacy creds works as expected. + """ + instance_name = 'Cloudant NoSQL DB-lv' + vcap_services = {'cloudantNoSQLDB': [{ + 'credentials': { + 'username': self.user, + 'password': self.pwd, + 'host': urlparse(self.url).hostname, + 'port': 443, + 'url': self.url + }, + 'name': instance_name, + }]} + + try: + with cloudant_bluemix(vcap_services, instance_name=instance_name) as c: + self.assertIsInstance(c, Cloudant) + self.assertIsInstance(c.r_session, requests.Session) + self.assertEqual(c.session()['userCtx']['name'], self.user) + except Exception as err: + self.fail('Exception {0} was raised.'.format(str(err))) + + @unittest.skipUnless(os.environ.get('IAM_API_KEY'), + 'Skipping Cloudant Bluemix context helper with IAM test') + def test_cloudant_bluemix_context_helper_with_iam(self): + """ + Test that the cloudant_bluemix context helper with IAM works as expected. + """ + instance_name = 'Cloudant NoSQL DB-lv' + vcap_services = {'cloudantNoSQLDB': [{ + 'credentials': { + 'apikey': self.iam_api_key, + 'username': self.user, + 'host': urlparse(self.url).hostname, + 'port': 443, + 'url': self.url + }, + 'name': instance_name, + }]} + + try: + with cloudant_bluemix(vcap_services, instance_name=instance_name) as c: + self.assertIsInstance(c, Cloudant) + self.assertIsInstance(c.r_session, requests.Session) + except Exception as err: + self.fail('Exception {0} was raised.'.format(str(err))) + + def test_cloudant_bluemix_context_helper_raise_error_for_missing_iam_and_creds(self): + """ + Test that the cloudant_bluemix context helper raises a CloudantClientException + when the IAM key, username, and password are missing in the VCAP_SERVICES env variable. + """ + instance_name = 'Cloudant NoSQL DB-lv' + vcap_services = {'cloudantNoSQLDB': [{ + 'credentials': { + 'host': urlparse(self.url).hostname, + 'port': 443, + 'url': self.url + }, + 'name': instance_name, + }]} + + try: + with cloudant_bluemix(vcap_services, instance_name=instance_name) as c: + self.assertIsInstance(c, Cloudant) + self.assertIsInstance(c.r_session, requests.Session) + except CloudantClientException as err: + self.assertEqual( + 'Invalid service: IAM API key or username/password credentials are required.', + str(err) + ) + + @skip_if_iam + def test_cloudant_bluemix_dedicated_context_helper(self): + """ + Test that the cloudant_bluemix context helper works as expected when + specifying a service name. + """ + instance_name = 'Cloudant NoSQL DB-wq' + service_name = 'cloudantNoSQLDB Dedicated' + vcap_services = {service_name: [{ + 'credentials': { + 'username': self.user, + 'password': self.pwd, + 'host': urlparse(self.url).hostname, + 'port': 443, + 'url': self.url + }, + 'name': instance_name, + }]} + + try: + with cloudant_bluemix(vcap_services, + instance_name=instance_name, + service_name=service_name) as c: + self.assertIsInstance(c, Cloudant) + self.assertIsInstance(c.r_session, requests.Session) + self.assertEqual(c.session()['userCtx']['name'], self.user) + except Exception as err: + self.fail('Exception {0} was raised.'.format(str(err))) + def test_constructor_with_account(self): """ Test instantiating a client object using an account name """ # Ensure that the client is new del self.client - self.client = Cloudant(self.user, self.pwd, account=self.account) + self.client = Cloudant('user', 'pass', account='foo') self.assertEqual( self.client.server_url, - 'https://{0}.cloudant.com'.format(self.account) + 'https://foo.cloudant.com' ) + @skip_if_not_cookie_auth + def test_bluemix_constructor_with_legacy_creds(self): + """ + Test instantiating a client object using a VCAP_SERVICES environment + variable. + """ + instance_name = 'Cloudant NoSQL DB-lv' + vcap_services = {'cloudantNoSQLDB': [{ + 'credentials': { + 'username': self.user, + 'password': self.pwd, + 'host': urlparse(self.url).hostname, + 'port': 443, + 'url': self.url + }, + 'name': instance_name + }]} + + # create Cloudant Bluemix client + c = Cloudant.bluemix(vcap_services) + + try: + c.connect() + self.assertIsInstance(c, Cloudant) + self.assertIsInstance(c.r_session, requests.Session) + self.assertEqual(c.session()['userCtx']['name'], self.user) + + except Exception as err: + self.fail('Exception {0} was raised.'.format(str(err))) + + finally: + c.disconnect() + + + @unittest.skipUnless(os.environ.get('IAM_API_KEY'), + 'Skipping Cloudant Bluemix constructor with IAM test') + def test_bluemix_constructor_with_iam(self): + """ + Test instantiating a client object using a VCAP_SERVICES environment + variable. + """ + instance_name = 'Cloudant NoSQL DB-lv' + vcap_services = {'cloudantNoSQLDB': [{ + 'credentials': { + 'apikey': self.iam_api_key, + 'username': self.user, + 'host': urlparse(self.url).hostname, + 'port': 443 + }, + 'name': instance_name + }]} + + # create Cloudant Bluemix client + c = Cloudant.bluemix(vcap_services) + + try: + c.connect() + self.assertIsInstance(c, Cloudant) + self.assertIsInstance(c.r_session, requests.Session) + + except Exception as err: + self.fail('Exception {0} was raised.'.format(str(err))) + + finally: + c.disconnect() + + @skip_if_iam + def test_bluemix_constructor_specify_instance_name(self): + """ + Test instantiating a client object using a VCAP_SERVICES environment + variable and specifying which instance name to use. + """ + instance_name = 'Cloudant NoSQL DB-lv' + vcap_services = {'cloudantNoSQLDB': [{ + 'credentials': { + 'username': self.user, + 'password': self.pwd, + 'host': urlparse(self.url).hostname, + 'port': 443, + 'url': self.url + }, + 'name': instance_name + }]} + + # create Cloudant Bluemix client + c = Cloudant.bluemix(vcap_services, instance_name=instance_name) + + try: + c.connect() + self.assertIsInstance(c, Cloudant) + self.assertIsInstance(c.r_session, requests.Session) + self.assertEqual(c.session()['userCtx']['name'], self.user) + + except Exception as err: + self.fail('Exception {0} was raised.'.format(str(err))) + + finally: + c.disconnect() + + @skip_if_not_cookie_auth + def test_bluemix_constructor_with_multiple_services(self): + """ + Test instantiating a client object using a VCAP_SERVICES environment + variable that contains multiple services. + """ + instance_name = 'Cloudant NoSQL DB-lv' + vcap_services = {'cloudantNoSQLDB': [ + { + 'credentials': { + 'apikey': '1234api', + 'host': urlparse(self.url).hostname, + 'port': 443, + 'url': self.url + }, + 'name': instance_name + }, + { + 'credentials': { + 'username': 'foo', + 'password': 'bar', + 'host': 'baz.com', + 'port': 1234, + 'url': 'https://foo:bar@baz.com:1234' + }, + 'name': 'Cloudant NoSQL DB-yu' + } + ]} + + # create Cloudant Bluemix client + c = Cloudant.bluemix(vcap_services, instance_name=instance_name) + + try: + c.connect() + self.assertIsInstance(c, Cloudant) + self.assertIsInstance(c.r_session, requests.Session) + self.assertEqual(c.session()['userCtx']['name'], self.user) + + except Exception as err: + self.fail('Exception {0} was raised.'.format(str(err))) + + finally: + c.disconnect() + def test_connect_headers(self): """ Test that the appropriate request headers are set """ try: self.client.connect() - self.assertEqual( - self.client.r_session.headers['X-Cloudant-User'], - self.account - ) + if (self.account): + self.assertEqual( + self.client.r_session.headers['X-Cloudant-User'], + self.account + ) agent = self.client.r_session.headers.get('User-Agent') ua_parts = agent.split('/') self.assertEqual(len(ua_parts), 6) @@ -538,6 +992,7 @@ def test_connect_headers(self): finally: self.client.disconnect() + @skip_if_not_cookie_auth def test_connect_timeout(self): """ Test that a connect timeout occurs when instantiating @@ -564,6 +1019,7 @@ def test_db_updates_infinite_feed_call(self): finally: self.client.disconnect() + @skip_if_not_cookie_auth def test_billing_data(self): """ Test the retrieval of billing data @@ -658,6 +1114,7 @@ def test_set_year_with_invalid_month_for_billing_data(self): finally: self.client.disconnect() + @skip_if_not_cookie_auth def test_volume_usage_data(self): """ Test the retrieval of volume usage data @@ -749,6 +1206,7 @@ def test_set_year_with_invalid_month_for_volume_usage_data(self): finally: self.client.disconnect() + @skip_if_not_cookie_auth def test_requests_usage_data(self): """ Test the retrieval of requests usage data @@ -840,6 +1298,7 @@ def test_set_year_with_invalid_month_for_requests_usage_data(self): finally: self.client.disconnect() + @skip_if_not_cookie_auth def test_shared_databases(self): """ Test the retrieval of shared database list @@ -850,6 +1309,7 @@ def test_shared_databases(self): finally: self.client.disconnect() + @skip_if_not_cookie_auth def test_generate_api_key(self): """ Test the generation of an API key for this client account @@ -863,6 +1323,7 @@ def test_generate_api_key(self): finally: self.client.disconnect() + @skip_if_not_cookie_auth def test_cors_configuration(self): """ Test the retrieval of the current CORS configuration for this client @@ -876,6 +1337,7 @@ def test_cors_configuration(self): finally: self.client.disconnect() + @skip_if_not_cookie_auth def test_cors_origins(self): """ Test the retrieval of the CORS origins list @@ -887,6 +1349,7 @@ def test_cors_origins(self): finally: self.client.disconnect() + @skip_if_not_cookie_auth def test_disable_cors(self): """ Test disabling CORS (assuming CORS is enabled) @@ -907,6 +1370,7 @@ def test_disable_cors(self): finally: self.client.disconnect() + @skip_if_not_cookie_auth def test_update_cors_configuration(self): """ Test updating CORS configuration @@ -951,4 +1415,4 @@ def test_update_cors_configuration(self): self.client.disconnect() if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/unit/cloud_foundry_tests.py b/tests/unit/cloud_foundry_tests.py index 043949f7..a832e4ce 100644 --- a/tests/unit/cloud_foundry_tests.py +++ b/tests/unit/cloud_foundry_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -29,20 +29,30 @@ class CloudFoundryServiceTests(unittest.TestCase): def __init__(self, *args, **kwargs): super(CloudFoundryServiceTests, self).__init__(*args, **kwargs) - self._test_vcap_services_single = json.dumps({'cloudantNoSQLDB': [ - { + self._test_vcap_services_single_legacy_credentials_enabled = json.dumps({'cloudantNoSQLDB': [{ + 'name': 'Cloudant NoSQL DB 1', # valid service with legacy creds enabled + 'credentials': { + 'apikey': '1234api', + 'username': 'user-bluemix', + 'password': 'password', + 'port': 443, + 'host': 'user-bluemix.cloudant.com' + } + } + ]}) + self._test_vcap_services_single = json.dumps({'cloudantNoSQLDB': [{ 'name': 'Cloudant NoSQL DB 1', # valid service 'credentials': { - 'host': 'example.cloudant.com', - 'password': 'pa$$w0rd01', - 'port': 1234, - 'username': 'example' + 'apikey': '1234api', + 'username': 'user-bluemix', + 'port': 443, + 'host': 'user-bluemix.cloudant.com' } } ]}) - self._test_vcap_services_multiple = json.dumps({'cloudantNoSQLDB': [ + self._test_legacy_vcap_services_multiple = json.dumps({'cloudantNoSQLDB': [ { - 'name': 'Cloudant NoSQL DB 1', # valid service + 'name': 'Cloudant NoSQL DB 1', # valid legacy service 'credentials': { 'host': 'example.cloudant.com', 'password': 'pa$$w0rd01', @@ -89,80 +99,154 @@ def __init__(self, *args, **kwargs): 'pa$$w0rd01', 'example' ] - } + }, + { + 'name': 'Cloudant NoSQL DB 7', # missing iam api key and creds + 'credentials': { + 'host': 'example.cloudant.com', + 'port': 1234, + 'username': 'example' + } + }, + { + 'name': 'Cloudant NoSQL DB 8', # valid service with IAM api + 'credentials': { + 'apikey': '1234api', + 'username': 'example', + 'host': 'example.cloudant.com', + 'port': 1234 + } + }, ]}) + self._test_vcap_services_dedicated = json.dumps({ + 'cloudantNoSQLDB Dedicated': [ # dedicated service name + { + 'name': 'Cloudant NoSQL DB 1', # valid service + 'credentials': { + 'host': 'example.cloudant.com', + 'password': 'pa$$w0rd01', + 'port': 1234, + 'username': 'example' + } + } + ] + }) + + def test_get_vcap_service_legacy_creds_success(self): + service = CloudFoundryService( + self._test_vcap_services_single_legacy_credentials_enabled, + service_name='cloudantNoSQLDB' + ) + self.assertEqual('Cloudant NoSQL DB 1', service.name) - def test_get_vcap_service_default_success(self): - service = CloudFoundryService(self._test_vcap_services_single) + def test_get_vcap_service_iam_api_no_creds_success(self): + service = CloudFoundryService( + self._test_vcap_services_single, + service_name='cloudantNoSQLDB' + ) self.assertEqual('Cloudant NoSQL DB 1', service.name) + self.assertEqual('1234api', service.iam_api_key) + with self.assertRaises(AttributeError) as cm: + service.password + self.assertEqual("'CloudFoundryService' object has no attribute '_password'", str(cm.exception)) def test_get_vcap_service_default_success_as_dict(self): service = CloudFoundryService( - json.loads(self._test_vcap_services_single) + json.loads(self._test_vcap_services_single_legacy_credentials_enabled), + service_name='cloudantNoSQLDB' ) self.assertEqual('Cloudant NoSQL DB 1', service.name) def test_get_vcap_service_default_failure_multiple_services(self): with self.assertRaises(CloudantException) as cm: - CloudFoundryService(self._test_vcap_services_multiple) + CloudFoundryService( + self._test_legacy_vcap_services_multiple, + service_name='cloudantNoSQLDB' + ) self.assertEqual('Missing service in VCAP_SERVICES', str(cm.exception)) def test_get_vcap_service_instance_host(self): service = CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 1' + self._test_legacy_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 1', + service_name='cloudantNoSQLDB' ) self.assertEqual('example.cloudant.com', service.host) def test_get_vcap_service_instance_password(self): service = CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 1' + self._test_legacy_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 1', + service_name='cloudantNoSQLDB' ) self.assertEqual('pa$$w0rd01', service.password) def test_get_vcap_service_instance_port(self): service = CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 1' + self._test_legacy_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 1', + service_name='cloudantNoSQLDB' ) self.assertEqual('1234', service.port) def test_get_vcap_service_instance_port_default(self): service = CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 2' + self._test_legacy_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 2', + service_name='cloudantNoSQLDB' ) self.assertEqual('443', service.port) def test_get_vcap_service_instance_url(self): service = CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 1' + self._test_legacy_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 1', + service_name='cloudantNoSQLDB' ) self.assertEqual('https://example.cloudant.com:1234', service.url) def test_get_vcap_service_instance_username(self): service = CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 1' + self._test_legacy_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 1', + service_name='cloudantNoSQLDB' ) self.assertEqual('example', service.username) + def test_get_vcap_service_instance_iam_api_key(self): + service = CloudFoundryService( + self._test_legacy_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 8', + service_name='cloudantNoSQLDB' + ) + self.assertEqual('1234api', service.iam_api_key) + def test_raise_error_for_missing_host(self): with self.assertRaises(CloudantException): CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 3' + self._test_legacy_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 3', + service_name='cloudantNoSQLDB' ) def test_raise_error_for_missing_password(self): with self.assertRaises(CloudantException) as cm: CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 4' + self._test_legacy_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 4', + service_name='cloudantNoSQLDB' ) self.assertEqual( - "Invalid service: 'password' missing", + 'Invalid service: IAM API key or username/password credentials are required.', str(cm.exception) ) def test_raise_error_for_missing_username(self): with self.assertRaises(CloudantException) as cm: CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 5' + self._test_legacy_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 5', + service_name='cloudantNoSQLDB' ) self.assertEqual( "Invalid service: 'username' missing", @@ -172,17 +256,33 @@ def test_raise_error_for_missing_username(self): def test_raise_error_for_invalid_credentials_type(self): with self.assertRaises(CloudantException) as cm: CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 6' + self._test_legacy_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 6', + service_name='cloudantNoSQLDB' ) self.assertEqual( 'Failed to decode VCAP_SERVICES service credentials', str(cm.exception) ) + def test_raise_error_for_missing_iam_api_key_and_credentials(self): + with self.assertRaises(CloudantException) as cm: + CloudFoundryService( + self._test_legacy_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 7', + service_name='cloudantNoSQLDB' + ) + self.assertEqual( + 'Invalid service: IAM API key or username/password credentials are required.', + str(cm.exception) + ) + def test_raise_error_for_missing_service(self): with self.assertRaises(CloudantException) as cm: CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 7' + self._test_legacy_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 9', + service_name='cloudantNoSQLDB' ) self.assertEqual('Missing service in VCAP_SERVICES', str(cm.exception)) @@ -190,3 +290,10 @@ def test_raise_error_for_invalid_vcap(self): with self.assertRaises(CloudantException) as cm: CloudFoundryService('{', 'Cloudant NoSQL DB 1') # invalid JSON self.assertEqual('Failed to decode VCAP_SERVICES JSON', str(cm.exception)) + + def test_get_vcap_service_with_dedicated_service_name_success(self): + service = CloudFoundryService( + self._test_vcap_services_dedicated, + service_name='cloudantNoSQLDB Dedicated' + ) + self.assertEqual('Cloudant NoSQL DB 1', service.name) diff --git a/tests/unit/database_partition_tests.py b/tests/unit/database_partition_tests.py new file mode 100644 index 00000000..8b3690dd --- /dev/null +++ b/tests/unit/database_partition_tests.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python +# Copyright (C) 2019 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +_database_partition_tests_ +""" + +from cloudant.design_document import DesignDocument +from cloudant.index import Index, SpecialIndex + +from nose.plugins.attrib import attr + +from .unit_t_db_base import UnitTestDbBase + + +@attr(db=['cloudant']) +class DatabasePartitionTests(UnitTestDbBase): + + def setUp(self): + super(DatabasePartitionTests, self).setUp() + self.db_set_up(partitioned=True) + + def tearDown(self): + self.db_tear_down() + super(DatabasePartitionTests, self).tearDown() + + def test_is_partitioned_database(self): + self.assertTrue(self.db.metadata()['props']['partitioned']) + + def test_create_partitioned_design_document(self): + ddoc_id = 'empty_ddoc' + + ddoc = DesignDocument(self.db, ddoc_id, partitioned=True) + ddoc.save() + + r = self.db.r_session.get(ddoc.document_url) + r.raise_for_status() + + self.assertTrue(r.json()['options']['partitioned']) + + def test_create_non_partitioned_design_document(self): + ddoc_id = 'empty_ddoc' + + ddoc = DesignDocument(self.db, ddoc_id, partitioned=False) + ddoc.save() + + r = self.db.r_session.get(ddoc.document_url) + r.raise_for_status() + + self.assertFalse(r.json()['options']['partitioned']) + + def test_partitioned_all_docs(self): + for partition_key in self.populate_db_with_partitioned_documents(5, 25): + docs = self.db.partitioned_all_docs(partition_key) + self.assertEqual(len(docs['rows']), 25) + + for doc in docs['rows']: + self.assertTrue(doc['id'].startswith(partition_key + ':')) + + def test_partition_metadata(self): + for partition_key in self.populate_db_with_partitioned_documents(5, 25): + meta = self.db.partition_metadata(partition_key) + self.assertEqual(meta['partition'], partition_key) + self.assertEqual(meta['doc_count'], 25) + + def test_partitioned_search(self): + ddoc = DesignDocument(self.db, 'partitioned_search', partitioned=True) + ddoc.add_search_index( + 'search1', + 'function(doc) { index("id", doc._id, {"store": true}); }' + ) + ddoc.save() + + for partition_key in self.populate_db_with_partitioned_documents(2, 10): + results = self.db.get_partitioned_search_result( + partition_key, ddoc['_id'], 'search1', query='*:*') + + i = 0 + for result in results['rows']: + print(result) + self.assertTrue(result['id'].startswith(partition_key + ':')) + i += 1 + self.assertEqual(i, 10) + + def test_get_partitioned_index(self): + index_name = 'test_partitioned_index' + + self.db.create_query_index(index_name=index_name, fields=['foo']) + + results = self.db.get_query_indexes() + self.assertEqual(len(results), 2) + + index_all_docs = results[0] + self.assertEqual(index_all_docs.name, '_all_docs') + self.assertEqual(type(index_all_docs), SpecialIndex) + self.assertFalse(index_all_docs.partitioned) + + index_partitioned = results[1] + self.assertEqual(index_partitioned.name, index_name) + self.assertEqual(type(index_partitioned), Index) + self.assertTrue(index_partitioned.partitioned) + + def test_partitioned_query(self): + self.db.create_query_index(fields=['foo']) + + for partition_key in self.populate_db_with_partitioned_documents(2, 10): + results = self.db.get_partitioned_query_result( + partition_key, selector={'foo': {'$eq': 'bar'}}) + + i = 0 + for result in results: + self.assertTrue(result['_id'].startswith(partition_key + ':')) + i += 1 + self.assertEqual(i, 10) + + def test_partitioned_view(self): + ddoc = DesignDocument(self.db, 'partitioned_view', partitioned=True) + ddoc.add_view('view1', 'function(doc) { emit(doc._id, 1); }') + ddoc.save() + + for partition_key in self.populate_db_with_partitioned_documents(2, 10): + results = self.db.get_partitioned_view_result( + partition_key, ddoc['_id'], 'view1') + + i = 0 + for result in results: + self.assertTrue( + result['id'].startswith(partition_key + ':')) + i += 1 + self.assertEqual(i, 10) diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index f9abac99..ae9898c7 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,25 +22,28 @@ """ -import unittest -import mock -import requests -import posixpath import os +import unittest import uuid -from cloudant.result import Result, QueryResult -from cloudant.error import CloudantArgumentError, CloudantDatabaseException -from cloudant.document import Document +import mock +import requests +from cloudant._2to3 import UNICHR +from cloudant._common_util import response_to_json_dict from cloudant.design_document import DesignDocument -from cloudant.security_document import SecurityDocument -from cloudant.index import Index, TextIndex, SpecialIndex +from cloudant.document import Document +from cloudant.error import CloudantArgumentError, CloudantDatabaseException from cloudant.feed import Feed, InfiniteFeed -from tests.unit._test_util import LONG_NUMBER +from cloudant.index import Index, TextIndex, SpecialIndex +from cloudant.result import Result, QueryResult +from cloudant.security_document import SecurityDocument +from nose.plugins.attrib import attr -from .unit_t_db_base import UnitTestDbBase +from tests.unit._test_util import LONG_NUMBER +from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase, skip_if_iam from .. import unicode_ + class CloudantDatabaseExceptionTests(unittest.TestCase): """ Ensure CloudantDatabaseException functions as expected. @@ -79,6 +82,7 @@ def test_raise_with_proper_code_and_args(self): raise CloudantDatabaseException(400, 'foo') self.assertEqual(cm.exception.status_code, 400) +@attr(db=['cloudant','couch']) class DatabaseTests(UnitTestDbBase): """ CouchDatabase/CloudantDatabase unit tests @@ -148,9 +152,10 @@ def test_retrieve_db_url(self): """ self.assertEqual( self.db.database_url, - posixpath.join(self.client.server_url, self.test_dbname) - ) + '/'.join((self.client.server_url, self.test_dbname)) + ) + @skip_if_not_cookie_auth def test_retrieve_creds(self): """ Test retrieving client credentials. The client credentials are None if @@ -232,9 +237,8 @@ def test_retrieve_db_metadata(self): same. Therefore comparing keys is a valid test of this functionality. """ resp = self.db.r_session.get( - posixpath.join(self.client.server_url, self.test_dbname) - ) - expected = resp.json() + '/'.join((self.client.server_url, self.test_dbname))) + expected = response_to_json_dict(resp) actual = self.db.metadata() self.assertListEqual(list(actual.keys()), list(expected.keys())) @@ -252,6 +256,8 @@ def test_create_document_with_id(self): data = {'_id': 'julia06', 'name': 'julia', 'age': 6} doc = self.db.create_document(data) self.assertEqual(self.db['julia06'], doc) + self.assertEqual(self.db.get('julia06'), doc) + self.assertEqual(self.db.get('julia06', remote=True), doc) self.assertEqual(doc['_id'], data['_id']) self.assertTrue(doc['_rev'].startswith('1-')) self.assertEqual(doc['name'], data['name']) @@ -267,6 +273,55 @@ def test_create_document_with_id(self): 'Document with id julia06 already exists.' ) + def test_get_non_existing_document_from_remote(self): + """ + Test dict's get on non existing document from remote. + """ + doc = self.db.get('non-existing', remote=True) + self.assertIsNone(doc) + + def test_get_non_existing_document_from_cache(self): + """ + Test dict's get on non existing document from cache. + """ + doc = self.db.get('non-existing') + self.assertIsNone(doc) + + def test_get_document_from_cache(self): + """ + Test dict's get on a document from cache. + """ + doc = Document(self.db, document_id='julia06') + self.db['julia06'] = doc + self.assertEqual(self.db.get('julia06'), doc) + # doc is fetched from the local dict preferentially to remote even with remote=True + self.assertEqual(self.db.get('julia06', remote=True), doc) + self.assertEqual(self.db['julia06'], doc) + + def test_get_document_from_remote(self): + """ + Test dict's get on a document from remote. + """ + data = {'_id': 'julia06','name': 'julia06', 'age': 6} + doc = self.db.create_document(data) + self.db.clear() + self.assertIsNone(self.db.get('julia06')) + self.assertEqual(self.db.get('julia06', remote=True), doc) + self.assertEqual(self.db['julia06'], doc) + + def test_create_document_that_already_exists(self): + """ + Test creating a document that already exists + """ + data = {'_id': 'julia'} + doc = self.db.create_document(data) + self.assertEqual(self.db['julia'], doc) + self.assertEqual(self.db.get('julia'), doc) + self.assertEqual(self.db.get('julia', remote=True), doc) + self.assertTrue(doc['_rev'].startswith('1-')) + # attempt to recreate document + self.db.create_document(data, throw_on_exists=False) + def test_create_document_without_id(self): """ Test creating a document without supplying a document id @@ -274,6 +329,8 @@ def test_create_document_without_id(self): data = {'name': 'julia', 'age': 6} doc = self.db.create_document(data) self.assertEqual(self.db[doc['_id']], doc) + self.assertEqual(self.db.get(doc['_id']), doc) + self.assertEqual(self.db.get(doc['_id'], remote=True), doc) self.assertTrue(doc['_rev'].startswith('1-')) self.assertEqual(doc['name'], data['name']) self.assertEqual(doc['age'], data['age']) @@ -287,6 +344,8 @@ def test_create_design_document(self): data = {'_id': '_design/julia06', 'name': 'julia', 'age': 6} doc = self.db.create_document(data) self.assertEqual(self.db['_design/julia06'], doc) + self.assertEqual(self.db.get('_design/julia06'), doc) + self.assertEqual(self.db.get('_design/julia06', remote=True), doc) self.assertEqual(doc['_id'], data['_id']) self.assertTrue(doc['_rev'].startswith('1-')) self.assertEqual(doc['name'], data['name']) @@ -301,6 +360,8 @@ def test_create_empty_document(self): """ empty_doc = self.db.new_document() self.assertEqual(self.db[empty_doc['_id']], empty_doc) + self.assertEqual(self.db.get(empty_doc['_id']), empty_doc) + self.assertEqual(self.db.get(empty_doc['_id'], remote=True), empty_doc) self.assertTrue(all(x in ['_id', '_rev'] for x in empty_doc.keys())) self.assertTrue(empty_doc['_rev'].startswith('1-')) @@ -349,7 +410,7 @@ def test_retrieve_design_document(self): """ # Get an empty design document object that does not exist remotely local_ddoc = self.db.get_design_document('_design/ddoc01') - self.assertEqual(local_ddoc, {'_id': '_design/ddoc01', 'indexes': {}, + self.assertEqual(local_ddoc, {'_id': '_design/ddoc01', 'indexes': {}, 'options': {'partitioned': False}, 'views': {}, 'lists': {}, 'shows': {}}) # Add the design document to the database map_func = 'function(doc) {\n emit(doc._id, 1); \n}' @@ -360,6 +421,7 @@ def test_retrieve_design_document(self): ddoc = self.db.get_design_document('_design/ddoc01') self.assertEqual(ddoc, local_ddoc) + @skip_if_not_cookie_auth def test_get_security_document(self): """ Test retrieving the database security document @@ -392,6 +454,34 @@ def test_retrieve_view_results(self): self.assertIsInstance(rslt, Result) self.assertEqual(rslt[:1], rslt['julia099']) + def test_retrieve_grouped_view_result_with_page_size(self): + """ + Test retrieving Result wrapped output from a design document grouped view + that uses a custom page size + + The view used here along with group=True will generate rows of + data where each key will be grouped into groups of 2. Such as: + {'key': 0, 'value': 2}, + {'key': 1, 'value': 2}, + ... + """ + map_func = 'function(doc) {\n emit(Math.floor(doc.age / 2), 1); \n}' + data = {'_id': '_design/ddoc01','views': {'view01': {"map": map_func, "reduce": "_count"}}} + self.db.create_document(data) + self.populate_db_with_documents(5) + + rslt = self.db.get_view_result( + '_design/ddoc01', + 'view01', + group=True, + page_size=1) + self.assertIsInstance(rslt, Result) + i = 0 + for row in rslt: + self.assertIsNotNone(row) + self.assertEqual(row['key'], i) + i += 1 + def test_retrieve_raw_view_results(self): """ Test retrieving raw output from a design document view @@ -421,6 +511,15 @@ def test_all_docs_post(self): keys_returned = [row['key'] for row in rows] self.assertTrue(all(x in keys_returned for x in keys_list)) + def test_all_docs_post_empty_key_list(self): + """ + Test the all_docs POST request functionality using empty keys param + """ + self.populate_db_with_documents() + # Request all_docs using an empty key list + rows = self.db.all_docs(keys=[]).get('rows') + self.assertEqual(len(rows), 0) + def test_all_docs_post_multiple_params(self): """ Test the all_docs POST request functionality using keys and other params @@ -459,6 +558,15 @@ def test_all_docs_get_with_long_type(self): data = self.db.all_docs(limit=1, skip=LONG_NUMBER) self.assertEqual(len(data.get('rows')), 1) + def test_all_docs_get_uses_custom_encoder(self): + """ + Test that all_docs uses the custom encoder. + """ + self.set_up_client(auto_connect=True, encoder="AEncoder") + database = self.client[self.test_dbname] + with self.assertRaises(CloudantArgumentError): + database.all_docs(endkey=['foo', 10]) + def test_custom_result_context_manager(self): """ Test using the database custom result context manager @@ -481,6 +589,20 @@ def test_keys(self): ['julia000', 'julia001', 'julia002'] ) + def test_doc_id_in_db(self): + """ + Test checking if a document exists in a DB with in operator + """ + self.populate_db_with_documents(1) + self.assertTrue('julia000' in self.db) + + def test_doc_id_not_in_db(self): + """ + Test checking if a document exists in a DB with in operator + """ + self.populate_db_with_documents(1) + self.assertFalse('julia001' in self.db) + def test_get_non_existing_doc_via_getitem(self): """ Test __getitem__ when retrieving a non-existing document @@ -585,6 +707,39 @@ def test_document_iteration_over_fetch_limit(self): self.assertEqual(doc['name'], 'julia') self.assertEqual(doc['age'], int(id[len(id) - 3: len(id)])) + def test_document_iteration_completeness(self): + """ + Test __iter__ works as expected, fetching all documents from the + database. + """ + for _ in self.db: + self.fail('There should be no documents in the database yet!!') + + # sample code point ranges + include_ranges = [ + (0x0023, 0x0026), + (0x00A1, 0x00AC), + (0x0370, 0x0377), + (0x037A, 0x037E), + (0x0384, 0x038A), + (0x16A0, 0x16F0), + (0x2C60, 0x2C7F) + ] + + all_docs = [{'_id': UNICHR(i) + UNICHR(j)} for a, b in include_ranges + for i in range(a, b) + for j in range(a, b)] + batch_size = 500 + for i in range(0, len(all_docs), batch_size): + self.db.bulk_docs(all_docs[i:i+batch_size]) + + doc_count = 0 + for i, doc in enumerate(self.db): + doc_count += 1 + self.assertEqual(doc['_id'], all_docs[i]['_id']) + + self.assertEqual(doc_count, len(all_docs)) + def test_document_iteration_returns_valid_documents(self): """ This test will check that the __iter__ method returns documents that are @@ -602,7 +757,7 @@ def test_document_iteration_returns_valid_documents(self): # A valid document must have a document_url self.assertEqual( doc.document_url, - posixpath.join(self.db.database_url, doc['_id']) + '/'.join((self.db.database_url, doc['_id'])) ) if isinstance(doc, DesignDocument): self.assertEqual(doc['_id'], '_design/ddoc001') @@ -701,19 +856,38 @@ def test_revisions_diff(self): # Test no differences self.assertEqual(self.db.revisions_diff('julia006', doc['_rev']), {}) - def test_get_set_revision_limit(self): + @mock.patch('cloudant._client_session.ClientSession.request') + def test_get_set_revision_limit(self, m_req): """ Test setting and getting revision limits """ - limit = self.db.get_revision_limit() - self.assertIsInstance(limit, int) + # Setup mock responses. + mock_200_get_1 = mock.MagicMock() + type(mock_200_get_1).status_code = mock.PropertyMock(return_value=200) + type(mock_200_get_1).text = mock.PropertyMock(return_value='4321') + + mock_200_get_2 = mock.MagicMock() + type(mock_200_get_2).status_code = mock.PropertyMock(return_value=200) + type(mock_200_get_2).text = mock.PropertyMock(return_value='1234') + + mock_200_set = mock.MagicMock() + type(mock_200_set).status_code = mock.PropertyMock(return_value=200) + type(mock_200_set).text = mock.PropertyMock(return_value='{"ok":true}') + + m_req.side_effect = [mock_200_get_1, mock_200_set, mock_200_get_2] + + # Get current revisions limit. + self.assertEqual(self.db.get_revision_limit(), 4321) + + # Set new revisions limit. self.assertEqual(self.db.set_revision_limit(1234), {'ok': True}) - new_limit = self.db.get_revision_limit() - self.assertNotEqual(new_limit, limit) - self.assertEqual(new_limit, 1234) - @unittest.skipIf(os.environ.get('RUN_CLOUDANT_TESTS'), - 'Skipping since view cleanup is automatic in Cloudant.') + # Get new revisions limit. + self.assertEqual(self.db.get_revision_limit(), 1234) + + self.assertEqual(m_req.call_count, 3) + + @attr(db='couch') def test_view_clean_up(self): """ Test cleaning up old view files @@ -809,7 +983,7 @@ def test_get_show_result(self): resp, 'Hello from doc001!' ) - + @skip_if_iam def test_create_doc_with_update_handler(self): """ Test update_handler_result executes an update handler function @@ -830,6 +1004,7 @@ def test_create_doc_with_update_handler(self): 'Created new doc: {"message":"hello","_id":"testDoc"}' ) + @skip_if_iam def test_update_doc_with_update_handler(self): """ Test update_handler_result executes an update handler function @@ -899,10 +1074,46 @@ def test_database_request_fails_after_client_disconnects(self): finally: self.client.connect() -@unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant specific Database tests' -) + @attr(couchapi=2) + def test_create_json_index(self): + """ + Ensure that a JSON index is created as expected. + """ + index = self.db.create_query_index(fields=['name', 'age']) + self.assertIsInstance(index, Index) + + ddoc = self.db[index.design_document_id] + + self.assertEqual(ddoc['_id'], index.design_document_id) + self.assertTrue(ddoc['_rev'].startswith('1-')) + + self.assertEqual(ddoc['indexes'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + + index = ddoc['views'][index.name] + self.assertEqual(index['map']['fields']['age'], 'asc') + self.assertEqual(index['map']['fields']['name'], 'asc') + self.assertEqual(index['options']['def']['fields'], ['name', 'age']) + self.assertEqual(index['reduce'], '_count') + + @attr(couchapi=2) + def test_delete_json_index(self): + """ + Ensure that a JSON index is deleted as expected. + """ + index = self.db.create_query_index( + 'ddoc001', + 'index001', + fields=['name', 'age']) + self.assertIsInstance(index, Index) + ddoc = self.db['_design/ddoc001'] + self.assertTrue(ddoc.exists()) + self.db.delete_query_index('ddoc001', 'json', 'index001') + self.assertFalse(ddoc.exists()) + +@attr(db='cloudant') class CloudantDatabaseTests(UnitTestDbBase): """ Cloudant specific Database unit tests @@ -943,6 +1154,7 @@ def test_unshare_database_uses_custom_encoder(self): with self.assertRaises(TypeError): database.unshare_database(share) + @skip_if_not_cookie_auth def test_security_document(self): """ Test the retrieval of the security document. @@ -952,6 +1164,7 @@ def test_security_document(self): expected = {'cloudant': {share: ['_reader']}} self.assertDictEqual(self.db.security_document(), expected) + @skip_if_not_cookie_auth def test_share_database_default_permissions(self): """ Test the sharing of a database applying default permissions. @@ -962,6 +1175,7 @@ def test_share_database_default_permissions(self): expected = {'cloudant': {share: ['_reader']}} self.assertDictEqual(self.db.security_document(), expected) + @skip_if_not_cookie_auth def test_share_database(self): """ Test the sharing of a database. @@ -972,6 +1186,7 @@ def test_share_database(self): expected = {'cloudant': {share: ['_writer']}} self.assertDictEqual(self.db.security_document(), expected) + @skip_if_not_cookie_auth def test_share_database_with_redundant_role_entries(self): """ Test the sharing of a database works when the list of roles contains @@ -1014,6 +1229,7 @@ def test_share_database_empty_role_list(self): '\'_db_updates\', \'_design\', \'_shards\', \'_security\']' ) + @skip_if_not_cookie_auth def test_unshare_database(self): """ Test the un-sharing of a database from a specified user. @@ -1140,29 +1356,6 @@ def test_get_query_result_with_empty_fields_list(self): ['julia001', 'julia002', 'julia003', 'julia004'] ) - def test_create_json_index(self): - """ - Ensure that a JSON index is created as expected. - """ - index = self.db.create_query_index(fields=['name', 'age']) - self.assertIsInstance(index, Index) - ddoc = self.db[index.design_document_id] - self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': index.design_document_id, - '_rev': ddoc['_rev'], - 'indexes': {}, - 'lists': {}, - 'shows': {}, - 'language': 'query', - 'views': {index.name: {'map': {'fields': {'name': 'asc', - 'age': 'asc'}}, - 'reduce': '_count', - 'options': {'def': {'fields': ['name', - 'age']}, - }}}} - ) - def test_create_text_index(self): """ Ensure that a text index is created as expected. @@ -1173,25 +1366,26 @@ def test_create_text_index(self): {'name': 'age', 'type':'number'}] ) self.assertIsInstance(index, TextIndex) + ddoc = self.db[index.design_document_id] + + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': index.design_document_id, - '_rev': ddoc['_rev'], - 'language': 'query', - 'views': {}, - 'lists': {}, - 'shows': {}, - 'indexes': {index.name: {'index': {'index_array_lengths': True, - 'fields': [{'name': 'name', 'type': 'string'}, - {'name': 'age', 'type': 'number'}], - 'default_field': {}, - 'default_analyzer': 'keyword', - 'selector': {}}, - 'analyzer': {'name': 'perfield', - 'default': 'keyword', - 'fields': {'$default': 'standard'}}}}} - ) + + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + self.assertEqual(ddoc['views'], {}) + + text_index = ddoc['indexes'][index.name] + self.assertEqual(text_index['analyzer']['default'], 'keyword') + self.assertEqual(text_index['analyzer']['fields']['$default'], 'standard') + self.assertEqual(text_index['analyzer']['name'], 'perfield') + self.assertEqual(text_index['index']['default_analyzer'], 'keyword') + self.assertEqual(text_index['index']['default_field'], {}) + self.assertEqual(text_index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) + self.assertEqual(text_index['index']['selector'], {}) + self.assertTrue(text_index['index']['index_array_lengths']) def test_create_all_fields_text_index(self): """ @@ -1199,36 +1393,38 @@ def test_create_all_fields_text_index(self): """ index = self.db.create_query_index(index_type='text') self.assertIsInstance(index, TextIndex) + ddoc = self.db[index.design_document_id] + + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': index.design_document_id, - '_rev': ddoc['_rev'], - 'language': 'query', - 'views': {}, - 'lists': {}, - 'shows': {}, - 'indexes': {index.name: {'index': {'index_array_lengths': True, - 'fields': 'all_fields', - 'default_field': {}, - 'default_analyzer': 'keyword', - 'selector': {}}, - 'analyzer': {'name': 'perfield', - 'default': 'keyword', - 'fields': {'$default': 'standard'}}}}} - ) + + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + self.assertEqual(ddoc['views'], {}) + + index = ddoc['indexes'][index.name] + self.assertEqual(index['analyzer']['default'], 'keyword') + self.assertEqual(index['analyzer']['fields'], {'$default': 'standard'}) + self.assertEqual(index['analyzer']['name'], 'perfield') + self.assertEqual(index['index']['default_analyzer'], 'keyword') + self.assertEqual(index['index']['default_field'], {}) + self.assertEqual(index['index']['fields'], 'all_fields') + self.assertEqual(index['index']['selector'], {}) + self.assertTrue(index['index']['index_array_lengths']) def test_create_multiple_indexes_one_ddoc(self): """ Tests that multiple indexes of different types can be stored in one design document. """ - json_index = self.db.create_query_index( + index = self.db.create_query_index( 'ddoc001', 'json-index-001', fields=['name', 'age'] ) - self.assertIsInstance(json_index, Index) + self.assertIsInstance(index, Index) search_index = self.db.create_query_index( 'ddoc001', 'text-index-001', @@ -1237,32 +1433,31 @@ def test_create_multiple_indexes_one_ddoc(self): {'name': 'age', 'type':'number'}] ) self.assertIsInstance(search_index, TextIndex) + ddoc = self.db['_design/ddoc001'] + + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('2-')) - self.assertEqual(ddoc, - {'_id': '_design/ddoc001', - '_rev': ddoc['_rev'], - 'language': 'query', - 'lists': {}, - 'shows': {}, - 'views': {'json-index-001': { - 'map': {'fields': {'name': 'asc', - 'age': 'asc'}}, - 'reduce': '_count', - 'options': {'def': {'fields': ['name', - 'age']}, - }}}, - 'indexes': {'text-index-001': { - 'index': {'index_array_lengths': True, - 'fields': [{'name': 'name', 'type': 'string'}, - {'name': 'age', 'type': 'number'}], - 'default_field': {}, - 'default_analyzer': 'keyword', - 'selector': {}}, - 'analyzer': {'name': 'perfield', - 'default': 'keyword', - 'fields': {'$default': 'standard'}}}}} - ) + + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + + json_index = ddoc['views']['json-index-001'] + self.assertEqual(json_index['map']['fields']['age'], 'asc') + self.assertEqual(json_index['map']['fields']['name'], 'asc') + self.assertEqual(json_index['options']['def']['fields'], ['name', 'age']) + self.assertEqual(json_index['reduce'], '_count') + + text_index = ddoc['indexes']['text-index-001'] + self.assertEqual(text_index['analyzer']['default'], 'keyword') + self.assertEqual(text_index['analyzer']['fields']['$default'], 'standard') + self.assertEqual(text_index['analyzer']['name'], 'perfield') + self.assertEqual(text_index['index']['default_analyzer'], 'keyword') + self.assertEqual(text_index['index']['default_field'], {}) + self.assertEqual(text_index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) + self.assertEqual(text_index['index']['selector'], {}) + self.assertTrue(text_index['index']['index_array_lengths']) def test_create_query_index_failure(self): """ @@ -1283,20 +1478,6 @@ def test_create_query_index_failure(self): 'Index type must be either \"json\" or \"text\".' ) - def test_delete_json_index(self): - """ - Ensure that a JSON index is deleted as expected. - """ - index = self.db.create_query_index( - 'ddoc001', - 'index001', - fields=['name', 'age']) - self.assertIsInstance(index, Index) - ddoc = self.db['_design/ddoc001'] - self.assertTrue(ddoc.exists()) - self.db.delete_query_index('ddoc001', 'json', 'index001') - self.assertFalse(ddoc.exists()) - def test_delete_text_index(self): """ Ensure that a text index is deleted as expected. @@ -1329,28 +1510,32 @@ def test_get_query_indexes_raw(self): """ self.db.create_query_index('ddoc001', 'json-idx-001', fields=['name', 'age']) self.db.create_query_index('ddoc001', 'text-idx-001', 'text') - self.assertEqual( - self.db.get_query_indexes(raw_result=True), - {'indexes': [ - {'ddoc': None, - 'name': '_all_docs', - 'type': 'special', - 'def': {'fields': [{'_id': 'asc'}]}}, - {'ddoc': '_design/ddoc001', - 'name': 'json-idx-001', - 'type': 'json', - 'def': {'fields': [{'name': 'asc'}, {'age': 'asc'}]}}, - {'ddoc': '_design/ddoc001', - 'name': 'text-idx-001', - 'type': 'text', - 'def': {'index_array_lengths': True, - 'fields': [], - 'default_field': {}, - 'default_analyzer': 'keyword', - 'selector': {}}} - ], - 'total_rows' : 3} - ) + + indexes = self.db.get_query_indexes(raw_result=True) + + self.assertEqual(indexes['total_rows'], 3) + + all_docs_index = indexes['indexes'][0] + self.assertEqual(all_docs_index['ddoc'], None) + self.assertEqual(all_docs_index['def']['fields'], [{'_id': 'asc'}]) + self.assertEqual(all_docs_index['name'], '_all_docs') + self.assertEqual(all_docs_index['type'], 'special') + + json_index = indexes['indexes'][1] + self.assertEqual(json_index['ddoc'], '_design/ddoc001') + self.assertEqual(json_index['def']['fields'], [{'name': 'asc'}, {'age': 'asc'}]) + self.assertEqual(json_index['name'], 'json-idx-001') + self.assertEqual(json_index['type'], 'json') + + text_index = indexes['indexes'][2] + self.assertEqual(text_index['ddoc'], '_design/ddoc001') + self.assertEqual(text_index['def']['default_analyzer'], 'keyword') + self.assertEqual(text_index['def']['default_field'], {}) + self.assertEqual(text_index['def']['fields'], []) + self.assertEqual(text_index['def']['selector'], {}) + self.assertEqual(text_index['name'], 'text-idx-001') + self.assertEqual(text_index['type'], 'text') + self.assertTrue(text_index['def']['index_array_lengths']) def test_get_query_indexes(self): """ diff --git a/tests/unit/db_updates_tests.py b/tests/unit/db_updates_tests.py index 711b1937..0c83e75f 100644 --- a/tests/unit/db_updates_tests.py +++ b/tests/unit/db_updates_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,46 +16,96 @@ Unit tests for _db_updates feed """ -import unittest -from requests import Session import json import os +import unittest -from cloudant.feed import Feed -from cloudant.document import Document -from cloudant.design_document import DesignDocument -from cloudant.error import CloudantArgumentError, CloudantException from cloudant._2to3 import unicode_ +from cloudant.error import CloudantArgumentError +from cloudant.feed import Feed +from nose.plugins.attrib import attr +from requests import Session from .unit_t_db_base import UnitTestDbBase from .. import BYTETYPE + class DbUpdatesTestsBase(UnitTestDbBase): """ Common _db_updates tests methods """ - + def setUp(self): """ Set up test attributes """ super(DbUpdatesTestsBase, self).setUp() self.client.connect() + self.db_names = list() self.new_dbs = list() + if not self.is_couchdb_1x_version(): + self.create_db_updates() + self.create_dbs() def tearDown(self): """ Reset test attributes """ + test_dbs_deleted = False + changes = list() [db.delete() for db in self.new_dbs] + # Check the changes in the _db_updates feed to assert that the test databases are deleted + if not self.is_couchdb_1x_version(): + while not test_dbs_deleted and not self.is_couchdb_1x_version(): + feed = Feed(self.client, timeout=1000) + for change in feed: + if change['db_name'] in self.db_names and change['type'] == 'deleted': + changes.append(change) + if len(changes) == 2: + test_dbs_deleted = True + feed.stop() + self.delete_db_updates() self.client.disconnect() super(DbUpdatesTestsBase, self).tearDown() - def create_dbs(self, count=3): - self.new_dbs += [(self.client.create_database(self.dbname())) for x in range(count)] - -@unittest.skipIf(os.environ.get('RUN_CLOUDANT_TESTS'), - 'Skipping CouchDB _db_updates feed tests') + def create_dbs(self): + if not self.is_couchdb_1x_version(): + self.db_names = [self.dbname() for x in range(2)] + self.new_dbs += [self.client.create_database(dbname) for dbname in self.db_names] + # Verify that all created databases are listed in _db_updates + all_dbs_exist = False + while not all_dbs_exist: + changes = list() + feed = Feed(self.client, timeout=1000) + for change in feed: + changes.append(change) + if len(changes) == 3: + all_dbs_exist = True + feed.stop() + else: + self.new_dbs += [(self.client.create_database(self.dbname())) for x in range(3)] + + def assert_changes_in_db_updates_feed(self, changes): + """ + Assert that databases created in setup for db_updates_tests exist when looping through _db_updates feed + Note: During the creation of _global_changes database, a doc called '_dbs' is created and seen in _db_updates + """ + if not self.is_couchdb_1x_version(): + self.dbs = ['_dbs', self.new_dbs[0].database_name, self.new_dbs[1].database_name] + types = ['created', 'updated'] + for doc in changes: + self.assertIsNotNone(doc['seq']) + self.assertTrue(doc['db_name'] in self.dbs) + self.assertTrue(doc['type'] in types) + else: + self.assertDictEqual( + changes[0], {'db_name': self.new_dbs[0].database_name, 'type': 'created'}) + self.assertDictEqual( + changes[1], {'db_name': self.new_dbs[1].database_name, 'type': 'created'}) + self.assertDictEqual( + changes[2], {'db_name': self.new_dbs[2].database_name, 'type': 'created'}) + +@attr(db='couch') class CouchDbUpdatesTests(DbUpdatesTestsBase): """ CouchDB _db_updates feed unit tests @@ -80,45 +130,31 @@ def test_stop_iteration_of_continuous_feed_with_heartbeat(self): feed = Feed(self.client, feed='continuous', timeout=100) changes = list() for change in feed: - if not change: - if not self.new_dbs: - self.create_dbs(5) - else: - continue + if not change and self.is_couchdb_1x_version(): + self.create_dbs() else: changes.append(change) if len(changes) == 3: feed.stop() - self.assertEqual(len(self.new_dbs), 5) + self.assert_changes_in_db_updates_feed(changes) self.assertEqual(len(changes), 3) - self.assertDictEqual( - changes[0], {'db_name': self.new_dbs[0].database_name, 'type': 'created'}) - self.assertDictEqual( - changes[1], {'db_name': self.new_dbs[1].database_name, 'type': 'created'}) - self.assertDictEqual( - changes[2], {'db_name': self.new_dbs[2].database_name, 'type': 'created'}) def test_get_raw_content(self): """ Test getting raw feed content """ - feed = Feed(self.client, raw_data='True', feed='continuous', timeout=100) + feed = Feed(self.client, raw_data=True, feed='continuous', timeout=100) raw_content = list() for raw_line in feed: self.assertIsInstance(raw_line, BYTETYPE) - if not raw_line: - self.create_dbs(3) + if not raw_line and self.is_couchdb_1x_version(): + self.create_dbs() else: raw_content.append(raw_line) if len(raw_content) == 3: feed.stop() changes = [json.loads(unicode_(x)) for x in raw_content] - self.assertDictEqual( - changes[0], {'db_name': self.new_dbs[0].database_name, 'type': 'created'}) - self.assertDictEqual( - changes[1], {'db_name': self.new_dbs[1].database_name, 'type': 'created'}) - self.assertDictEqual( - changes[2], {'db_name': self.new_dbs[2].database_name, 'type': 'created'}) + self.assert_changes_in_db_updates_feed(changes) def test_get_longpoll_feed_as_default(self): """ @@ -126,11 +162,20 @@ def test_get_longpoll_feed_as_default(self): """ feed = Feed(self.client, timeout=1000) changes = list() - for change in feed: - self.assertIsNone(change) - changes.append(change) - self.assertEqual(len(changes), 1) - self.assertIsNone(changes[0]) + if self.is_couchdb_1x_version(): + for change in feed: + self.assertIsNone(change) + changes.append(change) + self.assertEqual(len(changes), 1) + self.assertIsNone(changes[0]) + else: + for change in feed: + self.assertIsNotNone(change) + changes.append(change) + if len(changes) == 3: + feed.stop() + self.assert_changes_in_db_updates_feed(changes) + self.assertEqual(len(changes), 3) def test_get_longpoll_feed_explicit(self): """ @@ -139,11 +184,20 @@ def test_get_longpoll_feed_explicit(self): """ feed = Feed(self.client, timeout=1000, feed='longpoll') changes = list() - for change in feed: - self.assertIsNone(change) - changes.append(change) - self.assertEqual(len(changes), 1) - self.assertIsNone(changes[0]) + if self.is_couchdb_1x_version(): + for change in feed: + self.assertIsNone(change) + changes.append(change) + self.assertEqual(len(changes), 1) + self.assertIsNone(changes[0]) + else: + for change in feed: + self.assertIsNotNone(change) + changes.append(change) + if len(changes) == 3: + feed.stop() + self.assert_changes_in_db_updates_feed(changes) + self.assertEqual(len(changes), 3) def test_get_continuous_with_timeout(self): """ @@ -151,7 +205,17 @@ def test_get_continuous_with_timeout(self): and no heartbeat """ feed = Feed(self.client, feed='continuous', heartbeat=False, timeout=1000) - self.assertListEqual([x for x in feed], []) + changes = list() + if self.is_couchdb_1x_version(): + self.assertListEqual([x for x in feed], []) + else: + for change in feed: + self.assertIsNotNone(change) + changes.append(change) + if len(changes) == 3: + feed.stop() + self.assert_changes_in_db_updates_feed(changes) + self.assertEqual(len(changes), 3) def test_invalid_argument(self): """ @@ -212,8 +276,8 @@ def test_invalid_feed_value(self): self.assertTrue(str(cm.exception).startswith( 'Invalid value (normal) for feed option.')) -@unittest.skipIf(not os.environ.get('RUN_CLOUDANT_TESTS') or - os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') +@attr(db='cloudant') +@unittest.skipIf(os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') class CloudantDbUpdatesTests(DbUpdatesTestsBase): """ Cloudant _db_updates feed unit tests @@ -249,7 +313,6 @@ def test_stop_iteration_of_continuous_feed_using_since_now(self): feed = Feed(self.client, feed='continuous', since='now') count = 0 changes = list() - self.create_dbs(3) for change in feed: self.assertTrue(all(x in change for x in ('seq', 'type'))) changes.append(change) @@ -281,7 +344,6 @@ def test_get_normal_feed_default(self): Test getting content back for a "normal" feed without feed option. Also using limit since we don't know how many updates have occurred on client. """ - self.create_dbs(3) feed = Feed(self.client, limit=3) changes = list() for change in feed: @@ -296,7 +358,6 @@ def test_get_normal_feed_explicit(self): Test getting content back for a "normal" feed using feed option. Also using limit since we don't know how many updates have occurred on client. """ - self.create_dbs(3) feed = Feed(self.client, feed='normal', limit=3) changes = list() for change in feed: @@ -310,7 +371,6 @@ def test_get_longpoll_feed(self): """ Test getting content back for a "longpoll" feed """ - self.create_dbs(3) feed = Feed(self.client, feed='longpoll', limit=3) changes = list() for change in feed: diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index 44ede8fa..86a2b5f1 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2020 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,15 +24,18 @@ import json import os import unittest + import mock import requests - -from cloudant.document import Document +from cloudant._common_util import response_to_json_dict from cloudant.design_document import DesignDocument -from cloudant.view import View, QueryIndexView +from cloudant.document import Document from cloudant.error import CloudantArgumentError, CloudantDesignDocumentException +from cloudant.view import View, QueryIndexView +from nose.plugins.attrib import attr + +from .unit_t_db_base import UnitTestDbBase, skip_if_iam -from .unit_t_db_base import UnitTestDbBase class CloudantDesignDocumentExceptionTests(unittest.TestCase): """ @@ -72,6 +75,7 @@ def test_raise_with_proper_code_and_args(self): raise CloudantDesignDocumentException(104, 'foo') self.assertEqual(cm.exception.status_code, 104) +@attr(db=['cloudant','couch']) class DesignDocumentTests(UnitTestDbBase): """ DesignDocument unit tests @@ -349,6 +353,7 @@ def test_fetch_map_reduce(self): self.assertEqual(ddoc_remote, { '_id': '_design/ddoc001', '_rev': ddoc['_rev'], + 'options': {'partitioned': False}, 'lists': {}, 'shows': {}, 'indexes': {}, @@ -360,10 +365,7 @@ def test_fetch_map_reduce(self): self.assertIsInstance(ddoc_remote['views']['view001'], View) self.assertIsInstance(ddoc_remote['views']['view003'], View) - @unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant fetch dbcopy test' - ) + @attr(db='cloudant') def test_fetch_dbcopy(self): """ Ensure that the document fetch from the database returns the @@ -388,7 +390,8 @@ def test_fetch_dbcopy(self): # before comparison also. Compare the removed values with # the expected content in each case. self.assertEqual(db_copy, ddoc['views']['view002'].pop('dbcopy')) - self.assertEqual({'epi': {'dbcopy': {'view002': db_copy}}}, ddoc_remote.pop('options')) + self.assertEqual({'epi': {'dbcopy': {'view002': db_copy}}, 'partitioned': False}, ddoc_remote.pop('options')) + self.assertEqual({'partitioned': False}, ddoc.pop('options')) self.assertEqual(ddoc_remote, ddoc) self.assertTrue(ddoc_remote['_rev'].startswith('1-')) self.assertEqual(ddoc_remote, { @@ -414,7 +417,7 @@ def test_fetch_no_views(self): ddoc_remote = DesignDocument(self.db, '_design/ddoc001') ddoc_remote.fetch() self.assertEqual(set(ddoc_remote.keys()), - {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + {'_id', '_rev', 'indexes', 'views', 'options', 'lists', 'shows'}) self.assertEqual(ddoc_remote['_id'], '_design/ddoc001') self.assertTrue(ddoc_remote['_rev'].startswith('1-')) self.assertEqual(ddoc_remote['_rev'], ddoc['_rev']) @@ -431,6 +434,7 @@ def test_fetch_query_views(self): data = { '_id': '_design/ddoc001', 'indexes': {}, + 'options': {'partitioned': False}, 'lists': {}, 'shows': {}, 'language': 'query', @@ -462,6 +466,7 @@ def test_fetch_text_indexes(self): data = { '_id': '_design/ddoc001', 'language': 'query', + 'options': {'partitioned': False}, 'lists': {}, 'shows': {}, 'indexes': {'index001': @@ -498,6 +503,7 @@ def test_fetch_text_indexes_and_query_views(self): 'language': 'query', 'lists': {}, 'shows': {}, + 'options': {'partitioned': False}, 'views': { 'view001': {'map': {'fields': {'name': 'asc', 'age': 'asc'}}, 'reduce': '_count', @@ -686,15 +692,15 @@ def test_save_with_no_views(self): ddoc.save() # Ensure that locally cached DesignDocument contains an # empty views dict. - self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'indexes', 'options', 'views', 'lists', 'shows'}) self.assertEqual(ddoc['_id'], '_design/ddoc001') self.assertTrue(ddoc['_rev'].startswith('1-')) self.assertEqual(ddoc.views, {}) # Ensure that remotely saved design document does not # include a views sub-document. resp = self.client.r_session.get(ddoc.document_url) - raw_ddoc = resp.json() - self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) + raw_ddoc = response_to_json_dict(resp) + self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev','options'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -775,7 +781,10 @@ def test_get_info(self): info = ddoc_remote.info() # Remove variable fields to make equality easier to check info['view_index'].pop('signature') - info['view_index'].pop('disk_size') + if 'disk_size' in info['view_index']: + info['view_index'].pop('disk_size') + if 'data_size' in info['view_index']: + info['view_index'].pop('data_size') # Remove Cloudant/Couch 2 fields if present to allow test to pass on Couch 1.6 if 'sizes' in info['view_index']: info['view_index'].pop('sizes') @@ -788,8 +797,7 @@ def test_get_info(self): {'view_index': {'update_seq': 0, 'waiting_clients': 0, 'language': 'javascript', 'purge_seq': 0, 'compact_running': False, - 'waiting_commit': False, 'updater_running': False, - 'data_size': 0 + 'waiting_commit': False, 'updater_running': False }, 'name': name }) @@ -810,10 +818,7 @@ def test_get_info_raises_httperror(self): self.client.r_session.get.assert_called_with( '/'.join([ddoc.document_url, '_info'])) - @unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant _search_info endpoint test' - ) + @attr(db='cloudant') def test_get_search_info(self): """ Test retrieval of search_info endpoint from the DesignDocument. @@ -829,22 +834,60 @@ def test_get_search_info(self): ddoc_remote = DesignDocument(self.db, '_design/ddoc001') ddoc_remote.fetch() + # Make a request to the search index to ensure it is built + self.db.get_search_result('_design/ddoc001', 'search001', query='name:julia*') + search_info = ddoc_remote.search_info('search001') # Check the search index name self.assertEqual(search_info['name'], '_design/ddoc001/search001', 'The search index name should be correct.') # Validate the metadata search_index_metadata = search_info['search_index'] self.assertIsNotNone(search_index_metadata) - self.assertEquals(search_index_metadata['doc_del_count'], 0, 'There should be no deleted docs.') + self.assertEqual(search_index_metadata['doc_del_count'], 0, 'There should be no deleted docs.') self.assertTrue(search_index_metadata['doc_count'] <= 100, 'There should be 100 or fewer docs.') - self.assertEquals(search_index_metadata['committed_seq'], 0, 'The committed_seq should be 0.') + self.assertEqual(search_index_metadata['committed_seq'], 0, 'The committed_seq should be 0.') self.assertTrue(search_index_metadata['pending_seq'] <= 101, 'The pending_seq should be 101 or fewer.') self.assertTrue(search_index_metadata['disk_size'] >0, 'The disk_size should be greater than 0.') - @unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant _search_info raises HTTPError test' - ) + @attr(db='cloudant') + def test_get_search_disk_size(self): + """ + Test retrieval of search_disk_size endpoint from the DesignDocument. + """ + self.populate_db_with_documents(100) + ddoc = DesignDocument(self.db, '_design/ddoc001') + ddoc.add_search_index( + 'search001', + 'function (doc) {\n index("default", doc._id); ' + 'if (doc._id) {index("name", doc.name, {"store": true}); }\n}' + ) + ddoc.save() + + ddoc_remote = DesignDocument(self.db, '_design/ddoc001') + ddoc_remote.fetch() + + # Make a request to the search index to ensure it is built + self.db.get_search_result('_design/ddoc001', 'search001', query='name:julia*') + + search_disk_size = ddoc_remote.search_disk_size('search001') + + self.assertEqual( + sorted(search_disk_size.keys()), ['name', 'search_index'], + 'The search disk size should contain only keys "name" and "search_index"') + self.assertEqual( + search_disk_size['name'], '_design/ddoc001/search001', + 'The search index "name" should be correct.') + self.assertEqual( + sorted(search_disk_size['search_index'].keys()), ['disk_size'], + 'The search index should contain only key "disk_size"') + self.assertTrue( + isinstance(search_disk_size['search_index']['disk_size'], int), + 'The "disk_size" value should be an integer.') + self.assertTrue( + search_disk_size['search_index']['disk_size'] > 0, + 'The "disk_size" should be greater than 0.') + + @attr(db='cloudant') def test_get_search_info_raises_httperror(self): """ Test get_search_info raises an HTTPError. @@ -1038,6 +1081,7 @@ def test_fetch_search_index(self): self.assertEqual(ddoc_remote, { '_id': '_design/ddoc001', '_rev': ddoc['_rev'], + 'options': {'partitioned': False}, 'indexes': { 'search001': {'index': search_index}, 'search002': {'index': search_index, 'analyzer': 'simple'}, @@ -1062,7 +1106,7 @@ def test_fetch_no_search_index(self): ddoc_remote = DesignDocument(self.db, '_design/ddoc001') ddoc_remote.fetch() self.assertEqual(set(ddoc_remote.keys()), - {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + {'_id', '_rev', 'indexes', 'options', 'views', 'lists', 'shows'}) self.assertEqual(ddoc_remote['_id'], '_design/ddoc001') self.assertTrue(ddoc_remote['_rev'].startswith('1-')) self.assertEqual(ddoc_remote['_rev'], ddoc['_rev']) @@ -1145,14 +1189,14 @@ def test_save_with_no_search_indexes(self): ddoc.save() # Ensure that locally cached DesignDocument contains an # empty search indexes and views dict. - self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'indexes','options', 'views', 'lists', 'shows'}) self.assertEqual(ddoc['_id'], '_design/ddoc001') self.assertTrue(ddoc['_rev'].startswith('1-')) # Ensure that remotely saved design document does not # include a search indexes sub-document. resp = self.client.r_session.get(ddoc.document_url) - raw_ddoc = resp.json() - self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) + raw_ddoc = response_to_json_dict(resp) + self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev','options'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -1212,6 +1256,7 @@ def test_get_search_index(self): '{"store": true}); }\n}'} ) + @skip_if_iam def test_rewrite_rule(self): """ Test that design document URL is rewritten to the expected test document. @@ -1230,8 +1275,8 @@ def test_rewrite_rule(self): doc = Document(self.db, 'rewrite_doc') doc.save() resp = self.client.r_session.get('/'.join([ddoc.document_url, '_rewrite'])) - self.assertEquals( - resp.json(), + self.assertEqual( + response_to_json_dict(resp), { '_id': 'rewrite_doc', '_rev': doc['_rev'] @@ -1374,6 +1419,7 @@ def test_fetch_list_functions(self): self.assertEqual(ddoc_remote, { '_id': '_design/ddoc001', '_rev': ddoc['_rev'], + 'options': {'partitioned': False}, 'lists': { 'list001': list_func, 'list002': list_func, @@ -1398,7 +1444,7 @@ def test_fetch_no_list_functions(self): ddoc_remote = DesignDocument(self.db, '_design/ddoc001') ddoc_remote.fetch() self.assertEqual(set(ddoc_remote.keys()), - {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + {'_id', '_rev', 'options', 'indexes', 'views', 'lists', 'shows'}) self.assertEqual(ddoc_remote['_id'], '_design/ddoc001') self.assertTrue(ddoc_remote['_rev'].startswith('1-')) self.assertEqual(ddoc_remote['_rev'], ddoc['_rev']) @@ -1413,14 +1459,14 @@ def test_save_with_no_list_functions(self): ddoc = DesignDocument(self.db, '_design/ddoc001') ddoc.save() # Ensure that locally cached DesignDocument contains lists dict - self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'lists', 'shows', 'indexes', 'views'}) + self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'lists', 'options', 'shows', 'indexes', 'views'}) self.assertEqual(ddoc['_id'], '_design/ddoc001') self.assertTrue(ddoc['_rev'].startswith('1-')) # Ensure that remotely saved design document does not # include a lists sub-document. resp = self.client.r_session.get(ddoc.document_url) - raw_ddoc = resp.json() - self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) + raw_ddoc = response_to_json_dict(resp) + self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev','options'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -1484,10 +1530,7 @@ def test_get_list_function(self): 'html += \'\'; return html; }); }' ) - @unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant specific Cloudant Geo tests' - ) + @attr(db='cloudant') def test_geospatial_index(self): """ Test retrieval and query of Cloudant Geo indexes from the DesignDocument. @@ -1511,7 +1554,8 @@ def test_geospatial_index(self): 'indexes': {}, 'views': {}, 'lists': {}, - 'shows': {} + 'shows': {}, + 'options': {'partitioned': False} }) # Document with geospatial point geodoc = Document(self.db, 'doc001') @@ -1678,6 +1722,7 @@ def test_fetch_show_functions(self): self.assertEqual(ddoc_remote, { '_id': '_design/ddoc001', '_rev': ddoc['_rev'], + 'options': {'partitioned': False}, 'lists': {}, 'shows': { 'show001': show_func, @@ -1702,7 +1747,7 @@ def test_fetch_no_show_functions(self): ddoc_remote = DesignDocument(self.db, '_design/ddoc001') ddoc_remote.fetch() self.assertEqual(set(ddoc_remote.keys()), - {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + {'_id', '_rev', 'indexes', 'options', 'views', 'lists', 'shows'}) self.assertEqual(ddoc_remote['_id'], '_design/ddoc001') self.assertTrue(ddoc_remote['_rev'].startswith('1-')) self.assertEqual(ddoc_remote['_rev'], ddoc['_rev']) @@ -1717,14 +1762,14 @@ def test_save_with_no_show_functions(self): ddoc = DesignDocument(self.db, '_design/ddoc001') ddoc.save() # Ensure that locally cached DesignDocument contains shows dict - self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'lists', 'shows', 'indexes', 'views'}) + self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'lists','options', 'shows', 'indexes', 'views'}) self.assertEqual(ddoc['_id'], '_design/ddoc001') self.assertTrue(ddoc['_rev'].startswith('1-')) # Ensure that remotely saved design document does not # include a shows sub-document. resp = self.client.r_session.get(ddoc.document_url) - raw_ddoc = resp.json() - self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) + raw_ddoc = response_to_json_dict(resp) + self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev','options'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -1805,7 +1850,7 @@ def test_update_validator(self): data=json.dumps({'_id': 'test001'}) ) self.assertEqual( - resp.json(), + response_to_json_dict(resp), {'reason': 'Document must have an address.', 'error': 'forbidden'} ) diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index 2d76389f..09cb09c4 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import cloudant """ _document_tests_ @@ -22,20 +23,22 @@ """ -import unittest -import mock -import posixpath +import inspect import json -import requests import os +import unittest import uuid -import inspect +from datetime import datetime +import mock +import requests from cloudant.document import Document from cloudant.error import CloudantDocumentException +from nose.plugins.attrib import attr -from .. import StringIO, unicode_ from .unit_t_db_base import UnitTestDbBase +from .. import StringIO, unicode_ + def find_fixture(name): import tests.unit.fixtures as fixtures @@ -81,6 +84,7 @@ def test_raise_with_proper_code_and_args(self): raise CloudantDocumentException(102, 'foo') self.assertEqual(cm.exception.status_code, 102) +@attr(db=['cloudant','couch']) class DocumentTests(UnitTestDbBase): """ Document unit tests @@ -115,7 +119,7 @@ def test_document_url(self): """ doc = Document(self.db, 'julia006') self.assertEqual( - doc.document_url, posixpath.join(self.db.database_url, 'julia006') + doc.document_url, '/'.join((self.db.database_url, 'julia006')) ) def test_document_url_encodes_correctly(self): @@ -124,10 +128,8 @@ def test_document_url_encodes_correctly(self): """ doc = Document(self.db, 'http://example.com') self.assertEqual( - doc.document_url, posixpath.join( - self.db.database_url, - 'http%3A%2F%2Fexample.com' - ) + doc.document_url, + '/'.join((self.db.database_url, 'http%3A%2F%2Fexample.com')) ) def test_design_document_url(self): @@ -137,10 +139,8 @@ def test_design_document_url(self): """ doc = Document(self.db, '_design/ddoc001') self.assertEqual( - doc.document_url, posixpath.join( - self.db.database_url, - '_design/ddoc001' - ) + doc.document_url, + '/'.join((self.db.database_url, '_design/ddoc001')) ) def test_design_document_url_encodes_correctly(self): @@ -149,10 +149,8 @@ def test_design_document_url_encodes_correctly(self): """ doc = Document(self.db, '_design/http://example.com') self.assertEqual( - doc.document_url, posixpath.join( - self.db.database_url, - '_design/http%3A%2F%2Fexample.com' - ) + doc.document_url, + '/'.join((self.db.database_url, '_design/http%3A%2F%2Fexample.com')) ) def test_constructor_without_docid(self): @@ -313,9 +311,9 @@ def test_appended_error_message_using_save_with_invalid_key(self): err = cm.exception # Should be a 400 error code, but CouchDB 1.6 issues a 500 if err.response.status_code == 500: - #Check this is CouchDB 1.6 - self.assertTrue(self.client.r_session.head(self.url).headers['Server'].find('CouchDB/1.6.') >= 0, - '500 returned but was not CouchDB 1.6.x') + # Check this is CouchDB 1.x + self.assertTrue(self.client.r_session.head(self.url).headers['Server'].find('CouchDB/1.') >= 0, + '500 returned but was not CouchDB 1.x') self.assertEqual( str(err.response.reason), 'Internal Server Error doc_validation Bad special document member: _invalid_key' @@ -481,6 +479,71 @@ def test_update_field(self): self.assertTrue(doc['_rev'].startswith('2-')) self.assertEqual(doc['pets'], ['cat', 'dog', 'fish']) + @mock.patch('cloudant.document.Document.save') + def test_update_field_maxretries(self, m_save): + """ + Test that conflict retries work for updating a single field. + """ + # Create a doc + doc = Document(self.db, 'julia006') + doc['name'] = 'julia' + doc['age'] = 6 + doc.create() + self.assertTrue(doc['_rev'].startswith('1-')) + self.assertEqual(doc['age'], 6) + # Mock conflicts when saving updates + m_save.side_effect = requests.HTTPError(response=mock.Mock(status_code=409, reason='conflict')) + # Tests that failing on retry eventually throws + with self.assertRaises(requests.HTTPError) as cm: + doc.update_field(doc.field_set, 'age', 7, max_tries=2) + + # There is an off-by-one error for "max_tries" + # It really means max_retries i.e. 1 attempt + # followed by a max of 2 retries + self.assertEqual(m_save.call_count, 3) + self.assertEqual(cm.exception.response.status_code, 409) + self.assertEqual(cm.exception.response.reason, 'conflict') + # Fetch again before asserting, otherwise we assert against + # the locally updated age field + doc.fetch() + self.assertFalse(doc['_rev'].startswith('2-')) + self.assertNotEqual(doc['age'], 7) + + def test_update_field_success_on_retry(self): + """ + Test that conflict retries work for updating a single field. + """ + # Create a doc + doc = Document(self.db, 'julia006') + doc['name'] = 'julia' + doc['age'] = 6 + doc.create() + self.assertTrue(doc['_rev'].startswith('1-')) + self.assertEqual(doc['age'], 6) + + # Mock when saving the document + # 1st call throw a 409 + # 2nd call delegate to the real doc.save() + + class SaveMock(object): + calls = 0 + def save(self): + if self.calls == 0: + self.calls += 1 + raise requests.HTTPError(response=mock.Mock(status_code=409, reason='conflict')) + else: + return cloudant.document.Document.save(doc) + + with mock.patch.object(doc, 'save', + side_effect=SaveMock().save) as m_save: + # A list of side effects containing only 1 element + doc.update_field(doc.field_set, 'age', 7, max_tries=1) + # Two calls to save, one with a 409 and one that succeeds + self.assertEqual(m_save.call_count, 2) + # Check that the _rev and age field were updated + self.assertTrue(doc['_rev'].startswith('2-')) + self.assertEqual(doc['age'], 7) + def test_delete_document_failure(self): """ Test failure condition when attempting to remove a document @@ -547,16 +610,78 @@ def test_document_context_manager(self): self.assertTrue(doc['_rev'].startswith('2-')) self.assertEqual(self.db['julia006'], doc) + def test_document_context_manager_no_doc_id(self): + """ + Test that the __enter__ and __exit__ methods perform as expected + with no document id when initiated through a document context manager + """ + with Document(self.db) as doc: + doc['_id'] = 'julia006' + doc['name'] = 'julia' + doc['age'] = 6 + self.assertTrue(doc['_rev'].startswith('1-')) + self.assertEqual(self.db['julia006'], doc) + + def test_document_context_manager_creation_failure_on_error(self): + """ + Test that the document context manager skips document creation if there + is an error. + """ + with self.assertRaises(ZeroDivisionError), Document(self.db, 'julia006') as doc: + doc['name'] = 'julia' + doc['age'] = 6 + raise ZeroDivisionError() + + doc = Document(self.db, 'julia006') + try: + doc.fetch() + except requests.HTTPError as err: + self.assertEqual(err.response.status_code, 404) + else: + self.fail('Above statement should raise a HTTPError.') + + def test_document_context_manager_update_failure_on_error(self): + """ + Test that the document context manager skips document update if there + is an error. + """ + # Create the document. + doc = Document(self.db, 'julia006') + doc['name'] = 'julia' + doc['age'] = 6 + doc.save() + + # Make a document update and then raise an error. + with self.assertRaises(ZeroDivisionError), Document(self.db, 'julia006') as doc: + doc['age'] = 7 + raise ZeroDivisionError() + + # Assert the change persists locally. + self.assertEqual(doc['age'], 7) + + # Assert the document has not been saved to remote server. + self.assertTrue(doc['_rev'].startswith('1-')) + self.assertEqual(self.db['julia006']['age'], 6) + + def test_document_context_manager_doc_create(self): + """ + Test that the document context manager will create a doc if it does + not yet exist. + """ + with Document(self.db, 'julia006') as doc: + doc['name'] = 'julia' + doc['age'] = 6 + self.assertTrue(doc['_rev'].startswith('1-')) + self.assertEqual(self.db['julia006'], doc) + def test_setting_id(self): """ Ensure that proper processing occurs when setting the _id """ doc = Document(self.db) self.assertIsNone(doc.get('_id')) - self.assertEqual(doc._document_id, None) doc['_id'] = 'julia006' self.assertEqual(doc['_id'], 'julia006') - self.assertEqual(doc._document_id, 'julia006') def test_removing_id(self): """ @@ -566,7 +691,6 @@ def test_removing_id(self): doc['_id'] = 'julia006' del doc['_id'] self.assertIsNone(doc.get('_id')) - self.assertEqual(doc._document_id, None) def test_get_text_attachment(self): """ @@ -787,5 +911,47 @@ def test_document_request_fails_after_client_disconnects(self): finally: self.client.connect() + def test_document_custom_json_encoder_and_decoder(self): + dt_format = '%Y-%m-%dT%H:%M:%S' + + class DTEncoder(json.JSONEncoder): + + def default(self, obj): + if isinstance(obj, datetime): + return { + '_type': 'datetime', + 'value': obj.strftime(dt_format) + } + return super(DTEncoder, self).default(obj) + + class DTDecoder(json.JSONDecoder): + + def __init__(self, *args, **kwargs): + json.JSONDecoder.__init__(self, object_hook=self.object_hook, + *args, **kwargs) + + def object_hook(self, obj): + if '_type' not in obj: + return obj + if obj['_type'] == 'datetime': + return datetime.strptime(obj['value'], dt_format) + return obj + + doc = Document(self.db, encoder=DTEncoder) + doc['name'] = 'julia' + doc['dt'] = datetime(2018, 7, 9, 15, 11, 10, 0) + doc.save() + + raw_doc = self.db.all_docs(include_docs=True)['rows'][0]['doc'] + + self.assertEqual(raw_doc['name'], 'julia') + self.assertEqual(raw_doc['dt']['_type'], 'datetime') + self.assertEqual(raw_doc['dt']['value'], '2018-07-09T15:11:10') + + doc2 = Document(self.db, doc['_id'], decoder=DTDecoder) + doc2.fetch() + + self.assertEqual(doc2['dt'], doc['dt']) + if __name__ == '__main__': unittest.main() diff --git a/tests/unit/document_validation_tests.py b/tests/unit/document_validation_tests.py new file mode 100644 index 00000000..021d8d6f --- /dev/null +++ b/tests/unit/document_validation_tests.py @@ -0,0 +1,909 @@ +#!/usr/bin/env python +# Copyright © 2021 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +from enum import Enum +from unittest.mock import Mock, patch + +from mock import create_autospec + +import requests +from urllib.parse import urlparse + +from cloudant import database +from cloudant.design_document import DesignDocument +from cloudant.document import Document +from cloudant.error import CloudantArgumentError + +class ValidationExceptionMsg(Enum): + DOC = 'Invalid document ID:' + ATTACHMENT = 'Invalid attachment name:' + +class Expect(Enum): + VALIDATION_EXCEPTION_DOCID = CloudantArgumentError(137, '') + VALIDATION_EXCEPTION_ATT = CloudantArgumentError(138, '') + RESPONSE_404 = 404 + RESPONSE_200 = 200 + RESPONSE_201 = 201 + + +class ValidationTests(unittest.TestCase): + """ + Document validation unit tests + """ + def setUp(self): + self.doc_r_session_patcher = patch('cloudant.document.Document.r_session') + self.requests_get_patcher = patch('requests.get') + + self.addCleanup(patch.stopall) + + self.doc_r_session_mock = self.doc_r_session_patcher.start() + self.requests_get_mock = self.requests_get_patcher.start() + + self.db = create_autospec(database) + self.db.client = Mock() + self.db.client.server_url = 'http://mocked.url.com' + self.db.database_url = 'http://mocked.url.com/my_db' + self.db.database_name = 'mydb' + + def teardown(self): + self.addCleanup(patch.stopall) + del self.db + del self.doc_r_session_patcher + del self.requests_get_patcher + del self.doc_r_session_mock + del self.requests_get_mock + + # GET and HEAD _all_docs + # EXPECTED: validation failure + def test_get_invalid_all_docs(self): + """ + Test GET/HEAD request for invalid '_all_docs' document ID + """ + self.get_document_variants('_all_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _design/foo + # EXPECTED: 200 + def test_get_valid_ddoc(self): + """ + Test GET/HEAD request for valid '_design/foo' document ID + """ + self.get_document_variants('_design/foo', Expect.RESPONSE_200.value, path_segment_count=3) + self.get_document_variants('_design/foo', Expect.RESPONSE_200.value, True, path_segment_count=3) + + # GET and HEAD _design + # EXPECTED: Validation exception + def test_get_invalid_design(self): + """ + Test GET/HEAD request for invalid '_design' document ID + """ + self.get_document_variants('_design', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_document_variants('_design', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # GET and HEAD /_design/foo with a slash + # EXPECTED: 404 + def test_get_missing_ddoc_with_slash(self): + """ + Test GET/HEAD request for missing '/_design/foo' document ID + """ + self.get_document_variants('/_design/foo', Expect.RESPONSE_404.value, path_segment_count=2) + + # GET and HEAD _design/foo/_view/bar + # EXPECTED: 404 + def test_get_invalid_view(self): + """ + Test GET/HEAD request for missing '_design/foo' document ID + """ + self.get_document_variants('_design/foo/_view/bar', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_view/bar', Expect.RESPONSE_404.value, True, path_segment_count=3) + + # GET and HEAD _design/foo/_info + # EXPECTED: 404 + def test_get_invalid_view_info(self): + """ + Test GET/HEAD request for missing '_design/foo/_info' document ID + """ + self.get_document_variants('_design/foo/_info', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_info', Expect.RESPONSE_404.value, True, path_segment_count=3) + + # GET and HEAD _design/foo/_search/bar + # EXPECTED: 404 + def test_get_invalid_search(self): + """ + Test GET/HEAD request for missing '_design/foo/_search/bar' document ID + """ + self.get_document_variants('_design/foo/_search/bar', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_search/bar', Expect.RESPONSE_404.value, True, path_segment_count=3) + + # GET and HEAD _design/foo/_search_info/bar + # EXPECTED: 404 + def test_get_invalid_search_info(self): + """ + Test GET/HEAD request for missing '_design/foo/_search_info/bar' document ID + """ + self.get_document_variants('_design/foo/_search_info/bar', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_search_info/bar', Expect.RESPONSE_404.value, True, path_segment_count=3) + + # GET and HEAD _design/foo/_geo/bar + # EXPECTED: 404 + def test_get_missing_geo(self): + """ + Test GET/HEAD request for missing '_design/foo/_geo/bar' document ID + """ + self.get_document_variants('_design/foo/_geo/bar', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_geo/bar', Expect.RESPONSE_404.value, True, path_segment_count=3) + # with a parameter + self.get_document_variants('_design/foo/_geo/bar?bbox=-50.52,-4.46,54.59,1.45', Expect.RESPONSE_404.value, + path_segment_count=3) + self.get_document_variants('_design/foo/_geo/bar?bbox=-50.52,-4.46,54.59,1.45', Expect.RESPONSE_404.value, True, + path_segment_count=3) + + # GET and HEAD _design/foo/_geo_info/bar + # EXPECTED: 404 + def test_get_missing_geo_info(self): + """ + Test GET/HEAD request for missing '_design/foo/_geo_info/bar' document ID + """ + self.get_document_variants('_design/foo/_geo_info/bar', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_geo_info/bar', Expect.RESPONSE_404.value, True, path_segment_count=3) + + # GET and HEAD _local/foo + # EXPECTED: 200 + def test_get_local_doc(self): + """ + Test GET/HEAD request for valid '_local/foo' document ID + """ + self.get_document_variants('_local/foo', Expect.RESPONSE_200.value, path_segment_count=3) + + # GET and HEAD _local + # EXPECTED: Validation exception + def test_get_invalid_local(self): + """ + Test GET/HEAD request for invalid '_local' document ID + """ + self.get_document_variants('_local', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _local_docs + # EXPECTED: Validation exception + def test_get_invalid_local_docs(self): + """ + Test GET/HEAD request for invalid '_local_docs' document ID + """ + self.get_document_variants('_local_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _design_docs + # EXPECTED: Validation exception + def test_get_invalid_design_docs(self): + """ + Test GET/HEAD request for invalid '_design_docs' document ID + """ + self.get_document_variants('_design_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _changes + # EXPECTED: Validation exception + def test_get_invalid_changes(self): + """ + Test GET/HEAD request for invalid '_changes' document ID + """ + self.get_document_variants('_changes', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _ensure_full_commit + # EXPECTED: Validation exception + def test_get_invalid_ensure_full_commit(self): + """ + Test GET/HEAD request for invalid '_ensure_full_commit' document ID + """ + self.get_document_variants('_ensure_full_commit', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _index + # EXPECTED: Validation exception + def test_get_invalid_index(self): + """ + Test GET/HEAD request for invalid '_index' document ID + """ + self.get_document_variants('_index', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _revs_limit + # EXPECTED: Validation exception + def test_get_invalid_revs_limit(self): + """ + Test GET/HEAD request for invalid '_revs_limit' document ID + """ + self.get_document_variants('_revs_limit', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _security + # EXPECTED: Validation exception + def test_get_invalid_security(self): + """ + Test GET/HEAD request for invalid '_security' document ID + """ + self.get_document_variants('_security', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _shards + # EXPECTED: Validation exception + def test_get_invalid_shards(self): + """ + Test GET/HEAD request for invalid '_shards' document ID + """ + self.get_document_variants('_shards', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # DELETE _index/_design/foo/json/bar + # EXPECTED: Validation exception + def test_delete_invalid_index(self): + """ + Test DELETE request for invalid '_index/_design/foo/json/bar' document ID + """ + self.delete_document_variants('_index/_design/foo/json/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # DELETE _design/foo + # EXPECTED: 200 + def test_delete_valid_ddoc(self): + """ + Test DELETE request for valid '_design/foo' document ID + """ + self.delete_document_variants('_design/foo', Expect.RESPONSE_200.value, path_segment_count=3) + + # DELETE _design + # EXPECTED: Validation exception + def test_delete_invalid_ddoc(self): + """ + Test DELETE request for invalid '_design' document ID + """ + # no trailing '/' on _design prefix + self.delete_document_variants('_design', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # DELETE _local/foo + # EXPECTED: 200 + def test_delete_valid_local_doc(self): + """ + Test DELETE request for valid '_local/foo' document ID + """ + self.delete_document_variants('_local/foo', Expect.RESPONSE_200.value, path_segment_count=3) + + # DELETE _local + # EXPECTED: Validation exception + def test_delete_invalid_local(self): + """ + Test DELETE request for invalid '_local' document ID + """ + # no trailing '/' on _local prefix + self.delete_document_variants('_local', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # PUT _design/foo + # EXPECTED: 201 + def test_put_valid_ddoc(self): + """ + Test PUT request for valid '_design/foo' document ID + """ + self.put_document_variants('_design/foo', Expect.RESPONSE_201.value, path_segment_count=3) + + # PUT _design + # EXPECTED: Validation exception + def test_put_invalid_ddoc(self): + """ + Test PUT request for invalid '_design' document ID + """ + self.put_document_variants('_design', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # PUT _local/foo + # EXPECTED: 201 + def test_put_valid_local_doc(self): + """ + Test PUT request for valid '_local/foo' document ID + """ + self.put_document_variants('_local/foo', Expect.RESPONSE_201.value, path_segment_count=3) + + # PUT _local + # EXPECTED: Validation exception + def test_put_invalid_local_doc(self): + """ + Test PUT request for invalid '_local' document ID + """ + self.put_document_variants('_local', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # PUT _revs_limit + # EXPECTED: Validation exception + def test_put_invalid_revs_limit(self): + """ + Test PUT request for invalid '_revs_limit' document ID + """ + self.put_document_variants('_revs_limit', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # PUT _security + # EXPECTED: Validation exception + def test_put_invalid_security(self): + """ + Test PUT request for invalid '_security' document ID + """ + self.put_document_variants('_security', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET _design/foo/bar + # EXPECTED: 200 + def test_get_valid_ddoc_attachment(self): + """ + Test PUT request for valid '_design/foo/bar' document ID + """ + self.get_doc_attachment_variants('_design/foo', 'bar', Expect.RESPONSE_200.value, True, path_segment_count=4) + + # PUT _design/foo/bar + # EXPECTED: 201 + def test_put_valid_ddoc_attachment(self): + """ + Test PUT request for valid '_design/foo/bar' document ID + """ + self.put_doc_attachment_variants('_design/foo', 'bar', Expect.RESPONSE_201.value, True, path_segment_count=4) + + # DELETE _design/foo/bar + # EXPECTED: 200 + def test_delete_valid_ddoc_attachment(self): + """ + Test DELETE request for valid '_design/foo/bar' document ID + """ + self.delete_doc_attachment_variants('_design/foo', 'bar', Expect.RESPONSE_200.value, True, path_segment_count=4) + + # GET _design/foo + # EXPECTED: Validation exception + def test_get_invalid_ddoc_attachment(self): + """ + Test GET request for invalid '_design/foo' document ID + """ + # with ddoc option enabled + self.get_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.get_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # PUT _design/foo + # EXPECTED: Validation exception + def test_put_invalid_ddoc_attachment(self): + """ + Test PUT request for invalid '_design/foo' document ID + """ + # with ddoc option enabled + self.put_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.put_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # DELETE _design/foo + # EXPECTED: Validation exception + def test_delete_invalid_ddoc_attachment(self): + """ + Test DELETE request for invalid '_design/foo' document ID + """ + # with ddoc option enabled + self.delete_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.delete_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # DELETE _index/_design/foo/json/bar + # EXPECTED: Validation exception + def test_delete_index_via_attachment(self): + """ + Test DELETE requests for invalid '_index/_design/foo/json/bar' + """ + self.delete_doc_attachment_variants('_index', '_design/foo/json/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_index', '_design/foo/json/bar', + Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.delete_doc_attachment_variants('_index/_design', 'foo/json/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_index/_design', 'foo/json/bar', + Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.delete_doc_attachment_variants('_index/_design/foo', 'json/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_index/_design/foo', 'json/bar', + Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.delete_doc_attachment_variants('_index/_design/foo/json', 'bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_index/_design/foo/json', 'bar', + Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # GET _design/foo/_view/bar + def test_get_view_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_view/bar' + """ + # EXPECTED: 404 + self.get_doc_attachment_variants('_design/foo/_view', 'bar', Expect.RESPONSE_404.value, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_view', 'bar', Expect.RESPONSE_404.value, True, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo', '/_view/bar', Expect.RESPONSE_404.value, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo', '/_view/bar', Expect.RESPONSE_404.value, True, path_segment_count=4) + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + self.get_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.get_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # PUT _design/foo/_view/bar + def test_put_view_via_ddoc_attachment(self): + """ + Test PUT requests for '_design/foo/_view/bar' + """ + # EXPECTED: Validation exception + self.put_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.put_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + self.put_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.put_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.put_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.put_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # DELETE _design/foo/_view/bar + def test_delete_view_via_ddoc_attachment(self): + """ + Test DELETE requests for '_design/foo/_view/bar' + """ + # EXPECTED: Validation exception + self.delete_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.delete_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + self.delete_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.delete_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # GET _design/foo/_info + def test_get_view_info_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_info' + """ + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_info', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_info', Expect.VALIDATION_EXCEPTION_ATT.value, True) + self.get_doc_attachment_variants('_design', 'foo/_info', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design', 'foo/_info', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.get_doc_attachment_variants('_design/', 'foo/_info', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design/', 'foo/_info', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # GET _design/foo/_search/bar + def test_get_search_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_search/bar' + """ + # EXPECTED: 404 + self.get_doc_attachment_variants('_design/foo/_search', 'bar', Expect.RESPONSE_404.value, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_search', 'bar', Expect.RESPONSE_404.value, True, + path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_search', 'bar?q=*.*', Expect.RESPONSE_404.value, + path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_search', 'bar?q=*.*', Expect.RESPONSE_404.value, True, + path_segment_count=4) + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_search/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_search/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + self.get_doc_attachment_variants('_design', 'foo/_search/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design', 'foo/_search/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.get_doc_attachment_variants('_design/', 'foo/_search/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design/', 'foo/_search/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # GET _design/foo/_search_info/bar + def test_get_search_info_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_search_info/bar' + """ + # EXPECTED: 404 + self.get_doc_attachment_variants('_design/foo/_search_info', 'bar', Expect.RESPONSE_404.value, + path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_search_info', 'bar', Expect.RESPONSE_404.value, True, + path_segment_count=4) + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_search_info/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_search_info/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + + # GET _design/foo/_geo/bar + def test_get_geo_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_geo/bar' + """ + # EXPECTED: 404 + self.get_doc_attachment_variants('_design/foo/_geo', 'bar', Expect.RESPONSE_404.value, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_geo', 'bar', Expect.RESPONSE_404.value, True, + path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_geo', 'bar?bbox=-50.52,-4.46,54.59,1.45', + Expect.RESPONSE_404.value, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_geo', 'bar?bbox=-50.52,-4.46,54.59,1.45', + Expect.RESPONSE_404.value, True, path_segment_count=4) + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_geo/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_geo/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + + # GET _design/foo/_geo_info/bar + def test_get_geo_info_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_geo_info/bar' + """ + # EXPECTED: 404 + self.get_doc_attachment_variants('_design/foo/_geo_info', 'bar', Expect.RESPONSE_404.value, + path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_geo_info', 'bar', Expect.RESPONSE_404.value, True, + path_segment_count=4) + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_geo_info/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_geo_info/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + + # GET _partition/foo + # EXPECTED: Validation exception + def test_get_invalid_partition_info(self): + """ + Test GET requests for '_partition/foo' + """ + self.get_document_variants('_partition/foo', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET _partition/foo + # EXPECTED: Validation exception + def test_get_invalid_partition_info_via_attachment(self): + """ + Test GET requests for '_partition/foo' + """ + self.get_doc_attachment_variants('_partition', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET _partition/foo/_all_docs + # EXPECTED: Validation exception + def test_get_partition_info(self): + """ + Test GET requests for '_partition/foo/_all_docs' + """ + self.get_document_variants('_partition/foo/_all_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET _partition/foo/_all_docs + # EXPECTED: Validation exception + def test_get_invalid_partition_all_docs_via_attachment(self): + """ + Test GET requests for '_partition/foo/_all_docs' + """ + self.get_doc_attachment_variants('_partition', 'foo/_all_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_partition/foo', '_all_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + + """UTIL FUNCTIONS""" + def mocked_get_requests(self, rev=None, override_status_code=None): + """ + Create a mock GET request for documents with the expected status code + :param rev: the doc's revision (default None) + :param override_status_code: override the status code for handling + inner `fetch` request call within `get_attachment` + :return: mocked Response object + """ + resp_mock = create_autospec(requests.Response) + if override_status_code is not None: + resp_mock.status_code = override_status_code + else: + resp_mock.status_code = self.expected_enum + if (resp_mock.status_code == 200 or resp_mock.status_code == 201 + and self.doc_id is not None): + if rev is not None: + resp_mock.text = f"""{{"_id": "{self.doc_id}", "_rev": "{rev}"}}""" + else: + resp_mock.text = f"""{{"_id": "{self.doc_id}", "_rev": "1-abc"}}""" + elif resp_mock.status_code == 404: + resp_mock.raise_for_status.side_effect = requests.exceptions.HTTPError + resp_mock.encoding = None + + return resp_mock + + def mocked_get_att_requests(self): + """ + Create a mock GET request for attachments with the expected status code + """ + self.expected_att_content = f"""this is a text attachment""" + # first fetch doc call with rev + fetch_mock = self.mocked_get_requests(rev=None, override_status_code=200) + # second get to attachment + resp_mock = create_autospec(requests.Response) + resp_mock.status_code = self.expected_enum + if self.expected_enum == 200 and self.doc_id is not None and self.att_name is not None: + resp_mock.text = self.expected_att_content + if self.expected_enum == 404: + resp_mock.raise_for_status.side_effect = requests.exceptions.HTTPError + self.doc_r_session_mock.get.side_effect = [fetch_mock, resp_mock] + + def mocked_head_requests(self, override_status_code=None): + """ + Create a mock HEAD request for documents and attachments with the expected status code + """ + resp_mock = create_autospec(requests.Response) + if override_status_code is not None: + resp_mock.status_code = override_status_code + else: + resp_mock.status_code = self.expected_enum + self.doc_r_session_mock.head = Mock(return_value=resp_mock) + + def mocked_delete_requests(self): + """ + Create a mock DELETE request for documents with the expected status code + """ + resp_mock = create_autospec(requests.Response) + resp_mock.status_code = self.expected_enum + if self.expected_enum == 201 and self.doc_id is not None: + resp_mock.text = f"""{{"id": "{self.doc_id}", "rev": "2-abc", "ok": true}}""" + self.doc_r_session_mock.delete = Mock(return_value=resp_mock) + + def mocked_delete_att_requests(self): + """ + Create a mock DELETE request for attachments with the expected status code + """ + # first `fetch` document call with rev + self.doc_r_session_mock.get = Mock(return_value=self.mocked_get_requests(rev=None, override_status_code=200)) + # second delete to attachment + resp_mock = create_autospec(requests.Response) + resp_mock.status_code = self.expected_enum + resp_mock.encoding = None + if self.expected_enum == 200 and self.doc_id is not None and self.att_name is not None: + resp_mock.text = f"""{{"id": "{self.doc_id}", "rev": "2-abc", "ok": true}}""" + elif self.expected_enum == 404: + resp_mock.raise_for_status.side_effect = requests.exceptions.HTTPError + + self.doc_r_session_mock.delete = Mock(return_value=resp_mock) + + def mocked_put_doc_requests(self): + """ + Create a mock PUT request for documents with the expected status code + """ + # mock 'doc.exists' request call within 'doc.save' function + self.mocked_head_requests(200) + resp_mock = create_autospec(requests.Response) + resp_mock.status_code = self.expected_enum + resp_mock.encoding = None + if self.expected_enum == 201 and self.doc_id is not None: + resp_mock.text = f"""{{"id": "{self.doc_id}", "rev": "1-abc", "ok": true}}""" + if self.expected_enum == 404: + resp_mock.raise_for_status.side_effect = requests.exceptions.HTTPError + self.doc_r_session_mock.put = Mock(return_value=resp_mock) + + def mocked_put_att_requests(self): + """ + Create a mock PUT request for attachments with the expected status code + """ + # first `fetch` document call within `put_attachment` + fetch_mock = self.mocked_get_requests(rev=None, override_status_code=200) + # create Response object for PUT attachment + resp_mock = create_autospec(requests.Response) + resp_mock.status_code = self.expected_enum + resp_mock.encoding = None + if self.expected_enum == 201 and self.doc_id is not None: + resp_mock.text = f"""{{"id": "{self.doc_id}", "rev": "2-def", "ok": true}}""" + if self.expected_enum == 404: + resp_mock.raise_for_status.side_effect = requests.exceptions.HTTPError + # final fetch doc call + second_fetch_mock = self.mocked_get_requests(rev='2-def', override_status_code=200) + self.doc_r_session_mock.get.side_effect = [fetch_mock, second_fetch_mock] + self.doc_r_session_mock.put = Mock(return_value=resp_mock) + + def get_document_variants(self, doc_id, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute GET/HEAD document requests + """ + self.doc_id = doc_id + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_head_requests() + self.head_document() + self.doc_r_session_mock.get.return_value = self.mocked_get_requests() + self.fetch_document() + self.assert_path_segments(self.doc_r_session_mock.get.call_args_list, path_segment_count) + + def get_doc_attachment_variants(self, doc_id, att_name, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute GET attachment requests + """ + self.att_name = att_name + self.doc_id = doc_id + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_get_att_requests() + self.get_doc_attachment() + self.assert_path_segments(self.doc_r_session_mock.get.call_args_list, path_segment_count) + + def put_document_variants(self, doc_id, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute PUT document requests + """ + self.doc_id = doc_id + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_put_doc_requests() + self.put_document() + self.assert_path_segments(self.doc_r_session_mock.put.call_args_list, path_segment_count) + + def put_doc_attachment_variants(self, doc_id, att_name, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute PUT attachment requests + """ + self.att_name = att_name + self.doc_id = doc_id + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_put_att_requests() + self.put_doc_attachment() + self.assert_path_segments(self.doc_r_session_mock.put.call_args_list, path_segment_count) + + def delete_document_variants(self, doc_id, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute DELETE document requests + """ + self.doc_id = doc_id + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_delete_requests() + self.delete_document() + self.assert_path_segments(self.doc_r_session_mock.delete.call_args_list, path_segment_count) + + def delete_doc_attachment_variants(self, doc_id, attname, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute DELETE attachment requests + """ + self.doc_id = doc_id + self.att_name = attname + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_delete_att_requests() + self.delete_doc_attachment() + self.assert_path_segments(self.doc_r_session_mock.delete.call_args_list, path_segment_count) + + """HTTP REQUEST FUNCTIONS""" + def head_document(self): + try: + resp = self.create_doc(self.doc_id, self.is_ddoc).exists() + if self.expected_enum == 200 or self.expected_enum == 201: + self.assertTrue(resp) + elif self.expected_enum == 404: + self.assertFalse(resp) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + + def delete_document(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + doc['_rev'] = '1-abc' + doc.delete() + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, doc['_id']) + self.assertFalse('rev' in doc) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + def fetch_document(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + doc.fetch() + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, doc['_id']) + self.assertIsNotNone(doc['_rev']) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + def put_document(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + doc.save() + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, doc['_id']) + self.assertIsNotNone(doc['_rev']) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + def delete_doc_attachment(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + doc['_rev'] = '1-abc' + resp = doc.delete_attachment(self.att_name) + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, doc['_id']) + self.assertEqual(self.doc_id, resp['id']) + self.assertEqual(doc['_rev'], resp['rev']) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + def get_doc_attachment(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + resp_att = doc.get_attachment(self.att_name, attachment_type='text') + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, doc['_id']) + self.assertIsNotNone(resp_att) + self.assertEqual(resp_att, self.expected_att_content) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + def put_doc_attachment(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + resp_att = doc.put_attachment(self.att_name, content_type='utf-8', data='test') + self.assertIsNotNone(resp_att) + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, resp_att['id']) + self.assertEqual(resp_att['id'], doc['_id']) + self.assertEqual(doc['_rev'], resp_att['rev']) + self.assertEqual(resp_att['ok'], True) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + """HELPER FUNCTIONS""" + def create_doc(self, doc_id=None, is_ddoc=False): + """ + Function to create and return a Document or DesignDocument object. + """ + if is_ddoc: + if doc_id is not None: + doc = DesignDocument(self.db, doc_id) + else: + doc = DesignDocument(self.db) + elif doc_id is not None: + doc = Document(self.db, doc_id) + else: + doc = Document(self.db) + self.assertIsNone(doc.get('_rev')) + return doc + + def assert_exception_msg(self, cae): + """ + Function to assert whether the exception message is for an invalid + document ID or an attachment name. + """ + self.assertTrue(id(self.expected_enum), id(cae)) + # Check that actual exception message starts with the expected msg + if str(cae).startswith(str(self.expected_enum)): + # Figure out which exception msg to assert against + if str(cae).startswith(ValidationExceptionMsg.ATTACHMENT.value): + self.assertEqual(str(cae), f"""{ValidationExceptionMsg.ATTACHMENT.value} {self.att_name}""") + elif str(cae).startswith(ValidationExceptionMsg.DOC.value): + self.assertEqual(str(cae), f"""{ValidationExceptionMsg.DOC.value} {self.doc_id}""") + else: + self.fail('Expected CloudantArgumentError message should equal actual error message.') + + def assert_path_segments(self, actual_call_args_list, exp_segment_count): + """ + Function to assert the number of path segments from a mock request + """ + # If there's no segment count, verify that the test case expects an argument error + if exp_segment_count is None: + self.assertTrue(isinstance(self.expected_enum, CloudantArgumentError), 'Path segment count should exist ' + 'when testing against valid ' + 'document or attachment names.') + else: + # get latest call in list + url, headers = actual_call_args_list[len(actual_call_args_list) - 1] + # there should only be one mocked url + self.assertEqual(len(url), 1) + # parse path of url and remove first / path segment + path = urlparse(url[0]).path[1:] + actual_segment_count = len(path.split('/')) + self.assertEqual(actual_segment_count, exp_segment_count) diff --git a/tests/unit/iam_auth_tests.py b/tests/unit/iam_auth_tests.py new file mode 100644 index 00000000..3e0c7cf2 --- /dev/null +++ b/tests/unit/iam_auth_tests.py @@ -0,0 +1,382 @@ +#!/usr/bin/env python +# Copyright (c) 2017, 2019 IBM. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Unit tests for IAM authentication. """ +import time +import unittest +import json +import mock + +from cloudant._2to3 import Cookie +from cloudant.client import Cloudant +from cloudant._client_session import IAMSession + +MOCK_API_KEY = 'CqbrIYzdO3btWV-5t4teJLY_etfT_dkccq-vO-5vCXSo' + +MOCK_ACCESS_TOKEN = ('eyJraWQiOiIyMDE3MDQwMi0wMDowMDowMCIsImFsZyI6IlJTMjU2In0.e' + 'yJpYW1faWQiOiJJQk1pZC0yNzAwMDdHRjBEIiwiaWQiOiJJQk1pZC0yNz' + 'AwMDdHRjBEIiwicmVhbG1pZCI6IklCTWlkIiwiaWRlbnRpZmllciI6IjI' + '3MDAwN0dGMEQiLCJnaXZlbl9uYW1lIjoiVG9tIiwiZmFtaWx5X25hbWUi' + 'OiJCbGVuY2giLCJuYW1lIjoiVG9tIEJsZW5jaCIsImVtYWlsIjoidGJsZ' + 'W5jaEB1ay5pYm0uY29tIiwic3ViIjoidGJsZW5jaEB1ay5pYm0uY29tIi' + 'wiYWNjb3VudCI6eyJic3MiOiI1ZTM1ZTZhMjlmYjJlZWNhNDAwYWU0YzN' + 'lMWZhY2Y2MSJ9LCJpYXQiOjE1MDA0NjcxMDIsImV4cCI6MTUwMDQ3MDcw' + 'MiwiaXNzIjoiaHR0cHM6Ly9pYW0ubmcuYmx1ZW1peC5uZXQvb2lkYy90b' + '2tlbiIsImdyYW50X3R5cGUiOiJ1cm46aWJtOnBhcmFtczpvYXV0aDpncm' + 'FudC10eXBlOmFwaWtleSIsInNjb3BlIjoib3BlbmlkIiwiY2xpZW50X2l' + 'kIjoiZGVmYXVsdCJ9.XAPdb5K4n2nYih-JWTWBGoKkxTXM31c1BB1g-Ci' + 'auc2LxuoNXVTyz_mNqf1zQL07FUde1Cb_dwrbotjickNcxVPost6byQzt' + 'fc0mRF1x2S6VR8tn7SGiRmXBjLofkTh1JQq-jutp2MS315XbTG6K6m16u' + 'YzL9qfMnRvQHxsZWErzfPiJx-Trg_j7OX-qNFjdNUGnRpU7FmULy0r7Rx' + 'Ld8mhG-M1yxVzRBAZzvM63s0XXfMnk1oLi-BuUUTqVOdrM0KyYMWfD0Q7' + '2PTo4Exa17V-R_73Nq8VPCwpOvZcwKRA2sPTVgTMzU34max8b5kpTzVGJ' + '6SXSItTVOUdAygZBng') + +MOCK_IAM_TOKEN_RESPONSE = '{"access_token": "%s",\ + "refresh_token": "MO61FKNvVRWkSa4vmBZqYv_Jt1kkGMUc-XzTcNnR-GnIhVKXHUWxJVV3\ + RddE8Kqh3X_TZRmyK8UySIWKxoJ2t6obUSUalPm90SBpTdoXtaljpNyo\ + rmqCCYPROnk6JBym72ikSJqKHHEZVQkT0B5ggZCwPMnKagFj0ufs-VIh\ + CF97xhDxDKcIPMWG02xxPuESaSTJJug7e_dUDoak_ZXm9xxBmOTRKwOx\ + n5sTKthNyvVpEYPE7jIHeiRdVDOWhN5LomgCn3TqFCLpMErnqwgNYbyC\ + Bd9rNm-alYKDb6Jle4njuIBpXxQPb4euDwLd1osApaSME3nEarFWqRBz\ + hjoqCe1Kv564s_rY7qzD1nHGvKOdpSa0ZkMcfJ0LbXSQPs7gBTSVrBFZ\ + qwlg-2F-U3Cto62-9qRR_cEu_K9ZyVwL4jWgOlngKmxV6Ku4L5mHp4Kg\ + EJSnY_78_V2nm64E--i2ZA1FhiKwIVHDOivVNhggE9oabxg54vd63glp\ + 4GfpNnmZsMOUYG9blJJpH4fDX4Ifjbw-iNBD7S2LRpP8b8vG9pb4WioG\ + zN43lE5CysveKYWrQEZpThznxXlw1snDu_A48JiL3Lrvo1LobLhF3zFV\ + -kQ=",\ + "token_type": "Bearer",\ + "expires_in": 3600,\ + "expiration": 1500470702}'%(MOCK_ACCESS_TOKEN) + + +class IAMAuthTests(unittest.TestCase): + """ Unit tests for IAM authentication. """ + + @staticmethod + def _mock_cookie(expires_secs=300): + return Cookie( + version=0, + name='IAMSession', + value=('SQJCaUQxMqEfMEAyRKU6UopLVXceS0c9RPuQgDArCEYoN3l_TEY4gdf-DJ7' + '4sHfjcNEUVjfdOvA'), + port=None, + port_specified=False, + domain='localhost', + domain_specified=False, + domain_initial_dot=False, + path="/", + path_specified=True, + secure=True, + expires=int(time.time() + expires_secs), + discard=False, + comment=None, + comment_url=None, + rest={'HttpOnly': None}, + rfc2109=True) + + def test_iam_set_credentials(self): + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984') + self.assertEqual(iam._api_key, MOCK_API_KEY) + + new_api_key = 'some_new_api_key' + iam.set_credentials(None, new_api_key) + + self.assertEqual(iam._api_key, new_api_key) + + @mock.patch('cloudant._client_session.ClientSession.request') + def test_iam_get_access_token(self, m_req): + m_response = mock.MagicMock() + mock_token_response_text = mock.PropertyMock(return_value=MOCK_IAM_TOKEN_RESPONSE) + type(m_response).text = mock_token_response_text + m_req.return_value = m_response + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984') + access_token = iam._get_access_token() + + m_req.assert_called_once_with( + 'POST', + iam._token_url, + auth=None, + headers={'Accepts': 'application/json'}, + data={ + 'grant_type': 'urn:ibm:params:oauth:grant-type:apikey', + 'response_type': 'cloud_iam', + 'apikey': MOCK_API_KEY + } + ) + + self.assertEqual(access_token, MOCK_ACCESS_TOKEN) + self.assertTrue(m_response.raise_for_status.called) + mock_token_response_text.assert_called_with() + + @mock.patch('cloudant._client_session.ClientSession.request') + def test_iam_get_access_token_with_iam_client_id_and_secret(self, m_req): + m_response = mock.MagicMock() + mock_token_response_text = mock.PropertyMock(return_value=MOCK_IAM_TOKEN_RESPONSE) + type(m_response).text = mock_token_response_text + m_req.return_value = m_response + + iam_client_id = 'foo' + iam_client_secret = 'bar' + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984', + client_id=iam_client_id, + client_secret=iam_client_secret) + access_token = iam._get_access_token() + + m_req.assert_called_once_with( + 'POST', + iam._token_url, + auth=(iam_client_id, iam_client_secret), + headers={'Accepts': 'application/json'}, + data={ + 'grant_type': 'urn:ibm:params:oauth:grant-type:apikey', + 'response_type': 'cloud_iam', + 'apikey': MOCK_API_KEY + } + ) + + self.assertEqual(access_token, MOCK_ACCESS_TOKEN) + self.assertTrue(m_response.raise_for_status.called) + mock_token_response_text.assert_called_with() + + @mock.patch('cloudant._client_session.ClientSession.request') + @mock.patch('cloudant._client_session.IAMSession._get_access_token') + def test_iam_login(self, m_token, m_req): + m_token.return_value = MOCK_ACCESS_TOKEN + m_response = mock.MagicMock() + m_req.return_value = m_response + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984') + iam.login() + + m_req.assert_called_once_with( + 'POST', + iam._session_url, + headers={'Content-Type': 'application/json'}, + data=json.dumps({'access_token': MOCK_ACCESS_TOKEN}) + ) + + self.assertEqual(m_token.call_count, 1) + self.assertTrue(m_response.raise_for_status.called) + + def test_iam_logout(self): + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984') + # add a valid cookie to jar + iam.cookies.set_cookie(self._mock_cookie()) + self.assertEqual(len(iam.cookies.keys()), 1) + iam.logout() + self.assertEqual(len(iam.cookies.keys()), 0) + + @mock.patch('cloudant._client_session.ClientSession.get') + def test_iam_get_session_info(self, m_get): + m_info = '{"ok": true, "info": {"authentication_db": "_users"}}' + + m_response = mock.MagicMock() + type(m_response).text = mock.PropertyMock(return_value=m_info) + m_get.return_value = m_response + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984') + info = iam.info() + + m_get.assert_called_once_with(iam._session_url) + + self.assertEqual(info, json.loads(m_info)) + self.assertTrue(m_response.raise_for_status.called) + + @mock.patch('cloudant._client_session.IAMSession.login') + @mock.patch('cloudant._client_session.ClientSession.request') + def test_iam_first_request(self, m_req, m_login): + # mock 200 + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + type(m_response_ok).text = mock.PropertyMock(return_value='{"ok": true}') + + m_req.return_value = m_response_ok + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984', auto_renew=True) + iam.login() + + self.assertEqual(m_login.call_count, 1) + self.assertEqual(m_req.call_count, 0) + + # add a valid cookie to jar + iam.cookies.set_cookie(self._mock_cookie()) + + resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') + + self.assertEqual(m_login.call_count, 1) + self.assertEqual(m_req.call_count, 1) + self.assertEqual(resp.status_code, 200) + + @mock.patch('cloudant._client_session.IAMSession.login') + @mock.patch('cloudant._client_session.ClientSession.request') + def test_iam_renew_cookie_on_expiry(self, m_req, m_login): + # mock 200 + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + type(m_response_ok).text = mock.PropertyMock(return_value='{"ok": true}') + + m_req.return_value = m_response_ok + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984', auto_renew=True) + iam.login() + + # add an expired cookie to jar + iam.cookies.set_cookie(self._mock_cookie(expires_secs=-300)) + + resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') + + self.assertEqual(m_login.call_count, 2) + self.assertEqual(m_req.call_count, 1) + self.assertEqual(resp.status_code, 200) + + @mock.patch('cloudant._client_session.IAMSession.login') + @mock.patch('cloudant._client_session.ClientSession.request') + def test_iam_renew_cookie_on_401_success(self, m_req, m_login): + # mock 200 + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + type(m_response_ok).text = mock.PropertyMock(return_value='{"ok": true}') + # mock 401 + m_response_bad = mock.MagicMock() + type(m_response_bad).status_code = mock.PropertyMock(return_value=401) + + m_req.side_effect = [m_response_bad, m_response_ok, m_response_ok] + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984', auto_renew=True) + iam.login() + self.assertEqual(m_login.call_count, 1) + + # add a valid cookie to jar + iam.cookies.set_cookie(self._mock_cookie()) + + resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') + self.assertEqual(resp.status_code, 200) + self.assertEqual(m_login.call_count, 2) + self.assertEqual(m_req.call_count, 2) + + resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') + self.assertEqual(resp.status_code, 200) + self.assertEqual(m_login.call_count, 2) + self.assertEqual(m_req.call_count, 3) + + @mock.patch('cloudant._client_session.IAMSession.login') + @mock.patch('cloudant._client_session.ClientSession.request') + def test_iam_renew_cookie_on_401_failure(self, m_req, m_login): + # mock 401 + m_response_bad = mock.MagicMock() + type(m_response_bad).status_code = mock.PropertyMock(return_value=401) + + m_req.return_value = m_response_bad + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984', auto_renew=True) + iam.login() + self.assertEqual(m_login.call_count, 1) + + # add a valid cookie to jar + iam.cookies.set_cookie(self._mock_cookie()) + + resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') + self.assertEqual(resp.status_code, 401) + self.assertEqual(m_login.call_count, 2) + self.assertEqual(m_req.call_count, 2) + + resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') + self.assertEqual(resp.status_code, 401) + self.assertEqual(m_login.call_count, 3) + self.assertEqual(m_req.call_count, 4) + + @mock.patch('cloudant._client_session.IAMSession.login') + @mock.patch('cloudant._client_session.ClientSession.request') + def test_iam_renew_cookie_disabled(self, m_req, m_login): + # mock 401 + m_response_bad = mock.MagicMock() + type(m_response_bad).status_code = mock.PropertyMock(return_value=401) + + m_req.return_value = m_response_bad + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984', auto_renew=False) + iam.login() + self.assertEqual(m_login.call_count, 1) + + resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') + self.assertEqual(resp.status_code, 401) + self.assertEqual(m_login.call_count, 1) # no attempt to renew + self.assertEqual(m_req.call_count, 1) + + resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') + self.assertEqual(resp.status_code, 401) + self.assertEqual(m_login.call_count, 1) # no attempt to renew + self.assertEqual(m_req.call_count, 2) + + @mock.patch('cloudant._client_session.IAMSession.login') + @mock.patch('cloudant._client_session.ClientSession.request') + def test_iam_client_create(self, m_req, m_login): + # mock 200 + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + type(m_response_ok).text = mock.PropertyMock(return_value='["animaldb"]') + + m_req.return_value = m_response_ok + + # create IAM client + client = Cloudant.iam('foo', MOCK_API_KEY) + client.connect() + + # add a valid cookie to jar + client.r_session.cookies.set_cookie(self._mock_cookie()) + + dbs = client.all_dbs() + + self.assertEqual(m_login.call_count, 1) + self.assertEqual(m_req.call_count, 1) + self.assertEqual(dbs, ['animaldb']) + + @mock.patch('cloudant._client_session.IAMSession.login') + @mock.patch('cloudant._client_session.IAMSession.set_credentials') + def test_iam_client_session_login(self, m_set, m_login): + # create IAM client + client = Cloudant.iam('foo', MOCK_API_KEY) + client.connect() + + # add a valid cookie to jar + client.r_session.cookies.set_cookie(self._mock_cookie()) + + client.session_login() + + m_set.assert_called_with(None, None) + self.assertEqual(m_login.call_count, 2) + self.assertEqual(m_set.call_count, 2) + + @mock.patch('cloudant._client_session.IAMSession.login') + @mock.patch('cloudant._client_session.IAMSession.set_credentials') + def test_iam_client_session_login_with_new_credentials(self, m_set, m_login): + # create IAM client + client = Cloudant.iam('foo', MOCK_API_KEY) + client.connect() + + # add a valid cookie to jar + client.r_session.cookies.set_cookie(self._mock_cookie()) + + client.session_login('bar', 'baz') # new creds + + m_set.assert_called_with('bar', 'baz') + self.assertEqual(m_login.call_count, 2) + self.assertEqual(m_set.call_count, 2) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit/index_tests.py b/tests/unit/index_tests.py index 398e3080..f12cf158 100644 --- a/tests/unit/index_tests.py +++ b/tests/unit/index_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2020 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,21 +22,22 @@ """ from __future__ import absolute_import +import os import unittest + import mock -import os -import posixpath import requests - -from cloudant.index import Index, TextIndex, SpecialIndex -from cloudant.query import Query -from cloudant.view import QueryIndexView from cloudant.design_document import DesignDocument from cloudant.document import Document from cloudant.error import CloudantArgumentError, CloudantIndexException +from cloudant.index import Index, TextIndex, SpecialIndex +from cloudant.query import Query +from cloudant.view import QueryIndexView +from nose.plugins.attrib import attr -from .. import PY2 from .unit_t_db_base import UnitTestDbBase +from .. import PY2 + class CloudantIndexExceptionTests(unittest.TestCase): """ @@ -67,10 +68,8 @@ def test_raise_with_proper_code_and_args(self): raise CloudantIndexException(101) self.assertEqual(cm.exception.status_code, 101) -@unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant Index tests' - ) +@attr(db=['cloudant','couch']) +@attr(couchapi=2) class IndexTests(UnitTestDbBase): """ Index unit tests @@ -122,7 +121,7 @@ def test_retrieve_index_url(self): index = Index(self.db) self.assertEqual( index.index_url, - posixpath.join(self.db.database_url, '_index') + '/'.join((self.db.database_url, '_index')) ) def test_index_to_dictionary(self): @@ -160,25 +159,23 @@ def test_create_an_index_using_ddoc_index_name(self): self.assertEqual(index.design_document_id, '_design/ddoc001') self.assertEqual(index.name, 'index001') with DesignDocument(self.db, index.design_document_id) as ddoc: + self.assertIsInstance(ddoc.get_view(index.name), QueryIndexView) + + self.assertEqual(ddoc['_id'], index.design_document_id) + self.assertTrue(ddoc['_rev'].startswith('1-')) + + self.assertEqual(ddoc['indexes'], {}) self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + self.assertListEqual(list(ddoc['views'].keys()), ['index001']) - self.assertIsInstance(ddoc.get_view('index001'), QueryIndexView) - self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': '_design/ddoc001', - '_rev': ddoc['_rev'], - 'indexes': {}, - 'language': 'query', - 'views': {'index001': {'map': {'fields': {'name': 'asc', - 'age': 'asc'}}, - 'reduce': '_count', - 'options': {'def': {'fields': ['name', - 'age']}, - }}}, - 'lists': {}, - 'shows': {} - } - ) + + view = ddoc['views'][index.name] + self.assertEqual(view['map']['fields']['age'], 'asc') + self.assertEqual(view['map']['fields']['name'], 'asc') + self.assertEqual(view['options']['def']['fields'], ['name', 'age']) + self.assertEqual(view['reduce'], '_count') def test_create_an_index_without_ddoc_index_name(self): """ @@ -190,25 +187,23 @@ def test_create_an_index_without_ddoc_index_name(self): self.assertTrue(index.design_document_id.startswith('_design/')) self.assertIsNotNone(index.name) with DesignDocument(self.db, index.design_document_id) as ddoc: - self.assertEqual(ddoc['language'], 'query') - self.assertListEqual(list(ddoc['views'].keys()), [index.name]) self.assertIsInstance(ddoc.get_view(index.name), QueryIndexView) + + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': index.design_document_id, - '_rev': ddoc['_rev'], - 'indexes': {}, - 'language': 'query', - 'views': {index.name: {'map': {'fields': {'name': 'asc', - 'age': 'asc'}}, - 'reduce': '_count', - 'options': {'def': {'fields': ['name', - 'age']}, - }}}, - 'lists': {}, - 'shows': {} - } - ) + + self.assertEqual(ddoc['indexes'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + + self.assertListEqual(list(ddoc['views'].keys()), [index.name]) + + view = ddoc['views'][index.name] + self.assertEqual(view['map']['fields']['age'], 'asc') + self.assertEqual(view['map']['fields']['name'], 'asc') + self.assertEqual(view['options']['def']['fields'], ['name', 'age']) + self.assertEqual(view['reduce'], '_count') def test_create_an_index_with_empty_ddoc_index_name(self): """ @@ -220,25 +215,23 @@ def test_create_an_index_with_empty_ddoc_index_name(self): self.assertTrue(index.design_document_id.startswith('_design/')) self.assertIsNotNone(index.name) with DesignDocument(self.db, index.design_document_id) as ddoc: - self.assertEqual(ddoc['language'], 'query') - self.assertListEqual(list(ddoc['views'].keys()), [index.name]) self.assertIsInstance(ddoc.get_view(index.name), QueryIndexView) + + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': index.design_document_id, - '_rev': ddoc['_rev'], - 'indexes': {}, - 'language': 'query', - 'views': {index.name: {'map': {'fields': {'name': 'asc', - 'age': 'asc'}}, - 'reduce': '_count', - 'options': {'def': {'fields': ['name', - 'age']}, - }}}, - 'lists': {}, - 'shows': {} - } - ) + + self.assertEqual(ddoc['indexes'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + + self.assertListEqual(list(ddoc['views'].keys()), [index.name]) + + view = ddoc['views'][index.name] + self.assertEqual(view['map']['fields']['age'], 'asc') + self.assertEqual(view['map']['fields']['name'], 'asc') + self.assertEqual(view['options']['def']['fields'], ['name', 'age']) + self.assertEqual(view['reduce'], '_count') def test_create_an_index_using_design_prefix(self): """ @@ -250,25 +243,23 @@ def test_create_an_index_using_design_prefix(self): self.assertEqual(index.design_document_id, '_design/ddoc001') self.assertEqual(index.name, 'index001') with DesignDocument(self.db, index.design_document_id) as ddoc: - self.assertEqual(ddoc['language'], 'query') - self.assertListEqual(list(ddoc['views'].keys()), ['index001']) - self.assertIsInstance(ddoc.get_view('index001'), QueryIndexView) + self.assertIsInstance(ddoc.get_view(index.name), QueryIndexView) + + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': '_design/ddoc001', - '_rev': ddoc['_rev'], - 'indexes': {}, - 'language': 'query', - 'views': {'index001': {'map': {'fields': {'name': 'asc', - 'age': 'asc'}}, - 'reduce': '_count', - 'options': {'def': {'fields': ['name', - 'age']}, - }}}, - 'lists': {}, - 'shows': {} - } - ) + + self.assertEqual(ddoc['indexes'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + + self.assertListEqual(list(ddoc['views'].keys()), [index.name]) + + view = ddoc['views'][index.name] + self.assertEqual(view['map']['fields']['age'], 'asc') + self.assertEqual(view['map']['fields']['name'], 'asc') + self.assertEqual(view['options']['def']['fields'], ['name', 'age']) + self.assertEqual(view['reduce'], '_count') def test_create_uses_custom_encoder(self): """ @@ -399,12 +390,9 @@ def test_index_usage_via_query(self): self.populate_db_with_documents(100) result = self.db.get_query_result(fields=['name', 'age'], selector={'age': {'$eq': 6}}, raw_result=True) - self.assertTrue(str(result['warning']).startswith("no matching index found")) + self.assertTrue(str(result['warning']).lower().startswith("no matching index found")) -@unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant Text Index tests' - ) +@attr(db='cloudant') class TextIndexTests(UnitTestDbBase): """ Search Index unit tests @@ -458,27 +446,23 @@ def test_create_a_search_index_no_kwargs(self): self.assertEqual(index.design_document_id, '_design/ddoc001') self.assertEqual(index.name, 'index001') with DesignDocument(self.db, index.design_document_id) as ddoc: - self.assertEqual(ddoc['language'], 'query') - self.assertListEqual(list(ddoc['indexes'].keys()), ['index001']) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': '_design/ddoc001', - '_rev': ddoc['_rev'], - 'language': 'query', - 'views': {}, - 'indexes': {'index001': - {'index': {'index_array_lengths': True, - 'fields': 'all_fields', - 'default_field': {}, - 'default_analyzer': 'keyword', - 'selector': {}}, - 'analyzer': {'name': 'perfield', - 'default': 'keyword', - 'fields': {'$default': 'standard'}}}}, - 'lists': {}, - 'shows': {} - } - ) + + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + self.assertEqual(ddoc['views'], {}) + + index = ddoc['indexes']['index001'] + self.assertEqual(index['analyzer']['default'], 'keyword') + self.assertEqual(index['analyzer']['fields']['$default'], 'standard') + self.assertEqual(index['analyzer']['name'], 'perfield') + self.assertEqual(index['index']['default_analyzer'], 'keyword') + self.assertEqual(index['index']['default_field'], {}) + self.assertEqual(index['index']['fields'], 'all_fields') + self.assertEqual(index['index']['selector'], {}) + self.assertTrue(index['index']['index_array_lengths']) def test_create_a_search_index_with_kwargs(self): """ @@ -496,29 +480,24 @@ def test_create_a_search_index_with_kwargs(self): self.assertEqual(index.design_document_id, '_design/ddoc001') self.assertEqual(index.name, 'index001') with DesignDocument(self.db, index.design_document_id) as ddoc: - self.assertEqual(ddoc['language'], 'query') - self.assertListEqual(list(ddoc['indexes'].keys()), ['index001']) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': '_design/ddoc001', - '_rev': ddoc['_rev'], - 'language': 'query', - 'views': {}, - 'indexes': {'index001': - {'index': {'index_array_lengths': True, - 'fields': [{'name': 'name', 'type': 'string'}, - {'name': 'age', 'type': 'number'}], - 'default_field': {'enabled': True, - 'analyzer': 'german'}, - 'default_analyzer': 'keyword', - 'selector': {}}, - 'analyzer': {'name': 'perfield', - 'default': 'keyword', - 'fields': {'$default': 'german'}}}}, - 'lists': {}, - 'shows': {} - } - ) + + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + self.assertEqual(ddoc['views'], {}) + + index = ddoc['indexes']['index001'] + self.assertEqual(index['analyzer']['default'], 'keyword') + self.assertEqual(index['analyzer']['fields']['$default'], 'german') + self.assertEqual(index['analyzer']['name'], 'perfield') + self.assertEqual(index['index']['default_analyzer'], 'keyword') + self.assertEqual(index['index']['default_field']['analyzer'], 'german') + self.assertEqual(index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) + self.assertEqual(index['index']['selector'], {}) + self.assertTrue(index['index']['default_field']['enabled']) + self.assertTrue(index['index']['index_array_lengths']) def test_create_a_search_index_invalid_argument(self): """ @@ -575,6 +554,21 @@ def test_create_a_search_index_invalid_selector_value(self): '<{} \'dict\'>'.format('type' if PY2 else 'class') ) + def test_create_unpartitioned_query_index(self): + """ + Test that create_query_index works on an unpartitioned database + """ + ddoc = DesignDocument(self.db, document_id="unpartitioned_query_index_ddoc") + ddoc["language"] = "query" + ddoc.save() + index = self.db.create_query_index( + design_document_id="_design/unpartitioned_query_index_ddoc", + fields=["key"], + partitioned=False + ) + index.create() + self.assertGreater(len(self.db.get_query_indexes()), 0) + def test_search_index_via_query(self): """ Test that a created TEXT index will produce expected query results. diff --git a/tests/unit/infinite_feed_tests.py b/tests/unit/infinite_feed_tests.py index 544aca60..7891cb91 100644 --- a/tests/unit/infinite_feed_tests.py +++ b/tests/unit/infinite_feed_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,18 +16,18 @@ feed module - Unit tests for Feed class """ -import unittest -from requests import Session -import json import os +import unittest from time import sleep -from cloudant.feed import InfiniteFeed, Feed -from cloudant.client import CouchDB from cloudant.error import CloudantArgumentError, CloudantFeedException +from cloudant.feed import InfiniteFeed, Feed +from nose.plugins.attrib import attr +from requests import Session from .unit_t_db_base import UnitTestDbBase + class MethodCallCount(object): """ This callable class is used as a proxy by the infinite feed tests to wrap @@ -71,6 +71,7 @@ def test_raise_with_proper_code_and_args(self): raise CloudantFeedException(101) self.assertEqual(cm.exception.status_code, 101) +@attr(db=['cloudant','couch']) class InfiniteFeedTests(UnitTestDbBase): """ Infinite Feed unit tests @@ -126,8 +127,7 @@ def test_constructor_with_invalid_feed_option(self): 'Invalid infinite feed option: longpoll. Must be set to continuous.' ) - @unittest.skipIf(os.environ.get('RUN_CLOUDANT_TESTS'), - 'Skipping since test is possible only when testing against CouchDB.') + @attr(db='couch') def test_invalid_source_couchdb(self): """ Ensure that a CouchDB client cannot be used with an infinite feed. @@ -137,8 +137,8 @@ def test_invalid_source_couchdb(self): self.assertEqual(str(cm.exception), 'Infinite _db_updates feed not supported for CouchDB.') - @unittest.skipIf(not os.environ.get('RUN_CLOUDANT_TESTS') or - os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') + @unittest.skipIf(os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') + @attr(db='cloudant') def test_constructor_db_updates(self): """ Test constructing an infinite _db_updates feed. @@ -185,8 +185,8 @@ def test_infinite_feed(self): # the continuous feed was started/restarted 3 separate times. self.assertEqual(feed._start.called_count, 3) - @unittest.skipIf(not os.environ.get('RUN_CLOUDANT_TESTS') or - os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') + @unittest.skipIf(os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') + @attr(db='cloudant') def test_infinite_db_updates_feed(self): """ Test that an _db_updates infinite feed will continue to issue multiple diff --git a/tests/unit/param_translation_tests.py b/tests/unit/param_translation_tests.py index 0fca5e09..fda3c002 100644 --- a/tests/unit/param_translation_tests.py +++ b/tests/unit/param_translation_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -36,7 +36,7 @@ def test_valid_descending(self): {'descending': 'true'} ) self.assertEqual( - python_to_couch({'descending': False}), + python_to_couch({'descending': False}), {'descending': 'false'} ) @@ -44,9 +44,9 @@ def test_valid_endkey(self): """ Test endkey translation is successful. """ - self.assertEqual(python_to_couch({'endkey': 10}), {'endkey': 10}) + self.assertEqual(python_to_couch({'endkey': 10}), {'endkey': '10'}) # Test with long type - self.assertEqual(python_to_couch({'endkey': LONG_NUMBER}), {'endkey': LONG_NUMBER}) + self.assertEqual(python_to_couch({'endkey': LONG_NUMBER}), {'endkey': str(LONG_NUMBER)}) self.assertEqual( python_to_couch({'endkey': 'foo'}), {'endkey': '"foo"'} @@ -55,6 +55,10 @@ def test_valid_endkey(self): python_to_couch({'endkey': ['foo', 10]}), {'endkey': '["foo", 10]'} ) + self.assertEqual( + python_to_couch({'endkey': True}), + {'endkey': 'true'} + ) def test_valid_endkey_docid(self): """ @@ -120,14 +124,18 @@ def test_valid_key(self): """ Test key translation is successful. """ - self.assertEqual(python_to_couch({'key': 10}), {'key': 10}) + self.assertEqual(python_to_couch({'key': 10}), {'key': '10'}) # Test with long type - self.assertEqual(python_to_couch({'key': LONG_NUMBER}), {'key': LONG_NUMBER}) + self.assertEqual(python_to_couch({'key': LONG_NUMBER}), {'key': str(LONG_NUMBER)}) self.assertEqual(python_to_couch({'key': 'foo'}), {'key': '"foo"'}) self.assertEqual( python_to_couch({'key': ['foo', 10]}), {'key': '["foo", 10]'} ) + self.assertEqual( + python_to_couch({'key': True}), + {'key': 'true'} + ) def test_valid_keys(self): """ @@ -194,9 +202,9 @@ def test_valid_startkey(self): """ Test startkey translation is successful. """ - self.assertEqual(python_to_couch({'startkey': 10}), {'startkey': 10}) + self.assertEqual(python_to_couch({'startkey': 10}), {'startkey': '10'}) # Test with long type - self.assertEqual(python_to_couch({'startkey': LONG_NUMBER}), {'startkey': LONG_NUMBER}) + self.assertEqual(python_to_couch({'startkey': LONG_NUMBER}), {'startkey': str(LONG_NUMBER)}) self.assertEqual( python_to_couch({'startkey': 'foo'}), {'startkey': '"foo"'} @@ -205,6 +213,10 @@ def test_valid_startkey(self): python_to_couch({'startkey': ['foo', 10]}), {'startkey': '["foo", 10]'} ) + self.assertEqual( + python_to_couch({'startkey': True}), + {'startkey': 'true'} + ) def test_valid_startkey_docid(self): """ @@ -215,6 +227,14 @@ def test_valid_startkey_docid(self): {'startkey_docid': 'foo'} ) + def test_valid_update(self): + """ + Test lazy translation is successful. + """ + self.assertEqual(python_to_couch({'update': 'true'}), {'update': 'true'}) + self.assertEqual(python_to_couch({'update': 'false'}), {'update': 'false'}) + self.assertEqual(python_to_couch({'update': 'lazy'}), {'update': 'lazy'}) + def test_invalid_argument(self): """ Test translation fails when an invalid argument is passed in. @@ -239,7 +259,7 @@ def test_invalid_endkey(self): """ msg = 'Argument endkey not instance of expected type:' with self.assertRaises(CloudantArgumentError) as cm: - python_to_couch({'endkey': True}) + python_to_couch({'endkey': {'foo': 'bar'}}) self.assertTrue(str(cm.exception).startswith(msg)) def test_invalid_endkey_docid(self): @@ -294,7 +314,7 @@ def test_invalid_key(self): """ msg = 'Argument key not instance of expected type:' with self.assertRaises(CloudantArgumentError) as cm: - python_to_couch({'key': True}) + python_to_couch({'key': {'foo': 'bar'}}) self.assertTrue(str(cm.exception).startswith(msg)) def test_invalid_keys_not_list(self): @@ -364,7 +384,7 @@ def test_invalid_startkey(self): """ msg = 'Argument startkey not instance of expected type:' with self.assertRaises(CloudantArgumentError) as cm: - python_to_couch({'startkey': True}) + python_to_couch({'startkey': {'foo': 'bar'}}) self.assertTrue(str(cm.exception).startswith(msg)) def test_invalid_startkey_docid(self): diff --git a/tests/unit/query_result_tests.py b/tests/unit/query_result_tests.py index d290f930..98a16f3a 100644 --- a/tests/unit/query_result_tests.py +++ b/tests/unit/query_result_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,11 +26,12 @@ from cloudant.query import Query from cloudant.result import QueryResult from cloudant.error import ResultException +from nose.plugins.attrib import attr from .unit_t_db_base import UnitTestDbBase -@unittest.skipUnless(os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant QueryResult tests') +@attr(db=['cloudant','couch']) +@attr(couchapi=2) class QueryResultTests(UnitTestDbBase): """ QueryResult unit tests @@ -424,24 +425,14 @@ def test_get_item_index_slice_using_stop_only_limit_skip(self): def test_iteration_with_invalid_options(self): """ - Test that iteration raises an exception when "skip" and/or "limit" are - used as options for the result. + Test that iteration raises an exception when "limit" is + used as option for the result. """ - result = self.create_result(q_parms={'skip': 10}) - with self.assertRaises(ResultException) as cm: - invalid_result = [row for row in result] - self.assertEqual(cm.exception.status_code, 103) - result = self.create_result(q_parms={'limit': 10}) with self.assertRaises(ResultException) as cm: invalid_result = [row for row in result] self.assertEqual(cm.exception.status_code, 103) - result = self.create_result(q_parms={'limit': 10, 'skip': 10}) - with self.assertRaises(ResultException) as cm: - invalid_result = [row for row in result] - self.assertEqual(cm.exception.status_code, 103) - def test_iteration_invalid_page_size(self): """ Test that iteration raises an exception when and invalid "page_size" is diff --git a/tests/unit/query_tests.py b/tests/unit/query_tests.py index 214fecc0..30f78bcd 100644 --- a/tests/unit/query_tests.py +++ b/tests/unit/query_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,20 +20,19 @@ """ -import unittest import os -import posixpath +import unittest +from cloudant.error import CloudantArgumentError from cloudant.query import Query from cloudant.result import QueryResult -from cloudant.error import CloudantArgumentError +from nose.plugins.attrib import attr from .unit_t_db_base import UnitTestDbBase -@unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant Query tests' - ) + +@attr(db=['cloudant','couch']) +@attr(couchapi=2) class QueryTests(UnitTestDbBase): """ Query unit tests @@ -78,7 +77,7 @@ def test_retrieve_query_url(self): query = Query(self.db) self.assertEqual( query.url, - posixpath.join(self.db.database_url, '_find') + '/'.join((self.db.database_url, '_find')) ) def test_callable_with_invalid_argument(self): diff --git a/tests/unit/replicator_mock_tests.py b/tests/unit/replicator_mock_tests.py new file mode 100644 index 00000000..f49b04d4 --- /dev/null +++ b/tests/unit/replicator_mock_tests.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python +# Copyright (C) 2018 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +_replicator_mock_tests_ + +replicator module - Mock unit tests for the Replicator class +""" + +import mock +import unittest + +from cloudant.database import CouchDatabase +from cloudant.replicator import Replicator + +from tests.unit.iam_auth_tests import MOCK_API_KEY + + +class ReplicatorDocumentValidationMockTests(unittest.TestCase): + """ + Replicator document validation tests + """ + + def setUp(self): + self.repl_id = 'rep_test' + + self.server_url = 'http://localhost:5984' + self.user_ctx = { + 'name': 'foo', + 'roles': ['erlanger', 'researcher'] + } + + self.source_db = 'source_db' + self.target_db = 'target_db' + + def setUpClientMocks(self, admin_party=False, iam_api_key=None): + m_client = mock.MagicMock() + type(m_client).server_url = mock.PropertyMock( + return_value=self.server_url) + + type(m_client).admin_party = mock.PropertyMock( + return_value=admin_party) + + iam_authenticated = False + + if iam_api_key is not None: + iam_authenticated = True + + m_session = mock.MagicMock() + type(m_session).get_api_key = mock.PropertyMock( + return_value=iam_api_key) + + type(m_client).r_session = mock.PropertyMock( + return_value=m_session) + + type(m_client).is_iam_authenticated = mock.PropertyMock( + return_value=iam_authenticated) + + return m_client + + def test_using_admin_party_source_and_target(self): + m_admin_party_client = self.setUpClientMocks(admin_party=True) + + m_replicator = mock.MagicMock() + type(m_replicator).creds = mock.PropertyMock(return_value=None) + m_admin_party_client.__getitem__.return_value = m_replicator + + # create source/target databases + src = CouchDatabase(m_admin_party_client, self.source_db) + tgt = CouchDatabase(m_admin_party_client, self.target_db) + + # trigger replication + rep = Replicator(m_admin_party_client) + rep.create_replication(src, tgt, repl_id=self.repl_id) + + kcall = m_replicator.create_document.call_args_list + self.assertEqual(len(kcall), 1) + args, kwargs = kcall[0] + self.assertEqual(len(args), 1) + + expected_doc = { + '_id': self.repl_id, + 'source': {'url': '/'.join((self.server_url, self.source_db))}, + 'target': {'url': '/'.join((self.server_url, self.target_db))} + } + + self.assertDictEqual(args[0], expected_doc) + self.assertTrue(kwargs['throw_on_exists']) + + def test_using_basic_auth_source_and_target(self): + test_basic_auth_header = 'abc' + + m_basic_auth_client = self.setUpClientMocks() + + m_replicator = mock.MagicMock() + m_basic_auth_client.__getitem__.return_value = m_replicator + m_basic_auth_client.basic_auth_str.return_value = test_basic_auth_header + + # create source/target databases + src = CouchDatabase(m_basic_auth_client, self.source_db) + tgt = CouchDatabase(m_basic_auth_client, self.target_db) + + # trigger replication + rep = Replicator(m_basic_auth_client) + rep.create_replication( + src, tgt, repl_id=self.repl_id, user_ctx=self.user_ctx) + + kcall = m_replicator.create_document.call_args_list + self.assertEqual(len(kcall), 1) + args, kwargs = kcall[0] + self.assertEqual(len(args), 1) + + expected_doc = { + '_id': self.repl_id, + 'user_ctx': self.user_ctx, + 'source': { + 'headers': {'Authorization': test_basic_auth_header}, + 'url': '/'.join((self.server_url, self.source_db)) + }, + 'target': { + 'headers': {'Authorization': test_basic_auth_header}, + 'url': '/'.join((self.server_url, self.target_db)) + } + } + + self.assertDictEqual(args[0], expected_doc) + self.assertTrue(kwargs['throw_on_exists']) + + def test_using_iam_auth_source_and_target(self): + m_iam_auth_client = self.setUpClientMocks(iam_api_key=MOCK_API_KEY) + + m_replicator = mock.MagicMock() + m_iam_auth_client.__getitem__.return_value = m_replicator + + # create source/target databases + src = CouchDatabase(m_iam_auth_client, self.source_db) + tgt = CouchDatabase(m_iam_auth_client, self.target_db) + + # trigger replication + rep = Replicator(m_iam_auth_client) + rep.create_replication( + src, tgt, repl_id=self.repl_id, user_ctx=self.user_ctx) + + kcall = m_replicator.create_document.call_args_list + self.assertEqual(len(kcall), 1) + args, kwargs = kcall[0] + self.assertEqual(len(args), 1) + + expected_doc = { + '_id': self.repl_id, + 'user_ctx': self.user_ctx, + 'source': { + 'auth': {'iam': {'api_key': MOCK_API_KEY}}, + 'url': '/'.join((self.server_url, self.source_db)) + }, + 'target': { + 'auth': {'iam': {'api_key': MOCK_API_KEY}}, + 'url': '/'.join((self.server_url, self.target_db)) + } + } + + self.assertDictEqual(args[0], expected_doc) + self.assertTrue(kwargs['throw_on_exists']) diff --git a/tests/unit/replicator_tests.py b/tests/unit/replicator_tests.py index 74d08465..9eb56b56 100644 --- a/tests/unit/replicator_tests.py +++ b/tests/unit/replicator_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2016, 2017 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2020 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,21 +22,22 @@ """ +import time import unittest import uuid -import time -from flaky import flaky import requests -from requests import ConnectionError - -from cloudant.replicator import Replicator from cloudant.document import Document from cloudant.error import CloudantReplicatorException, CloudantClientException +from cloudant.replicator import Replicator +from flaky import flaky +from nose.plugins.attrib import attr +from requests import ConnectionError from .unit_t_db_base import UnitTestDbBase from .. import unicode_ + class CloudantReplicatorExceptionTests(unittest.TestCase): """ Ensure CloudantReplicatorException functions as expected. @@ -75,6 +76,7 @@ def test_raise_with_proper_code_and_args(self): raise CloudantReplicatorException(404, 'foo') self.assertEqual(cm.exception.status_code, 404) +@attr(db=['cloudant','couch']) class ReplicatorTests(UnitTestDbBase): """ Replicator unit tests @@ -155,7 +157,11 @@ def test_constructor_failure(self): def test_replication_with_generated_id(self): clone = Replicator(self.client) - clone.create_replication(self.db, self.target_db) + repl_id = clone.create_replication( + self.db, + self.target_db + ) + self.replication_ids.append(repl_id['_id']) @flaky(max_runs=3) def test_create_replication(self): @@ -175,7 +181,7 @@ def test_create_replication(self): # Test that the replication document was created expected_keys = ['_id', '_rev', 'source', 'target', 'user_ctx'] # If Admin Party mode then user_ctx will not be in the key list - if self.client.admin_party: + if self.client.admin_party or self.client.is_iam_authenticated: expected_keys.pop() self.assertTrue(all(x in list(repl_doc.keys()) for x in expected_keys)) self.assertEqual(repl_doc['_id'], repl_id) @@ -214,10 +220,13 @@ def test_create_replication(self): def test_timeout_in_create_replication(self): """ Test that a read timeout exception is thrown when creating a - replicator with a timeout value of 500 ms. + replicator with a read timeout value of 5 s. """ - # Setup client with a timeout - self.set_up_client(auto_connect=True, timeout=.5) + # Setup client with a read timeout (but the standard connect timeout) + # Note that this timeout applies to all connections from this client + # setting it too short can cause intermittent failures when responses + # are not quick enough. Setting it too long makes the test take longer. + self.set_up_client(auto_connect=True, timeout=(30,5)) self.db = self.client[self.test_target_dbname] self.target_db = self.client[self.test_dbname] # Construct a replicator with the updated client @@ -233,7 +242,7 @@ def test_timeout_in_create_replication(self): # Test that the replication document was created expected_keys = ['_id', '_rev', 'source', 'target', 'user_ctx'] # If Admin Party mode then user_ctx will not be in the key list - if self.client.admin_party: + if self.client.admin_party or self.client.is_iam_authenticated: expected_keys.pop() self.assertTrue(all(x in list(repl_doc.keys()) for x in expected_keys)) self.assertEqual(repl_doc['_id'], repl_id) @@ -313,12 +322,14 @@ def test_retrieve_replication_state(self): ) self.replication_ids.append(repl_id) repl_state = None - valid_states = ['completed', 'error', 'triggered', None] + # note triggered is for versions prior to 2.1 + valid_states = ['completed', 'error', 'initializing', 'triggered', 'pending', 'running', 'failed', 'crashing', None] finished = False + # Wait for 5 minutes or a terminal replication state for _ in range(300): repl_state = self.replicator.replication_state(repl_id) self.assertTrue(repl_state in valid_states) - if repl_state in ('error', 'completed'): + if repl_state in ('error', 'failed', 'completed'): finished = True break time.sleep(1) @@ -352,7 +363,16 @@ def test_stop_replication(self): self.target_db, repl_id ) - self.replicator.stop_replication(repl_id) + max_retry = 3 + while True: + try: + max_retry -= 1 + self.replicator.stop_replication(repl_id) + break + except requests.HTTPError as err: + self.assertEqual(err.response.status_code, 409) + if max_retry == 0: + self.fail('Failed to stop replication: {0}'.format(err)) try: # The .fetch() will fail since the replication has been stopped # and the replication document has been removed from the db. @@ -389,11 +409,16 @@ def test_follow_replication(self): repl_id ) self.replication_ids.append(repl_id) - valid_states = ('completed', 'error', 'triggered', None) + # note triggered is for versions prior to 2.1 + valid_states = ['completed', 'error', 'initializing', 'triggered', 'pending', 'running', 'failed', 'crashing', None] repl_states = [] + if 'scheduler' in self.client.features(): + state_key = 'state' + else: + state_key = '_replication_state' for doc in self.replicator.follow_replication(repl_id): - self.assertIn(doc.get('_replication_state'), valid_states) - repl_states.append(doc.get('_replication_state')) + self.assertIn(doc.get(state_key), valid_states) + repl_states.append(doc.get(state_key)) self.assertTrue(len(repl_states) > 0) self.assertEqual(repl_states[-1], 'completed') self.assertNotIn('error', repl_states) diff --git a/tests/unit/result_tests.py b/tests/unit/result_tests.py index f33ce0e4..13b8eb7a 100644 --- a/tests/unit/result_tests.py +++ b/tests/unit/result_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,18 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import mock """ result module - Unit tests for Result class """ import unittest -import os -from requests.exceptions import HTTPError from cloudant.error import ResultException from cloudant.result import Result, ResultByKey +from cloudant.view import View +from nose.plugins.attrib import attr +from requests.exceptions import HTTPError from .unit_t_db_base import UnitTestDbBase + class ResultExceptionTests(unittest.TestCase): """ Ensure ResultException functions as expected. @@ -72,6 +75,7 @@ def test_raise_with_proper_code_and_args(self): raise ResultException(102, 'foo', 'bar') self.assertEqual(cm.exception.status_code, 102) +@attr(db=['cloudant','couch']) class ResultTests(UnitTestDbBase): """ Result unit tests @@ -248,6 +252,16 @@ def test_get_item_slice_no_start_no_stop(self): {'key': 'julia002', 'id': 'julia002', 'value': 1}] self.assertEqual(result[:], expected) + def test_get_all_items(self): + """ + Test that all results can be retrieved. + """ + result = Result(self.view001, limit=3) + expected = [{'key': 'julia000', 'id': 'julia000', 'value': 1}, + {'key': 'julia001', 'id': 'julia001', 'value': 1}, + {'key': 'julia002', 'id': 'julia002', 'value': 1}] + self.assertEqual(result.all(), expected) + def test_get_item_invalid_index_slice(self): """ Test that when invalid start and stop values are provided in a slice @@ -552,24 +566,14 @@ def test_get_item_key_slice_using_stop_only(self): def test_iteration_with_invalid_options(self): """ - Test that iteration raises an exception when "skip" and/or "limit" are - used as options for the result. + Test that iteration raises an exception when "limit" is + used as option for the result. """ - result = Result(self.view001, skip=10) - with self.assertRaises(ResultException) as cm: - invalid_result = [row for row in result] - self.assertEqual(cm.exception.status_code, 103) - result = Result(self.view001, limit=10) with self.assertRaises(ResultException) as cm: invalid_result = [row for row in result] self.assertEqual(cm.exception.status_code, 103) - result = Result(self.view001, skip=10, limit=10) - with self.assertRaises(ResultException) as cm: - invalid_result = [row for row in result] - self.assertEqual(cm.exception.status_code, 103) - def test_iteration_invalid_page_size(self): """ Test that iteration raises an exception when and invalid "page_size" is @@ -631,5 +635,49 @@ def test_iteration_no_data(self): result = Result(self.view001, startkey='ruby') self.assertEqual([x for x in result], []) + def test_iteration_integer_keys(self): + """ + Test that iteration works as expected when keys are integer. + """ + result = Result(self.view007, page_size=10) + self.assertEqual(len([x for x in result]), 100) + + def test_iteration_pagination(self): + """ + Test that iteration pagination works as expected. + """ + + class CallMock: + expected_calls = [ + {'limit': 28}, + {'limit': 28, 'startkey': 1, 'startkey_docid': 'julia027'}, + {'limit': 28, 'startkey': 1, 'startkey_docid': 'julia054'}, + {'limit': 28, 'startkey': 1, 'startkey_docid': 'julia081'}, + ] + + def __init__(self, outer): + self.outer = outer + self.expected_calls.reverse() + + def call(self, *args, **kwargs): + self.outer.assertEqual(dict(kwargs), + self.expected_calls.pop(), + 'pagination error') + return View.__call__(self.outer.view007, *args, **kwargs) + + with mock.patch.object(self, 'view007', + CallMock(self).call) as _: + + result = Result(self.view007, page_size=27) + + expected = [ + {'id': 'julia{0:03d}'.format(i), + 'key': 1, + 'value': 'julia'} + for i in range(100) + ] + self.assertEqual([x for x in result], expected) + + if __name__ == '__main__': unittest.main() diff --git a/tests/unit/scheduler_tests.py b/tests/unit/scheduler_tests.py new file mode 100644 index 00000000..5ef8cab9 --- /dev/null +++ b/tests/unit/scheduler_tests.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python +# Copyright (C) 2018 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Unit tests for the Scheduler class +""" + +import unittest +import requests +import json +import mock + +from cloudant.scheduler import Scheduler + +from .unit_t_db_base import UnitTestDbBase + +class SchedulerTests(UnitTestDbBase): + + def setUp(self): + """ + Set up test attributes + """ + super(SchedulerTests, self).setUp() + self.db_set_up() + + def tearDown(self): + """ + Reset test attributes + """ + self.db_tear_down() + super(SchedulerTests, self).tearDown() + + def test_scheduler_docs(self): + """ + Test scheduler docs + """ + # set up mock response using a real captured response + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + type(m_response_ok).text = mock.PropertyMock(return_value='{"total_rows":6,"offset":0,"docs":[\ + {"database":"tomblench/_replicator",\ + "doc_id":"296e48244e003eba8764b2156b3bf302",\ + "id":null,\ + "source":"https://tomblench.cloudant.com/animaldb/",\ + "target":"https://tomblench.cloudant.com/animaldb_copy/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":15,\ + "missing_revisions_found":2,\ + "docs_read":2,\ + "docs_written":2,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"19-g1AAAAGjeJyVz10KwjAMB_BoJ4KX8AZF2tWPJ3eVpqnO0XUg27PeTG9Wa_VhwmT6kkDIPz_iACArGcGS0DRnWxDmHE9HdJ3lxjUdad9yb1sXF6cacB9CqEqmZ3UczKUh2uGhHxeD8U9i_Z3AIla8vJVJUlBIZYTqX5A_KMM7SfFZrHCNLUK3p7RIkl5tSRD-K6kx6f6S0k8sScpYJTb5uFQ9AI9Ch9c"},\ + "start_time":null,\ + "last_updated":"2017-04-13T14:53:50+00:00"},\ + {"database":"tomblench/_replicator",\ + "doc_id":"3b749f320867d703550b0f758a4000ae",\ + "id":null,\ + "source":"https://examples.cloudant.com/animaldb/",\ + "target":"https://tomblench.cloudant.com/animaldb/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":15,\ + "missing_revisions_found":15,\ + "docs_read":15,\ + "docs_written":15,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"56-g1AAAAGveJzLYWBgYMlgTmFQSElKzi9KdUhJstDLTS3KLElMT9VLzskvTUnMK9HLSy3JAapkSmRIsv___39WBnMiby5QgN04JS3FLDUJWb8Jdv0gSxThigyN8diS5AAkk-qhFvFALEo2MTEwMSXGDDSbTPHYlMcCJBkagBTQsv0g28TBtpkbGCQapaF4C4cxJFt2AGIZ2GscYMuMDEzMUizMkC0zw25MFgBKoovi"},\ + "start_time":null,\ + "last_updated":"2017-04-27T12:28:44+00:00"},\ + {"database":"tomblench/_replicator",\ + "doc_id":"ad8f7896480b8081c8f0a2267ffd1859",\ + "id":null,\ + "source":"https://tortytherlediffecareette:*****@mikerhodestesty008.cloudant.com/moviesdb/",\ + "target":"https://tomblench.cloudant.com/moviesdb_rep/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":5997,\ + "missing_revisions_found":5997,\ + "docs_read":5997,\ + "docs_written":5997,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"5997-g1AAAANreJy10UEKwjAQAMBgBcVP2BeUpEm1PdmfaDYJSKkVtB486U_0J_oBTz5AHyAI3jxIjUml1x7ayy67LDssmyKE-nNHIleCWK5ULIF6uVrnW4xDT6TLjeRZ7mUqT_VkhyMYFkWRzB3Q1XOhez3iczKKghor6jvg6giTiroYiuNQYYqbpeIfNa2oh72KhQGosFlq9qN2FfUyFPgUCKOnullXR7TXSWuHkvsYjjEWjQVvgTta7lRyV_szKgmRbVx3ttzNcs7AcEoKCHAb3N1y_9-9DYeBYzEiNTYlX3EcE0s"},\ + "start_time":null,\ + "last_updated":"2016-08-23T13:11:26+00:00"},\ + {"database":"tomblench/_replicator",\ + "doc_id":"b63c053ecd95a4047b55ed8847b046f1",\ + "id":null,\ + "source":"https://tomblench.cloudant.com/atestdb2/",\ + "target":"https://tomblench.cloudant.com/atestdb1/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":1,\ + "missing_revisions_found":1,\ + "docs_read":1,\ + "docs_written":1,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"2-g1AAAAFHeJyNjkEOgjAQRSdAYjyFN2jSFCtdyVU6nSKQWhJC13ozvVktsoEF0c2fTPL_-98BQNHmBCdCM4y2JuQMuxu6YJlxQyDtJ-bt5JIx04DXGGOvYRsR-xGsk-JjTrW5hnv6Dg0XplRngmPwZJvOW9ry5D7PF0nhmU5CvmZm9mVKVVacLr8pfy9fmt5L02q9qEhJbtbr-w-AQmfD"},\ + "start_time":null,\ + "last_updated":"2017-05-16T16:25:22+00:00"},\ + {"database":"tomblench/_replicator",\ + "doc_id":"c71c9e69e30a182dc91d8938277bc85e",\ + "id":null,\ + "source":"https://tomblench.cloudant.com/animaldb/",\ + "target":"https://tomblench.cloudant.com/animaldb_copy/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":15,\ + "missing_revisions_found":15,\ + "docs_read":15,\ + "docs_written":15,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"14-g1AAAAEueJzLYWBgYMlgTmGQSUlKzi9KdUhJMtTLTU1M0UvOyS9NScwr0ctLLckBqmJKZEiy____f1YGUyJrLlCAPdHEPCktJZk43UkOQDKpHmoAI9gAw2STxCTzJOIMyGMBkgwNQApoxv6sDGaoK0yN04wsk80IGEGKHQcgdoAdygxxaIplklFaWhYAu2FdOA"},\ + "start_time":null,\ + "last_updated":"2015-05-12T11:47:33+00:00"},\ + {"database":"tomblench/_replicator",\ + "doc_id":"e6242d1e9ce059b0388fc75af3116a39",\ + "id":null,\ + "source":"https://tomblench.cloudant.com/atestdb1/",\ + "target":"https://tomblench.cloudant.com/atestdb2/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":1,\ + "missing_revisions_found":1,\ + "docs_read":1,\ + "docs_written":1,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"1-g1AAAAFheJyFzkEOgjAQBdBRSIyn8AZNgEJgJVeZ6bQCqSUhdK0305th1Q1dEDYzyWTy_rcAkHYJw4VJjZNumQpB_Y2s10LZ0TO6WTg92_B4RKDrsixDlyDcw-FUVUiFahjO3rE2vdMcY9k2Rm2Y9Ig8bWqspdz25Lbn0jDhGVYgX1_z8DMblnlp8n0lTir3kt7_pFV7NE2WYbluP3wATr5vQA"},\ + "start_time":null,\ + "last_updated":"2017-05-16T16:24:02+00:00"}]}') + + self.client.r_session.get = mock.Mock(return_value=m_response_ok) + scheduler = Scheduler(self.client) + response = scheduler.list_docs(skip=0, limit=10) + # assert on request and response + self.client.r_session.get.assert_called_with( + self.url + '/_scheduler/docs', + params={"skip":0, "limit":10}, + ) + self.assertEqual(response["total_rows"], 6) + + def test_scheduler_doc(self): + """ + Test scheduler doc + """ + # set up mock response using a real captured response + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + type(m_response_ok).text = mock.PropertyMock(return_value='{"database":"tomblench/_replicator",\ + "doc_id":"296e48244e003eba8764b2156b3bf302",\ + "id":null,\ + "source":"https://tomblench.cloudant.com/animaldb/",\ + "target":"https://tomblench.cloudant.com/animaldb_copy/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":15,\ + "missing_revisions_found":2,\ + "docs_read":2,\ + "docs_written":2,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"19-g1AAAAGjeJyVz10KwjAMB_BoJ4KX8AZF2tWPJ3eVpqnO0XUg27PeTG9Wa_VhwmT6kkDIPz_iACArGcGS0DRnWxDmHE9HdJ3lxjUdad9yb1sXF6cacB9CqEqmZ3UczKUh2uGhHxeD8U9i_Z3AIla8vJVJUlBIZYTqX5A_KMM7SfFZrHCNLUK3p7RIkl5tSRD-K6kx6f6S0k8sScpYJTb5uFQ9AI9Ch9c"},\ + "start_time":null,\ + "last_updated":"2017-04-13T14:53:50+00:00"}') + self.client.r_session.get = mock.Mock(return_value=m_response_ok) + scheduler = Scheduler(self.client) + response = scheduler.get_doc("296e48244e003eba8764b2156b3bf302") + # assert on request and response + self.client.r_session.get.assert_called_with( + self.url + '/_scheduler/docs/_replicator/296e48244e003eba8764b2156b3bf302', + ) + self.assertEqual(response["doc_id"], "296e48244e003eba8764b2156b3bf302") + + + def test_scheduler_jobs(self): + """ + Test scheduler jobs + """ + # set up mock response using a real captured response + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + type(m_response_ok).text = mock.PropertyMock(return_value='{"total_rows":1,"offset":0,\ + "jobs":[{"database":null,\ + "id":"f11105eaaded4981d21ff8ebf846f48b+create_target",\ + "pid":"<0.5866.6800>",\ + "source":"https://clientlibs-test:*****@clientlibs-test.cloudant.com/largedb1g/",\ + "target":"https://tomblench:*****@tomblench.cloudant.com/largedb1g/",\ + "user":"tomblench",\ + "doc_id":null,\ + "history":[{"timestamp":"2018-04-12T13:06:20Z",\ + "type":"started"},\ + {"timestamp":"2018-04-12T13:06:20Z",\ + "type":"added"}],\ + "node":"dbcore@db2.bigblue.cloudant.net",\ + "start_time":"2018-04-12T13:06:20Z"}]}') + self.client.r_session.get = mock.Mock(return_value=m_response_ok) + scheduler = Scheduler(self.client) + response = scheduler.list_jobs(skip=0, limit=10) + # assert on request and response + self.client.r_session.get.assert_called_with( + self.url + '/_scheduler/jobs', + params={"skip":0, "limit":10}, + ) + self.assertEqual(response["total_rows"], 1) diff --git a/tests/unit/security_document_tests.py b/tests/unit/security_document_tests.py index 4e8d1ed6..0e2ee024 100644 --- a/tests/unit/security_document_tests.py +++ b/tests/unit/security_document_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,16 +19,16 @@ module docstring. """ -import unittest -import requests import json -import os +import unittest from cloudant.security_document import SecurityDocument +from nose.plugins.attrib import attr from .unit_t_db_base import UnitTestDbBase +@attr(db=['cloudant','couch']) class SecurityDocumentTests(UnitTestDbBase): """ SecurityDocument unit tests diff --git a/tests/unit/unit_t_db_base.py b/tests/unit/unit_t_db_base.py index c0b62b7a..2067af0c 100644 --- a/tests/unit/unit_t_db_base.py +++ b/tests/unit/unit_t_db_base.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2016, 2017 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2020 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -65,9 +65,26 @@ from cloudant.client import CouchDB, Cloudant from cloudant.design_document import DesignDocument +from cloudant.error import CloudantClientException from .. import unicode_ + +def skip_if_not_cookie_auth(f): + def wrapper(*args): + if not args[0].use_cookie_auth: + raise unittest.SkipTest('Test only supports cookie authentication') + return f(*args) + return wrapper + + +def skip_if_iam(f): + def wrapper(*args): + if os.environ.get('IAM_API_KEY'): + raise unittest.SkipTest('Test only supports non-IAM authentication') + return f(*args) + return wrapper + class UnitTestDbBase(unittest.TestCase): """ The base class for all unit tests targeting a database @@ -91,17 +108,31 @@ def setUpClass(cls): return if os.environ.get('DB_USER') is None: + # Get couchdb docker node name + if os.environ.get('COUCHDB_VERSION') == '2.3.1': + os.environ['NODENAME'] = requests.get( + '{0}/_membership'.format(os.environ['DB_URL'])).json()['all_nodes'][0] os.environ['DB_USER_CREATED'] = '1' os.environ['DB_USER'] = 'user-{0}'.format( unicode_(uuid.uuid4()) ) os.environ['DB_PASSWORD'] = 'password' - resp = requests.put( - '{0}/_config/admins/{1}'.format( - os.environ['DB_URL'], - os.environ['DB_USER'] + if os.environ.get('COUCHDB_VERSION') == '2.3.1': + resp = requests.put( + '{0}/_node/{1}/_config/admins/{2}'.format( + os.environ['DB_URL'], + os.environ['NODENAME'], + os.environ['DB_USER'] + ), + data='"{0}"'.format(os.environ['DB_PASSWORD']) + ) + else: + resp = requests.put( + '{0}/_config/admins/{1}'.format( + os.environ['DB_URL'], + os.environ['DB_USER'] ), - data='"{0}"'.format(os.environ['DB_PASSWORD']) + data='"{0}"'.format(os.environ['DB_PASSWORD']) ) resp.raise_for_status() @@ -112,13 +143,25 @@ def tearDownClass(cls): """ if (os.environ.get('RUN_CLOUDANT_TESTS') is None and os.environ.get('DB_USER_CREATED') is not None): - resp = requests.delete( - '{0}://{1}:{2}@{3}/_config/admins/{4}'.format( - os.environ['DB_URL'].split('://', 1)[0], - os.environ['DB_USER'], - os.environ['DB_PASSWORD'], - os.environ['DB_URL'].split('://', 1)[1], - os.environ['DB_USER'] + if os.environ.get('COUCHDB_VERSION') == '2.3.1': + resp = requests.delete( + '{0}://{1}:{2}@{3}/_node/{4}/_config/admins/{5}'.format( + os.environ['DB_URL'].split('://', 1)[0], + os.environ['DB_USER'], + os.environ['DB_PASSWORD'], + os.environ['DB_URL'].split('://', 1)[1], + os.environ['NODENAME'], + os.environ['DB_USER'] + ) + ) + else: + resp = requests.delete( + '{0}://{1}:{2}@{3}/_config/admins/{4}'.format( + os.environ['DB_URL'].split('://', 1)[0], + os.environ['DB_USER'], + os.environ['DB_PASSWORD'], + os.environ['DB_URL'].split('://', 1)[1], + os.environ['DB_USER'] ) ) del os.environ['DB_USER_CREATED'] @@ -133,14 +176,20 @@ def setUp(self): def set_up_client(self, auto_connect=False, auto_renew=False, encoder=None, timeout=(30,300)): + self.user = os.environ.get('DB_USER', None) + self.pwd = os.environ.get('DB_PASSWORD', None) + self.use_cookie_auth = True + self.iam_api_key = os.environ.get('IAM_API_KEY', None) + if os.environ.get('RUN_CLOUDANT_TESTS') is None: + self.url = os.environ['DB_URL'] + admin_party = False - if (os.environ.get('ADMIN_PARTY') and - os.environ.get('ADMIN_PARTY') == 'true'): + if os.environ.get('ADMIN_PARTY') == 'true': admin_party = True - self.user = os.environ.get('DB_USER', None) - self.pwd = os.environ.get('DB_PASSWORD', None) - self.url = os.environ['DB_URL'] + + self.use_cookie_auth = False + # construct Cloudant client (using admin party mode) self.client = CouchDB( self.user, self.pwd, @@ -153,22 +202,50 @@ def set_up_client(self, auto_connect=False, auto_renew=False, encoder=None, ) else: self.account = os.environ.get('CLOUDANT_ACCOUNT') - self.user = os.environ.get('DB_USER') - self.pwd = os.environ.get('DB_PASSWORD') self.url = os.environ.get( 'DB_URL', 'https://{0}.cloudant.com'.format(self.account)) - self.client = Cloudant( - self.user, - self.pwd, - url=self.url, - x_cloudant_user=self.account, - connect=auto_connect, - auto_renew=auto_renew, - encoder=encoder, - timeout=timeout - ) + if os.environ.get('RUN_BASIC_AUTH_TESTS'): + self.use_cookie_auth = False + # construct Cloudant client (using basic access authentication) + self.client = Cloudant( + self.user, + self.pwd, + url=self.url, + x_cloudant_user=self.account, + connect=auto_connect, + auto_renew=auto_renew, + encoder=encoder, + timeout=timeout, + use_basic_auth=True, + ) + elif self.iam_api_key: + self.use_cookie_auth = False + # construct Cloudant client (using IAM authentication) + self.client = Cloudant( + None, # username is not required + self.iam_api_key, + url=self.url, + x_cloudant_user=self.account, + connect=auto_connect, + auto_renew=auto_renew, + encoder=encoder, + timeout=timeout, + use_iam=True, + ) + else: + # construct Cloudant client (using cookie authentication) + self.client = Cloudant( + self.user, + self.pwd, + url=self.url, + x_cloudant_user=self.account, + connect=auto_connect, + auto_renew=auto_renew, + encoder=encoder, + timeout=timeout + ) def tearDown(self): """ @@ -176,13 +253,14 @@ def tearDown(self): """ del self.client - def db_set_up(self): + def db_set_up(self, partitioned=False): """ Set up test attributes for Database tests """ self.client.connect() self.test_dbname = self.dbname() - self.db = self.client._DATABASE_CLASS(self.client, self.test_dbname) + self.db = self.client._DATABASE_CLASS( + self.client, self.test_dbname, partitioned=partitioned) self.db.create() def db_tear_down(self): @@ -205,6 +283,20 @@ def populate_db_with_documents(self, doc_count=100, **kwargs): ] return self.db.bulk_docs(docs) + def populate_db_with_partitioned_documents(self, key_count, docs_per_partition): + partition_keys = [uuid.uuid4().hex.upper()[:8] for _ in range(key_count)] + for partition_key in partition_keys: + docs = [] + for i in range(docs_per_partition): + docs.append({ + '_id': '{0}:doc{1}'.format(partition_key, i), + 'foo': 'bar' + }) + + self.db.bulk_docs(docs) + + return partition_keys + def create_views(self): """ Create a design document with views for use with tests. @@ -237,6 +329,10 @@ def create_views(self): 'function (doc) {\n emit([doc.name, doc.age], 1);\n}', '_count' ) + self.ddoc.add_view( + 'view007', + 'function (doc) {\n emit(1, doc.name);\n}' + ) self.ddoc.save() self.view001 = self.ddoc.get_view('view001') self.view002 = self.ddoc.get_view('view002') @@ -244,6 +340,7 @@ def create_views(self): self.view004 = self.ddoc.get_view('view004') self.view005 = self.ddoc.get_view('view005') self.view006 = self.ddoc.get_view('view006') + self.view007 = self.ddoc.get_view('view007') def create_search_index(self): """ @@ -285,17 +382,42 @@ def load_security_document_data(self): 'bar2': ['_reader'] } } - if os.environ.get('ADMIN_PARTY') == 'true': - resp = requests.put( - '/'.join([self.db.database_url, '_security']), - data=json.dumps(self.sdoc), - headers={'Content-Type': 'application/json'} - ) - else: - resp = requests.put( - '/'.join([self.db.database_url, '_security']), - auth=(self.user, self.pwd), - data=json.dumps(self.sdoc), - headers={'Content-Type': 'application/json'} - ) + resp = self.client.r_session.put( + '/'.join([self.db.database_url, '_security']), + data=json.dumps(self.sdoc), + headers={'Content-Type': 'application/json'} + ) self.assertEqual(resp.status_code, 200) + + def create_db_updates(self): + """ + Create '_global_changes' system database required for testing against _db_updates + """ + self.DB_UPDATES = '_global_changes' + try: + self.client.create_database(self.DB_UPDATES, throw_on_exists=True) + except CloudantClientException: + self.delete_db_updates() + self.create_db_updates() + + def delete_db_updates(self): + """ + Delete '_global_changes' system database used for _db_updates testing + """ + try: + self.client.delete_database(self.DB_UPDATES) + except CloudantClientException: + pass + + def is_couchdb_1x_version(self): + if os.environ.get('COUCHDB_VERSION') and os.environ.get('COUCHDB_VERSION').startswith('1'): + return True + else: + # Get version from server info + couchdb_info = json.loads(self.client.r_session.get(self.client.server_url).text) + if couchdb_info and couchdb_info['version'].startswith('1'): + return True + else: + return False + + diff --git a/tests/unit/view_execution_tests.py b/tests/unit/view_execution_tests.py index 680508f5..45a082c6 100644 --- a/tests/unit/view_execution_tests.py +++ b/tests/unit/view_execution_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,8 +17,12 @@ """ import unittest +from nose.plugins.attrib import attr + from .unit_t_db_base import UnitTestDbBase + +@attr(db=['cloudant','couch']) class QueryParmExecutionTests(UnitTestDbBase): """ Test cases for the execution of views queries using translated parameters. @@ -467,7 +471,7 @@ def test_stale_ok(self): try: self.view001(stale='ok') except Exception as err: - self.assertFail(str(err), 'An unexpected error was encountered.') + self.fail('An unexpected error was encountered: '+str(err)) def test_stale_update_after(self): """ @@ -480,8 +484,48 @@ def test_stale_update_after(self): try: self.view001(stale='update_after') except Exception as err: - self.assertFail(str(err), 'An unexpected error was encountered.') + self.fail('An unexpected error was encountered:' +str(err)) + + def test_stable_true(self): + """ + Test view query using the stable parameter set to true + + + Since there is no way to know whether the view will return a response from a stable set of + shards or not the test here focuses on ensuring that the call itself is successful. + + """ + try: + self.view001(stable=True) + except Exception as err: + self.fail('An unexpected error was encountered: '+str(err)) + + def test_stable_update_lazy(self): + """ + Test view query using the update parameter set to lazy + + Since there is no way to know whether the view will update lazily or not the test here + focuses on ensuring that the call itself is successful. + """ + try: + self.view001(update='lazy') + except Exception as err: + self.fail('An unexpected error was encountered: '+str(err)) + + def test_stable_update_true(self): + """ + Test view query using the update parameter set to true + + Since there is no way to know whether the view will update or not the test here focuses on + ensuring that the call itself is successful. + + """ + try: + self.view001(update='true') + except Exception as err: + self.fail('An unexpected error was encountered: '+str(err)) + def test_startkey_int(self): """ Test view query using startkey parameter as an integer. diff --git a/tests/unit/view_tests.py b/tests/unit/view_tests.py index 39d7efb4..a362fa77 100644 --- a/tests/unit/view_tests.py +++ b/tests/unit/view_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,19 +23,19 @@ """ import unittest + import mock -import posixpath import requests -import os - -from cloudant.design_document import DesignDocument -from cloudant.view import View, QueryIndexView from cloudant._common_util import _Code -from cloudant.result import Result +from cloudant.design_document import DesignDocument from cloudant.error import CloudantArgumentError, CloudantViewException +from cloudant.result import Result +from cloudant.view import View, QueryIndexView +from nose.plugins.attrib import attr from .unit_t_db_base import UnitTestDbBase + class CodeTests(unittest.TestCase): """ _Code class unit test @@ -79,6 +79,7 @@ def test_raise_with_proper_code(self): raise CloudantViewException(101) self.assertEqual(cm.exception.status_code, 101) +@attr(db=['cloudant','couch']) class ViewTests(UnitTestDbBase): """ View class unit tests @@ -182,7 +183,7 @@ def test_retrieve_view_url(self): view = View(ddoc, 'view001') self.assertEqual( view.url, - posixpath.join(ddoc.document_url, '_view/view001') + '/'.join((ddoc.document_url, '_view/view001')) ) def test_get_view_callable_raw_json(self): @@ -294,36 +295,6 @@ def test_view_callable_with_non_existing_view(self): except requests.HTTPError as err: self.assertEqual(err.response.status_code, 404) - @unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is None, - 'Only execute as part of CouchDB tests') - def test_view_callable_with_invalid_javascript(self): - """ - Test error condition when Javascript errors exist. This test is only - valid for CouchDB because the map function Javascript is validated on - the Cloudant server when attempting to save a design document so invalid - Javascript is not possible there. - """ - self.populate_db_with_documents() - ddoc = DesignDocument(self.db, 'ddoc001') - ddoc.add_view( - 'view001', - 'This is not valid Javascript' - ) - ddoc.save() - # Verify that the ddoc and view were saved remotely - # along with the invalid Javascript - del ddoc - ddoc = DesignDocument(self.db, 'ddoc001') - ddoc.fetch() - view = ddoc.get_view('view001') - self.assertEqual(view.map, 'This is not valid Javascript') - try: - for row in view.result: - self.fail('Above statement should raise an Exception') - except requests.HTTPError as err: - self.assertEqual(err.response.status_code, 500) - def test_custom_result_context_manager(self): """ Test that the context manager for custom results returns