diff --git a/.codeclimate.yml b/.codeclimate.yml new file mode 100644 index 0000000..67e2e7c --- /dev/null +++ b/.codeclimate.yml @@ -0,0 +1,20 @@ +# +# ---Choose Your Languages--- +# To disable analysis for a certain language, set the language to `false`. +# For help setting your languages: +# http://docs.codeclimate.com/article/169-configuring-analysis-languages +# +languages: + Ruby: false + Javascript: false + PHP: false + Python: true + +# +# ---Exclude Files or Directories--- +# List the files or directories you would like excluded from analysis. +# For help setting your exclude paths: +# http://docs.codeclimate.com/article/166-excluding-files-folders +# +exclude_paths: + - devicecloud/test/** diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..afda6a4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,25 @@ +# Set the default behavior, in case people don't have core.autocrlf set. +* text=auto + +# Explicitly declare text files you want to always be normalized and converted +# to native line endings on checkout. +*.c text +*.h text + +# Declare files that will always have LF line endings on checkout. +*.sh text eol=lf + +# Denote all files that are truly binary and should not be modified. +*.png binary +*.jpg binary + +# Existing attributes +*.java diff +*.js diff +*.pl diff +*.txt diff +*.ts diff +*.html diff +*.sh diff +*.xml diff +*.py diff diff --git a/.gitignore b/.gitignore index e768771..e118fe4 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,9 @@ env3/ .env/ .env3/ .toxenv/ +venv/ +.venv/ +.venv2/ #------------------------------------------------------------------------------- # Setuptools Stuff @@ -73,4 +76,6 @@ tmp/ #------------------------------------------------------------------------------- # Generated Files #------------------------------------------------------------------------------- -README.rst \ No newline at end of file +README.rst +*.csv +*.log diff --git a/.travis.yml b/.travis.yml index 6d1d6be..70e51a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,30 @@ +# +# This is the Travis-CI configuration. +# +# The actual dependency installation and test execution is done via tox as a +# way to share the same process between Travis-CI and Buildbot. +# language: python +# allow travis to use new, faster container based +# infrastructure to perform the testing +sudo: false # this version of python is only used to run tox - the version specified by TOX_ENV # is used to install and run tests python: 2.7 env: - TOX_ENV=py27 - - TOX_ENV=py32 - - TOX_ENV=py33 - TOX_ENV=py34 - - TOX_ENV=pypy - TOX_ENV=coverage # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors install: - - pip install tox + - pip install tox-travis coveralls - pip install -r test-requirements.txt # command to run tests, e.g. python setup.py test script: - - tox -e $TOX_ENV + - tox + +after_success: + - coveralls diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cf823c..fb1664c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,190 @@ ## Python Devicecloud Library Changelog + +### 0.5.10 / 2025-08-28 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.9...0.5.10) + +Enhancement: + +* core: loosen all package versions in requirements.txt +* core: add support for authentication using API keys + + +### 0.5.9 / 2021-02-18 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.8...0.5.9) + +Enhancement: + +* core: update other dependecies to "compatible" versions on install + + +### 0.5.8 / 2021-02-17 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.7...0.5.8) + +Enhancement: + +* datastreams: add "GEOJSON" format type to DataStreams + + +### 0.5.7 / 2020-04-22 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.6...0.5.7) + +Enhancement: + +* update requests dependency to 2.20 +* update other dependecies to "compatible" versions on install + + +### 0.5.6 / 2019-07-24 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.5...0.5.6) + +Enhancement: + +* core: update package long description + +### 0.5.5 / 2019-07-23 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.4...0.5.5) + +Enhancement: + +* core: remove subprocess during installation (make installation Windows compatible) + + +### 0.5.4 / 2019-05-22 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.3...0.5.4) + +Enhancement: + +* tests: added test for sci firmware update attributes + + +### 0.5.3 / 2018-07-24 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.1...0.5.3) + +Enhancements: + +* devicecore: add support for adding multiple tags + + +### 0.5.1 / 2018-07-20 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.0...0.5.1) + +Changes: + +* pypi: upgrade Development Status to "4 - Beta" + + +### 0.5.0 / 2018-07-20 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.4.3...0.5.0) + +Enhancements: + +* devicecore: add support for adding and removing tags from a device + + +### 0.4.3 / 2018-06-11 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.4.2...0.4.3) + +Bug Fixes: + +* core: updated Device Cloud CRT to latest +* core: updated requirements to latest + +### 0.4.2 / 2015-11-20 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.4.1...0.4.2) + +Bug Fixes: + +* core: All Etherios references have been replaced with Digi. Most + importantly, the default URL for the devicecloud is now + devicecloud.digi.com. The old URL may not redirect properly at some + point in the future. + +### 0.4.1 / 2015-11-13 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.4...0.4.1) + +Bug Fixes: + +* sci: Targetting groups now works + +### 0.4 / 2015-10-01 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.3...0.4) + +Enhancements: + +* monitors: basic support for creating HTTP monitors was added +* streams: support for the JSON data type added +* sci: support added for filesystem service added. This allows you to + access files and directories on any device supporting this service. + +Bug Fixes: + +* streams: fix data translations from device cloud <-> python types + when reading and writing data points. See + https://github.com/digidotcom/python-devicecloud/commit/5ac6c15cddf010709361c16b69e622aca93d6b28 + +### 0.3 / 2015-06-15 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.2...0.3) + +Enhancements: + +* monitor: Support for the Monitor API was added allowing for + querying, adding, and removing of monitors (limiited to TCP right + now). +* monitor: Support for listening for changes and receiving callbacks + when monitors are triggered has been added. Support is limited to + SSL/TCP for now. +* streams: get_stream() now optionally accepts a ``stream_prefix`` + that allow for restricting which streams are returned based on path. +* sci: support added for ``get_job_async`` +* devicecore: provisioning: support added for removing devices from an + account. +* core: HTTP sessions are now used in order to allow for HTTP + connection reuse and credential reuse (via cookies) + +Bug Fixes: + +* Unit tests for fixed for Python 3.4 +* filedata: ``raw`` option added to work around some issues when + retrieving binary data as base64 + +### 0.2 / 2015-01-23 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.1.1...0.2) + +Enhancements: + +* Integration tests were added for some APIs to further test library correctness +* Documentation for several parts of the system, including streams + was enhanced. +* Several APIs now support pagination on large result sets that previously lacked support. +* We now expose the underlying object used to send device cloud API requests via + `dc.get_connection()`. This allows users to more easily talk to currently unsupported + APIs. Support for the basic API verbs plus helpers for handling pagination and other + concerns are included. +* Support for group operations in devicecore is now present +* Filedata files and directories can now be deleted +* A new 'ws' API has been added for making direct web service requests +* DeviceCore now has support for provisioning one or multiple devices + +API Changes: + +* devicecore: `list_devices` was changed to `get_devices`. Previously, the + code did not match the documentation on this front. Calls to `dc.devicecore.list_devices` + will need to be changed to `dc.devicecore.get_devices` + +Bug Fixes: + +* streams: When creating a stream, the stream_id was used for the description rather + than the provided description. +* core: Library now freezes dependencies. Several dependencies updated to latest + versions (e.g. requests). Without this, some combinations of the library + and dependencies caused various errors. + +Thanks to Dan Harrison, Steve Stack, Tom Manley, and Paul Osborne for contributions +going into this release. + ### 0.1.1 / 2014-09-14 -[Full Changelog](https://github.com/etherios/python-devicecloud/compare/0.1...0.1.1) +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.1...0.1.1) Enhancements: diff --git a/HACKING.md b/CONTRIBUTING.md similarity index 69% rename from HACKING.md rename to CONTRIBUTING.md index 07cdde8..384a262 100644 --- a/HACKING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,7 @@ Environment Setup All the requirements in order to perform development on the product should be installable in a virtualenv. - $ pip install -r test-requirements.txt + $ pip install -r dev-requirements.txt In order to build a release you will also need to install pandoc. On Ubuntu, you should be able to do: @@ -20,7 +20,7 @@ Running the Unit Tests ### Running Tests with Nose -Running the tests is easy with nose (including in +Running the tests is easy with nose (included in test-requirements.txt). From the project root: $ nosetests . @@ -44,8 +44,24 @@ against all supported versions of python, just do the following: $ ./toxtest.sh This might take awhile the first time as it will build from source a -version of the interpreter for each version supported. +version of the interpreter for each version supported. If you recieve +errors from pyenv, there may be addition dependencies required. +Please visit https://github.com/yyuu/pyenv/wiki/Common-build-problems +for additional pointers. +### Running Integration and Unittests + +There are some additional integration tests that run against an actual +device cloud account. These are a bit more fragile and when something +fails, you may need to go to your device cloud account to clean things +up. + +To run those tests, you can just do the following. This script runs +the toxtest.sh script with environment variables set with your +account information. The tests that were skipped before will now +be run with each supported version of the interpreter: + + $ ./inttest.sh Build the Documentation ----------------------- @@ -60,8 +76,8 @@ activated): The docs that are built will be located at docs/_build/html/index.html. -The documentation for the project is published on github using a (Github -Pages)[https://pages.github.com/] Project Site. The process for +The documentation for the project is published on github using a [Github +Pages](https://pages.github.com/) Project Site. The process for releasing a new set of documentation is the following: 1. Create a fresh clone of the project and checkout the `gh-pages` @@ -81,5 +97,4 @@ Each source file should be prefixed with the following header: # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # - # Copyright (c) 2014 Etherios, Inc. All rights reserved. - # Etherios, Inc. is a Division of Digi International. + # Copyright (c) 2015-2018 Digi International Inc. All rights reserved. diff --git a/README.md b/README.md index a994f1c..841a778 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,24 @@ Python Device Cloud Library =========================== -[![Build Status](https://travis-ci.org/Etherios/python-devicecloud.svg?branch=master)](https://travis-ci.org/Etherios/python-devicecloud) -[![Coverage Status](https://img.shields.io/coveralls/Etherios/python-devicecloud.svg)](https://coveralls.io/r/Etherios/python-devicecloud) -[![Latest Version](https://pypip.in/version/devicecloud/badge.svg)](https://pypi.python.org/pypi/devicecloud/) -[![Supported Python versions](https://pypip.in/py_versions/devicecloud/badge.svg)](https://pypi.python.org/pypi/devicecloud/) -[![License](https://pypip.in/license/devicecloud/badge.svg)](https://pypi.python.org/pypi/devicecloud/) +[![Latest Version](https://img.shields.io/pypi/v/devicecloud.svg)](https://pypi.python.org/pypi/devicecloud/) +[![License](https://img.shields.io/badge/license-MPL%202.0-blue.svg)](https://github.com/digidotcom/python-devicecloud/blob/master/LICENSE) -Be sure to check out the [full documentation](http://etherios.github.io/python-devicecloud). -A [Changelog](https://github.com/etherios/python-devicecloud/blob/master/CHANGELOG.md) -is also available. +Be sure to check out the [full documentation](https://digidotcom.github.io/python-devicecloud). A [Changelog](https://github.com/digidotcom/python-devicecloud/blob/master/CHANGELOG.md) is also available. Overview -------- -Python-devicecloud is a library providing simple, intuitive access to -the [Device Cloud by Etherios](http://www.etherios.com/products/devicecloud/) for clients written in -Python. +Python-devicecloud is a library providing simple, intuitive access to [Digi Remote Manager](https://www.digi.com/products/iot-software-services/digi-remote-manager) for clients written in Python. -The library wraps the Device Cloud REST API and hides the details of forming HTTP requests in order to gain access to device information, -file data, streams, and other features of the device cloud. The API -wrapped can be found [here](http://ftp1.digi.com/support/documentation/90002008_redirect.htm). +The library wraps Digi Remote Manager's REST API and hides the details of forming HTTP requests in order to gain access to device information, file data, streams, and other features of Digi Remote Manager. The API can be found [here](https://doc-remotemanager.digi.com/). - -The primary target audience for this library is individuals -interfacing with the device cloud from the server side or developers -writing tools to aid device development. For efficient connectivity -from devices, we suggest that you first look at using the [Device Cloud -Connector](http://www.etherios.com/products/devicecloud/connector). -That being said, this library could also be used on devices if deemed -suitable. +The primary target audience for this library is individuals interfacing with Digi Remote Manager from the server side or developers writing tools to aid device development. Example ------- -The library provides access to a wide array of features, but here is a -couple quick examples to give you a taste of what the API looks like. +The library provides access to a wide array of features, but here is a couple quick examples to give you a taste of what the API looks like. ```python from devicecloud import DeviceCloud @@ -44,44 +27,45 @@ dc = DeviceCloud('user', 'pass') # show the MAC address of all devices that are currently connected # -# This is done using the device cloud DeviceCore functionality +# This is done using Device Cloud DeviceCore functionality print "== Connected Devices ==" -for device in dc.devicecore.list_devices(): +for device in dc.devicecore.get_devices(): if device.is_connected(): print device.get_mac() # get the name and current value of all data streams having values # with a floating point type # -# This is done using the device cloud stream functionality +# This is done using Device Cloud stream functionality for stream in dc.streams.get_streams(): if stream.get_data_type().lower() in ('float', 'double'): print "%s -> %s" % (stream.get_stream_id(), stream.get_current_value()) ``` -For more examples and detailed documentation, be sure to checkout out -the [Full API Documentation](https://etherios.github.io/python-devicecloud). +For more examples and detailed documentation, be sure to check out the [Full API Documentation](https://digidotcom.github.io/python-devicecloud). Installation ------------ -This library can be installed using -[pip](https://github.com/pypa/pip). Python versions 2.7+ (including -Python 3) are supported by the library. +This library can be installed using [pip](https://github.com/pypa/pip). Python versions 2.7+ and 3+ are supported by the library. ```sh -pip install python-devicecloud +pip install devicecloud ``` +If you already have an older version of the library installed, you can upgrade to the latest version by doing + +```sh +pip install --upgrade devicecloud +``` Supported Features ------------------ -Eventually, it is hoped that there will be complete feature parity -between the device cloud API and this library. For now, however, that -is not the case. The current features are supported by the library: +Eventually, it is hoped that there will be complete feature parity between Device Cloud API and this library. For now, however, that is not the case. The current features are supported by the library: * Getting basic device information via DeviceCore +* Provision and Delete devices via DeviceCore * Listing devices associated with a device cloud account * Interacting with Device Cloud Data Streams * Create Streams @@ -102,61 +86,46 @@ is not the case. The current features are supported by the library: * Get full metadata and contents of files and directories. * Low level support for performing basic SCI commands with limited parsing of results and support for only a subset of available services/commands. +* APIs to make direct web service calls to Device Cloud with some details + handled by the library (see DeviceCloudConnection and 'ws' documentation) +* Device Provisioning via Mac Address, IMEI or Device ID +* Monitors +* Creating a TCP or HTTP monitor The following features are *not* supported at this time. Feedback on which features should be highest priority is always welcome. * Alarms -* Monitors * Scheduled Operations -* Creating a TCP or HTTP monitor * Asynchronous SCI requests * High level access to many SCI/RCI operations * DeviceMetaData * DeviceVendor * FileDataHistory -* Group Operations (CRUD) * NetworkInterface support * XBee specific support (XBeeCore) -* Device Provisioning * Smart Energy APIs * SMS Support -* Satellite/Iridium Support * SM/UDP Support * Carrier Information Access Contributing ------------ -Contributions to the library are very welcome in whatever form can be -provided. This could include issue reports, bug fixes, or features -additions. For issue reports, please [create an issue against the -Github -project](https://github.com/Etherios/python-devicecloud/issues). +Contributions to the library are very welcome in whatever form can be provided. This could include issue reports, bug fixes, or features additions. For issue reports, please [create an issue against the Github project](https://github.com/digidotcom/python-devicecloud/issues). -For code changes, feel free to fork the project on Github and submit a -pull request with your changes. Additional instructions for -developers contributing to the project can be found in the [Developer's -Guide](https://github.com/Etherios/python-devicecloud/blob/master/HACKING.md). +For code changes, feel free to fork the project on Github and submit a pull request with your changes. Additional instructions for developers contributing to the project can be found in the [Developer's Guide](https://github.com/digidotcom/python-devicecloud/blob/master/CONTRIBUTING.md). License ------- This software is open-source software. -Copyright (c) 2014, Etherios, Inc. All rights reserved. -Etherios, Inc. is a Division of Digi International. +Copyright (c) 2015-2025 Digi International Inc. -This Source Code Form is subject to the terms of the Mozilla Public -License, v. 2.0. If a copy of the MPL was not distributed with this file, -you can obtain one at http://mozilla.org/MPL/2.0/. +This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, you can obtain one at https://mozilla.org/MPL/2.0/. -Digi, Digi International, the Digi logo, the Digi website, Etherios, -the Etherios logo, the Etherios website, Device Cloud by Etherios, and -Etherios Cloud Connector are trademarks or registered trademarks of -Digi International, Inc. in the United States and other countries -worldwide. All other trademarks are the property of their respective -owners. +Digi, Digi International, the Digi logo, the Digi website, Digi Device Cloud, Digi Remote Manager, and Digi Cloud Connector are trademarks or registered trademarks of Digi International Inc. in the United States and other countries worldwide. All other trademarks are the property of their respective owners. THE SOFTWARE AND RELATED TECHNICAL INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..d8f8f40 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,6 @@ +-r test-requirements.txt +coverage +tox +pyandoc +sphinx +sphinx_rtd_theme diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index fd8e6d6..92f5dd1 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -2,21 +2,24 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015-2025 Digi International Inc. -from requests.auth import HTTPBasicAuth import logging -import requests import time import json +from devicecloud.util import validate_type +from requests.auth import HTTPBasicAuth, AuthBase +import requests from devicecloud.version import __version__ +import six + __all__ = ( 'DeviceCloud', 'DeviceCloudException', 'DeviceCloudHttpException', + 'DeviceCloudConnection', ) SUCCESSFUL_STATUS_CODES = [ @@ -24,8 +27,31 @@ 201, # Created 202, # Accepted 204, # No Content (success for DELETE operation) + 207, # Multi-Status (some success for provisioning, parse it before raising exception) ] +HTTP_THROTTLED_CODES = [ + 429 +] + +# How long in seconds should we delay if a request is throttled? +# +# With a start of 1 second delay and a max of 10 and a default of 5 retries with a backoff coefficient +# of 1.5, then the backoff sequence will be the following: +# +# 1, 1.5, 2.25, 3.375, ~5 +# +# With total delays of the following adding up at each period: +# +# 1, 2.5, 4.75, 8.125, 13.1875 +# +# Assuming a 10s window, this will ensure that we hit a new window before the 5 retries +# are exhausted +DEFAULT_THROTTLE_RETRIES = 5 +DEFAULT_THROTTLE_DELAY_INIT = 1.0 +DEFAULT_THROTTLE_DELAY_MAX = 10.0 +DEFAULT_THROTTLE_DELAY_BACKOFF_COEFFICIENT = 1.5 + logger = logging.getLogger("devicecloud") @@ -34,44 +60,156 @@ class DeviceCloudException(Exception): class DeviceCloudHttpException(DeviceCloudException): - """Exception raised when we failed a request to the DC over HTTP""" + """Exception raised when we failed a request to the DC over HTTP + + This exception will be raised whenever a non-success HTTP status + code is received from Device Cloud and there is no other logic + in place for gracefully handling the error case. + + Often, if there is an error with a request to Device Cloud, the device + cloud will respond with an error status and include additional information + about the nature of the error in the response body. This information + can be accessed via the :attr:`~response` property. + + """ def __init__(self, response, *args, **kwargs): DeviceCloudException.__init__(self, *args, **kwargs) - self.response = response + self._response = response + + def __str__(self): + return "HTTP Status {code}: {content}".format( + code=self.response.status_code, + content=self.response.content + ) + + @property + def response(self): + """Get the requests response object for the failing HTTP request + + This object will be an instance of :class:`requests.Response` which + in turn provides information including the content (body) of the response + and the HTTP status code:: + + try: + dc.sci.send_request(...) + except DeviceCloudHttpException as e: + print "HTTP Error: %s" % e.response.status_code + print e.response.content + + """ + return self._response + +class DeviceCloudConnection(object): + """Provide low-level access to Device Cloud web services -class _DeviceCloudConnection(object): - """Encapsulate information about a connection to the device cloud + This is a convenience object that provides methods that make sending requests to the + device cloud easier. This object is used extensively within the library but can + also be used externally (for instance, to support an API exposed by the device + cloud that is not currently supported in the library). - This class is used internally and does not represent a part of the public API - to the device cloud. + This object is accessible via :meth:`~DeviceCloud.get_connection`. """ - def __init__(self, auth, base_url): + def __init__(self, auth, base_url, + throttle_retries=DEFAULT_THROTTLE_RETRIES, + throttle_delay_init=DEFAULT_THROTTLE_DELAY_INIT, + throttle_delay_max=DEFAULT_THROTTLE_DELAY_MAX, + throttle_delay_backoff_coefficient=DEFAULT_THROTTLE_DELAY_BACKOFF_COEFFICIENT): self._auth = auth self._base_url = base_url + self._throttle_retries = throttle_retries + self._throttle_delay_init = throttle_delay_init + self._throttle_delay_max = throttle_delay_max + self._throttle_delay_backoff_coefficient = throttle_delay_backoff_coefficient + self._session = requests.Session() + self._session.auth = auth + + @property + def hostname(self): + """Get the hostname that this connection is associated with""" + from six.moves.urllib.parse import urlparse + return urlparse(self._base_url).netloc.split(':', 1)[0] + + @property + def username(self): + return self._auth.username + + @property + def password(self): + return self._auth.password def _make_url(self, path): if not path.startswith("/"): path = "/" + path return "%s%s" % (self._base_url, path) - def _make_request(self, retries, method, url, **kwargs): - remaining_attempts = retries + 1 + def _make_request(self, method, url, **kwargs): + # + # Make a request, retrying up to 'retries' times backing off according to defined constants + # + throttle_retries = kwargs.pop('throttle_retries', kwargs.pop('retries', self._throttle_retries)) + throttle_delay_init = kwargs.pop('throttle_delay_init', self._throttle_delay_init) + throttle_delay_max = kwargs.pop('throttle_delay_max', self._throttle_delay_max) + throttle_delay_backoff_coefficient = \ + kwargs.pop('throttle_delay_backoff_coefficient', self._throttle_delay_backoff_coefficient) + + remaining_attempts = throttle_retries + 1 + retry_delay = throttle_delay_init while remaining_attempts > 0: - response = requests.request(method, url, auth=self._auth, **kwargs) + response = self._session.request(method, url, **kwargs) if response.status_code in SUCCESSFUL_STATUS_CODES: return response - remaining_attempts -= 1 - time.sleep(1) + elif response.status_code in HTTP_THROTTLED_CODES: + remaining_attempts -= 1 + if remaining_attempts > 0: + logger.info("Request throttled on attempt {attempt}/{max_attempts}, retrying in {delay} seconds".format( + attempt=(throttle_retries + 1 - remaining_attempts), + max_attempts=throttle_retries, + delay=retry_delay + )) + time.sleep(retry_delay) + retry_delay = min(retry_delay * throttle_delay_backoff_coefficient, throttle_delay_max) + else: + break err = "DC %s to %s failed - HTTP(%s)" % (method, url, response.status_code) raise DeviceCloudHttpException(response, err) + def iter_json_pages(self, path, page_size=1000, **params): + """Return an iterator over JSON items from a paginated resource + + Legacy resources (prior to V1) implemented a common paging interfaces for + several different resources. This method handles the details of iterating + over the paged result set, yielding only the JSON data for each item + within the aggregate resource. + + :param str path: The base path to the resource being requested (e.g. /ws/Group) + :param int page_size: The number of items that should be requested for each page. A larger + page_size may mean fewer HTTP requests but could also increase the time to get a first + result back from Device Cloud. + :param params: These are additional query parameters that should be sent with each + request to Device Cloud. + + """ + path = validate_type(path, *six.string_types) + page_size = validate_type(page_size, *six.integer_types) + + offset = 0 + remaining_size = 1 # just needs to be non-zero + while remaining_size > 0: + reqparams = {"start": offset, "size": page_size} + reqparams.update(params) + response = self.get_json(path, params=reqparams) + offset += page_size + remaining_size = int(response.get("remainingSize", "0")) + for item_json in response.get("items", []): + yield item_json + def ping(self): - """Ping the Device Cloud using the authorization provided + """Ping Device Cloud using the authorization provided :return: The response of getting a single device from DeviceCore on success :raises: :class:`.DeviceCloudHttpException` if there is a problem @@ -79,34 +217,124 @@ def ping(self): """ return self.get("/ws/DeviceCore?size=1") - def get(self, path, retries=0, **kwargs): + def get(self, path, **kwargs): + """Perform an HTTP GET request of the specified path in Device Cloud + + Make an HTTP GET request against Device Cloud with this accounts + credentials and base url. This method uses the + `requests `_ library + `request method `_ + and all keyword arguments will be passed on to that method. + + :param str path: Device Cloud path to GET + :param int retries: The number of times the request should be retried if an + unsuccessful response is received. Most likely, you should leave this at 0. + :raises DeviceCloudHttpException: if a non-success response to the request is received + from Device Cloud + :returns: A requests ``Response`` object + + """ url = self._make_url(path) - return self._make_request(retries, "GET", url, **kwargs) + return self._make_request("GET", url, **kwargs) + + def get_json(self, path, **kwargs): + """Perform an HTTP GET request with JSON headers of the specified path against Device Cloud + + Make an HTTP GET request against Device Cloud with this accounts + credentials and base url. This method uses the + `requests `_ library + `request method `_ + and all keyword arguments will be passed on to that method. + + This method will automatically add the ``Accept: application/json`` and parse the + JSON response from Device Cloud. + + :param str path: Device Cloud path to GET + :param int retries: The number of times the request should be retried if an + unsuccessful response is received. Most likely, you should leave this at 0. + :raises DeviceCloudHttpException: if a non-success response to the request is received + from Device Cloud + :returns: A python data structure containing the results of calling ``json.loads`` on the + body of the response from Device Cloud. + + """ - def get_json(self, path, retries=0, **kwargs): url = self._make_url(path) headers = kwargs.setdefault('headers', {}) headers.update({'Accept': 'application/json'}) - response = self._make_request(retries, "GET", url, **kwargs) + response = self._make_request("GET", url, **kwargs) return json.loads(response.text) - def post(self, path, data, retries=0, **kwargs): + def post(self, path, data, **kwargs): + """Perform an HTTP POST request of the specified path in Device Cloud + + Make an HTTP POST request against Device Cloud with this accounts + credentials and base url. This method uses the + `requests `_ library + `request method `_ + and all keyword arguments will be passed on to that method. + + :param str path: Device Cloud path to POST + :param int retries: The number of times the request should be retried if an + unsuccessful response is received. Most likely, you should leave this at 0. + :param data: The data to be posted in the body of the POST request (see docs for + ``requests.post`` + :raises DeviceCloudHttpException: if a non-success response to the request is received + from Device Cloud + :returns: A requests ``Response`` object + + """ url = self._make_url(path) - return self._make_request(retries, "POST", url, data=data, **kwargs) + return self._make_request("POST", url, data=data, **kwargs) + + def put(self, path, data, **kwargs): + """Perform an HTTP PUT request of the specified path in Device Cloud + + Make an HTTP PUT request against Device Cloud with this accounts + credentials and base url. This method uses the + `requests `_ library + `request method `_ + and all keyword arguments will be passed on to that method. + + :param str path: Device Cloud path to PUT + :param int retries: The number of times the request should be retried if an + unsuccessful response is received. Most likely, you should leave this at 0. + :param data: The data to be posted in the body of the POST request (see docs for + ``requests.post`` + :raises DeviceCloudHttpException: if a non-success response to the request is received + from Device Cloud + :returns: A requests ``Response`` object + + """ - def put(self, path, data, retries=0, **kwargs): url = self._make_url(path) - return self._make_request(retries, "PUT", url, data=data, **kwargs) + return self._make_request("PUT", url, data=data, **kwargs) + + def delete(self, path, retries=DEFAULT_THROTTLE_RETRIES, **kwargs): + """Perform an HTTP DELETE request of the specified path in Device Cloud + + Make an HTTP DELETE request against Device Cloud with this accounts + credentials and base url. This method uses the + `requests `_ library + `request method `_ + and all keyword arguments will be passed on to that method. - def delete(self, path, retries=0): + :param str path: Device Cloud path to DELETE + :param int retries: The number of times the request should be retried if an + unsuccessful response is received. Most likely, you should leave this at 0. + :raises DeviceCloudHttpException: if a non-success response to the request is received + from Device Cloud + :returns: A requests ``Response`` object + + """ url = self._make_url(path) - return self._make_request(retries, "DELETE", url) + return self._make_request("DELETE", url, **kwargs) class DeviceCloud(object): """Provide access to core device cloud features - This class is the primary interface to the device cloud through which access to individual + This class is the primary interface to Device Cloud through which access to individual device cloud services is provided. Creating a ``DeviceCloud`` object is as easy as doing the following:: @@ -114,25 +342,55 @@ class DeviceCloud(object): dc = DeviceCloud('user', 'pass') if dc.has_valid_credentials(): - print dc.devicecore.list_devices() + print list(dc.devicecore.get_devices()) + + It's also possible to authenticate using API keys:: - From there, access to all of the device clouds features are possible. In some cases, methods + dc = DeviceCloud(api_key_id='my-id', api_key_secret='my-secret') + + From there, access to all of Device Clouds features are possible. In some cases, methods for quickly performing selected actions may be provided directly via the ``DeviceCloud`` object while advanced usage requires using functionality exposed through other interfaces. """ - def __init__(self, username, password, base_url="https://login.etherios.com"): - self._conn = _DeviceCloudConnection(HTTPBasicAuth(username, password), base_url) + def __init__(self, username=None, password=None, + api_key_id=None, api_key_secret=None, + base_url=None, + throttle_retries=DEFAULT_THROTTLE_RETRIES, + throttle_delay_init=DEFAULT_THROTTLE_DELAY_INIT, + throttle_delay_max=DEFAULT_THROTTLE_DELAY_MAX, + throttle_delay_backoff_coefficient=DEFAULT_THROTTLE_DELAY_BACKOFF_COEFFICIENT): + if base_url is None: + base_url = "https://devicecloud.digi.com" + + if username and password: + auth = HTTPBasicAuth(username, password) + elif api_key_id and api_key_secret: + auth = ApiKeyAuth(api_key_id, api_key_secret) + else: + raise ValueError("Must provide either username/password or api_key_id/api_key_secret") + + self._conn = DeviceCloudConnection( + auth=auth, + base_url=base_url, + throttle_retries=throttle_retries, + throttle_delay_init=throttle_delay_init, + throttle_delay_max=throttle_delay_max, + throttle_delay_backoff_coefficient=throttle_delay_backoff_coefficient + ) self._streams_api = None # streams property api ref self._filedata_api = None # filedata property api ref + self._fss_api = None # fileservice property api ref self._devicecore_api = None # devicecore property api ref self._sci_api = None # sci property api ref + self._monitor_api = None # monitor property of api ref + self._legacy_api = None # legacy property api ref def has_valid_credentials(self): - """Verify that the device cloud url, username, and password are valid + """Verify that Device Cloud url, username, and password are valid - This method will attempt to "ping" the device cloud in order to ensure that all + This method will attempt to "ping" Device Cloud in order to ensure that all of the provided information is correct. :return: True if the credentials are valid and false if not @@ -174,6 +432,34 @@ def sci(self): self._sci_api = self.get_sci_api() return self._sci_api + @property + def file_system_service(self): + """Property providing access to the :class:`.FileSystemServiceAPI`""" + if self._fss_api is None: + self._fss_api = self.get_fss_api() + return self._fss_api + + @property + def monitor(self): + """Property providing access to the :class:`.MonitorAPI`""" + if self._monitor_api is None: + self._monitor_api = self.get_monitor_api() + return self._monitor_api + + @property + def ws(self): + """Property providing access to the :class:`.WebServiceStub` with a base of ``/ws``""" + return self.get_web_service_stub() + + def get_connection(self): + """Get the low-level :class:`~DeviceCloudConnection` for this device cloud instance + + This object provides a low-level interface for making authenticated requests + to Device Cloud. + + """ + return self._conn + def get_streams_api(self): """Returns a :class:`.StreamsAPI` bound to this device cloud instance @@ -229,3 +515,58 @@ def get_sci_api(self): from devicecloud.sci import ServerCommandInterfaceAPI return ServerCommandInterfaceAPI(self._conn) + + def get_fss_api(self): + """Returns a :class:`.FileSystemServiceAPI` bound to this device cloud instance + + This provides access to the same API as :attr:`.DeviceCloud.file_system_service` but will create + a new object (with a new cache) each time called. + + :return: FSS API object bound to this device cloud account + :rtype: :class:`.FileSystemServiceAPI` + + """ + from devicecloud.file_system_service import FileSystemServiceAPI + + return FileSystemServiceAPI(self.sci) + + def get_monitor_api(self): + """Returns a :class:`.MonitorAPI` bound to this device cloud instance + + This provides access to the same API as :attr:`.DeviceCloud.monitor` but will create + a new object (with a new cache) each time called. + + :return: Monitor API object bound to this device cloud account + :rtype: :class:`.MonitorAPI` + + """ + from devicecloud.monitor import MonitorAPI + + return MonitorAPI(self._conn) + + def get_web_service_stub(self): + """Returns a :class:`.WebServiceStub` bound to this device cloud instance + + This provides access to the same API as :attr:`.DeviceCloud.legacy` but will create + a new object (with a new cache) each time called. + + :return: WebServiceStub object bound to this device cloud account with a base of ``/ws`` + :rtype: :class:`.WebServiceStub` + + """ + from devicecloud.ws import WebServiceStub + + return WebServiceStub(self._conn, "/ws") + + +class ApiKeyAuth(AuthBase): + """Custom Auth class for API key authentication.""" + + def __init__(self, api_key_id: str, api_key_secret: str): + self.api_key_id = api_key_id + self.api_key_secret = api_key_secret + + def __call__(self, r): + r.headers["X-API-KEY-ID"] = self.api_key_id + r.headers["X-API-KEY-SECRET"] = self.api_key_secret + return r diff --git a/devicecloud/apibase.py b/devicecloud/apibase.py index 96b4ba2..e7b7715 100644 --- a/devicecloud/apibase.py +++ b/devicecloud/apibase.py @@ -2,9 +2,21 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015-2018 Digi International Inc. class APIBase(object): + """Base class for all API Classes + + :type _conn: devicecloud.DeviceCloudConnection + """ def __init__(self, conn): self._conn = conn + + +class SCIAPIBase(object): + """Base class for API classes using SCI to communicate + + :type _sci_api: devicecloud.sci.ServerCommandInterfaceAPI + """ + def __init__(self, sci_api): + self._sci_api = sci_api diff --git a/devicecloud/conditions.py b/devicecloud/conditions.py index 81046e0..b6844ab 100644 --- a/devicecloud/conditions.py +++ b/devicecloud/conditions.py @@ -2,12 +2,11 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015-2018 Digi International Inc. """Module with functionality for building queries against cloud resources -This functionality is somewhat poorly documented in the device cloud documentation +This functionality is somewhat poorly documented in Device Cloud documentation in the `Compound Queries `_ section. @@ -16,14 +15,12 @@ from devicecloud.util import isoformat, to_none_or_dt -import six - def _quoted(value): """Return a single-quoted and escaped (percent-encoded) version of value This function will also perform transforms of known data types to a representation - that will be handled by the device cloud. For instance, datetime objects will be + that will be handled by Device Cloud. For instance, datetime objects will be converted to ISO8601. """ @@ -32,7 +29,7 @@ def _quoted(value): else: value = str(value) - return "'{}'".format(six.moves.urllib.parse.quote(value)) + return "'{}'".format(value) class Expression(object): @@ -108,8 +105,8 @@ def compile(self): class Attribute(object): """An attribute is a piece of data on which we may perform comparisons - Comparisons performed to attributes generated :class:`~Condition`s. - + Comparisons performed to attributes will in turn generate new + :class:`.Comparison` instances. """ def __init__(self, name): diff --git a/devicecloud/data/__init__.py b/devicecloud/data/__init__.py new file mode 100644 index 0000000..1c52222 --- /dev/null +++ b/devicecloud/data/__init__.py @@ -0,0 +1,6 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. +# diff --git a/devicecloud/data/devicecloud.crt b/devicecloud/data/devicecloud.crt new file mode 100644 index 0000000..928955f --- /dev/null +++ b/devicecloud/data/devicecloud.crt @@ -0,0 +1,121 @@ +-----BEGIN CERTIFICATE----- +MIIHXTCCBkWgAwIBAgIJANZr7DJFVmHfMA0GCSqGSIb3DQEBCwUAMIG0MQswCQYD +VQQGEwJVUzEQMA4GA1UECBMHQXJpem9uYTETMBEGA1UEBxMKU2NvdHRzZGFsZTEa +MBgGA1UEChMRR29EYWRkeS5jb20sIEluYy4xLTArBgNVBAsTJGh0dHA6Ly9jZXJ0 +cy5nb2RhZGR5LmNvbS9yZXBvc2l0b3J5LzEzMDEGA1UEAxMqR28gRGFkZHkgU2Vj +dXJlIENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTE3MDIxMzIxMzIwMVoX +DTE5MDMxMjEzMTgzOFowgdcxEzARBgsrBgEEAYI3PAIBAxMCVVMxGjAYBgsrBgEE +AYI3PAIBAhMJTWlubmVzb3RhMR0wGwYDVQQPExRQcml2YXRlIE9yZ2FuaXphdGlv +bjEOMAwGA1UEBRMFNDg4NTkxCzAJBgNVBAYTAlVTMRIwEAYDVQQIEwlNaW5uZXNv +dGExEzARBgNVBAcTCk1pbm5ldG9ua2ExIDAeBgNVBAoTF0RpZ2kgSW50ZXJuYXRp +b25hbCBJbmMuMR0wGwYDVQQDExRkZXZpY2VjbG91ZC5kaWdpLmNvbTCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBAOiQG1upAjqcisuXt+Kz/3RTbbWli8q1 +a4KtcIg0y19gClNsSWVioHH4KFBlYEafcL42MT3LP5+WMEGedBzf6rm7b7MW4IvY +zMUqilUWxXhMtgFRZP/NQlugFCGVL529qult/ZZ6Oo+DTdBukduEdCtXU1KJuFwh +MP1zL85TxTyAt2t7TrWO5Pat9lQGxd/LjC0mI7TdQ++9VNsAzDcYWrP9TSGM8wsr +lU739jLQhRz6Q4UYUPFk8PEvp6x4wdxFMhTqOIQ4McOkjokHLd6WXcjMhY4tPgTJ +hNuiYgtoGj+EGI2XDWPDAc+dQTKWfMa+ykZxbyb4lXyqldARir/OrwsCAwEAAaOC +A0swggNHMAwGA1UdEwEB/wQCMAAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUF +BwMCMA4GA1UdDwEB/wQEAwIFoDA1BgNVHR8ELjAsMCqgKKAmhiRodHRwOi8vY3Js +LmdvZGFkZHkuY29tL2dkaWcyczMtNy5jcmwwXAYDVR0gBFUwUzBIBgtghkgBhv1t +AQcXAzA5MDcGCCsGAQUFBwIBFitodHRwOi8vY2VydGlmaWNhdGVzLmdvZGFkZHku +Y29tL3JlcG9zaXRvcnkvMAcGBWeBDAEBMHYGCCsGAQUFBwEBBGowaDAkBggrBgEF +BQcwAYYYaHR0cDovL29jc3AuZ29kYWRkeS5jb20vMEAGCCsGAQUFBzAChjRodHRw +Oi8vY2VydGlmaWNhdGVzLmdvZGFkZHkuY29tL3JlcG9zaXRvcnkvZ2RpZzIuY3J0 +MB8GA1UdIwQYMBaAFEDCvSeOzDSDMKIz1/tss/C0LIDOMDkGA1UdEQQyMDCCFGRl +dmljZWNsb3VkLmRpZ2kuY29tghh3d3cuZGV2aWNlY2xvdWQuZGlnaS5jb20wHQYD +VR0OBBYEFFvYGl4ynfmb11I5vDrDq1/excTEMIIBfgYKKwYBBAHWeQIEAgSCAW4E +ggFqAWgAdgBWFAaaL9fC7NP14b1Esj7HRna5vJkRXMDvlJhV1onQ3QAAAVo5ZQR8 +AAAEAwBHMEUCIEQfzk/nT9zPlxJEnYsI0jdcPqIhFTYCevqyAE4EXrgYAiEAmeWb +BVXzM52QxKPOtXpgKOMPjUihpqqgJ9mKjmejgI4AdQDuS723dc5guuFCaR+r4Z5m +ow9+X7By2IMAxHuJeqj9ywAAAVo5ZQhLAAAEAwBGMEQCIAjkl2UpriTtD4bdTjTD +4IUMaYZMN0NdVsDk5oxSJxCiAiAOtR7KSAKkRn0gEdyTrJpQPolgCLNHj55GPOwZ +oDzX3AB3AKS5CZC0GFgUh7sTosxncAo8NZgE+RvfuON3zQ7IDdwQAAABWjllCWQA +AAQDAEgwRgIhAMw0R6P4MFmCGO1x+QgHeBBWYeVQvMXhGbwN+Ffh4OXoAiEA/kOC +UfexoAx+PmKQ1cQKTKkRCzLhTxzCAOaySMrm/iAwDQYJKoZIhvcNAQELBQADggEB +AGLrc578DVbChRu29j/+c4Q9jlLW8WARTcAbK3dlAX28Hx3jlLEkfWA2aFCoQsva +thgGik6sMOUTkAzV8Qof+6akyeNaVh2yIByNupsnXgsC1SM5qD4UmfBdPGwzW3Xd +RWGN+LM6cmGBzxNOVH2NxdZ/XCGm282tmBnDsXioIQ7fo0EW681svJRCUwbYv4mm +8wwX8ALNVlODfhs4TwxASOMB2+uFEXRzJ15BawGeXDPg7C1o/giHru6vbO2DVCga +7eeCxCnM4NJ/dCzc2+K6FqNGL3ddQzAmEtwh4frHEs2KK6/fP2FeZA2WzOTJ4ZXx +jcicTeajht5zfh8/I94rwJI= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIE0DCCA7igAwIBAgIBBzANBgkqhkiG9w0BAQsFADCBgzELMAkGA1UEBhMCVVMx +EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxGjAYBgNVBAoT +EUdvRGFkZHkuY29tLCBJbmMuMTEwLwYDVQQDEyhHbyBEYWRkeSBSb290IENlcnRp +ZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTExMDUwMzA3MDAwMFoXDTMxMDUwMzA3 +MDAwMFowgbQxCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6b25hMRMwEQYDVQQH +EwpTY290dHNkYWxlMRowGAYDVQQKExFHb0RhZGR5LmNvbSwgSW5jLjEtMCsGA1UE +CxMkaHR0cDovL2NlcnRzLmdvZGFkZHkuY29tL3JlcG9zaXRvcnkvMTMwMQYDVQQD +EypHbyBEYWRkeSBTZWN1cmUgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IC0gRzIwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC54MsQ1K92vdSTYuswZLiBCGzD +BNliF44v/z5lz4/OYuY8UhzaFkVLVat4a2ODYpDOD2lsmcgaFItMzEUz6ojcnqOv +K/6AYZ15V8TPLvQ/MDxdR/yaFrzDN5ZBUY4RS1T4KL7QjL7wMDge87Am+GZHY23e +cSZHjzhHU9FGHbTj3ADqRay9vHHZqm8A29vNMDp5T19MR/gd71vCxJ1gO7GyQ5HY +pDNO6rPWJ0+tJYqlxvTV0KaudAVkV4i1RFXULSo6Pvi4vekyCgKUZMQWOlDxSq7n +eTOvDCAHf+jfBDnCaQJsY1L6d8EbyHSHyLmTGFBUNUtpTrw700kuH9zB0lL7AgMB +AAGjggEaMIIBFjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNV +HQ4EFgQUQMK9J47MNIMwojPX+2yz8LQsgM4wHwYDVR0jBBgwFoAUOpqFBxBnKLbv +9r0FQW4gwZTaD94wNAYIKwYBBQUHAQEEKDAmMCQGCCsGAQUFBzABhhhodHRwOi8v +b2NzcC5nb2RhZGR5LmNvbS8wNQYDVR0fBC4wLDAqoCigJoYkaHR0cDovL2NybC5n +b2RhZGR5LmNvbS9nZHJvb3QtZzIuY3JsMEYGA1UdIAQ/MD0wOwYEVR0gADAzMDEG +CCsGAQUFBwIBFiVodHRwczovL2NlcnRzLmdvZGFkZHkuY29tL3JlcG9zaXRvcnkv +MA0GCSqGSIb3DQEBCwUAA4IBAQAIfmyTEMg4uJapkEv/oV9PBO9sPpyIBslQj6Zz +91cxG7685C/b+LrTW+C05+Z5Yg4MotdqY3MxtfWoSKQ7CC2iXZDXtHwlTxFWMMS2 +RJ17LJ3lXubvDGGqv+QqG+6EnriDfcFDzkSnE3ANkR/0yBOtg2DZ2HKocyQetawi +DsoXiWJYRBuriSUBAA/NxBti21G00w9RKpv0vHP8ds42pM3Z2Czqrpv1KrKQ0U11 +GIo/ikGQI31bS/6kA1ibRrLDYGCD+H1QQc7CoZDDu+8CL9IVVO5EFdkKrqeKM+2x +LXY2JtwE65/3YR8V3Idv7kaWKK2hJn0KCacuBKONvPi8BDAB +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEfTCCA2WgAwIBAgIDG+cVMA0GCSqGSIb3DQEBCwUAMGMxCzAJBgNVBAYTAlVT +MSEwHwYDVQQKExhUaGUgR28gRGFkZHkgR3JvdXAsIEluYy4xMTAvBgNVBAsTKEdv +IERhZGR5IENsYXNzIDIgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTQwMTAx +MDcwMDAwWhcNMzEwNTMwMDcwMDAwWjCBgzELMAkGA1UEBhMCVVMxEDAOBgNVBAgT +B0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxGjAYBgNVBAoTEUdvRGFkZHku +Y29tLCBJbmMuMTEwLwYDVQQDEyhHbyBEYWRkeSBSb290IENlcnRpZmljYXRlIEF1 +dGhvcml0eSAtIEcyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv3Fi +CPH6WTT3G8kYo/eASVjpIoMTpsUgQwE7hPHmhUmfJ+r2hBtOoLTbcJjHMgGxBT4H +Tu70+k8vWTAi56sZVmvigAf88xZ1gDlRe+X5NbZ0TqmNghPktj+pA4P6or6KFWp/ +3gvDthkUBcrqw6gElDtGfDIN8wBmIsiNaW02jBEYt9OyHGC0OPoCjM7T3UYH3go+ +6118yHz7sCtTpJJiaVElBWEaRIGMLKlDliPfrDqBmg4pxRyp6V0etp6eMAo5zvGI +gPtLXcwy7IViQyU0AlYnAZG0O3AqP26x6JyIAX2f1PnbU21gnb8s51iruF9G/M7E +GwM8CetJMVxpRrPgRwIDAQABo4IBFzCCARMwDwYDVR0TAQH/BAUwAwEB/zAOBgNV +HQ8BAf8EBAMCAQYwHQYDVR0OBBYEFDqahQcQZyi27/a9BUFuIMGU2g/eMB8GA1Ud +IwQYMBaAFNLEsNKR1EwRcbNhyz2h/t2oatTjMDQGCCsGAQUFBwEBBCgwJjAkBggr +BgEFBQcwAYYYaHR0cDovL29jc3AuZ29kYWRkeS5jb20vMDIGA1UdHwQrMCkwJ6Al +oCOGIWh0dHA6Ly9jcmwuZ29kYWRkeS5jb20vZ2Ryb290LmNybDBGBgNVHSAEPzA9 +MDsGBFUdIAAwMzAxBggrBgEFBQcCARYlaHR0cHM6Ly9jZXJ0cy5nb2RhZGR5LmNv +bS9yZXBvc2l0b3J5LzANBgkqhkiG9w0BAQsFAAOCAQEAWQtTvZKGEacke+1bMc8d +H2xwxbhuvk679r6XUOEwf7ooXGKUwuN+M/f7QnaF25UcjCJYdQkMiGVnOQoWCcWg +OJekxSOTP7QYpgEGRJHjp2kntFolfzq3Ms3dhP8qOCkzpN1nsoX+oYggHFCJyNwq +9kIDN0zmiN/VryTyscPfzLXs4Jlet0lUIDyUGAzHHFIYSaRt4bNYC8nY7NmuHDKO +KHAN4v6mF56ED71XcLNa6R+ghlO773z/aQvgSMO3kwvIClTErF0UZzdsyqUvMQg3 +qm5vjLyb4lddJIGvl5echK1srDdMZvNhkREg5L4wn3qkKQmw4TRfZHcYQFHfjDCm +rw== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEADCCAuigAwIBAgIBADANBgkqhkiG9w0BAQUFADBjMQswCQYDVQQGEwJVUzEh +MB8GA1UEChMYVGhlIEdvIERhZGR5IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBE +YWRkeSBDbGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTA0MDYyOTE3 +MDYyMFoXDTM0MDYyOTE3MDYyMFowYzELMAkGA1UEBhMCVVMxITAfBgNVBAoTGFRo +ZSBHbyBEYWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28gRGFkZHkgQ2xhc3Mg +MiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASAwDQYJKoZIhvcNAQEBBQADggEN +ADCCAQgCggEBAN6d1+pXGEmhW+vXX0iG6r7d/+TvZxz0ZWizV3GgXne77ZtJ6XCA +PVYYYwhv2vLM0D9/AlQiVBDYsoHUwHU9S3/Hd8M+eKsaA7Ugay9qK7HFiH7Eux6w +wdhFJ2+qN1j3hybX2C32qRe3H3I2TqYXP2WYktsqbl2i/ojgC95/5Y0V4evLOtXi +EqITLdiOr18SPaAIBQi2XKVlOARFmR6jYGB0xUGlcmIbYsUfb18aQr4CUWWoriMY +avx4A6lNf4DD+qta/KFApMoZFv6yyO9ecw3ud72a9nmYvLEHZ6IVDd2gWMZEewo+ +YihfukEHU1jPEX44dMX4/7VpkI+EdOqXG68CAQOjgcAwgb0wHQYDVR0OBBYEFNLE +sNKR1EwRcbNhyz2h/t2oatTjMIGNBgNVHSMEgYUwgYKAFNLEsNKR1EwRcbNhyz2h +/t2oatTjoWekZTBjMQswCQYDVQQGEwJVUzEhMB8GA1UEChMYVGhlIEdvIERhZGR5 +IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBEYWRkeSBDbGFzcyAyIENlcnRpZmlj +YXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQAD +ggEBADJL87LKPpH8EsahB4yOd6AzBhRckB4Y9wimPQoZ+YeAEW5p5JYXMP80kWNy +OO7MHAGjHZQopDH2esRU1/blMVgDoszOYtuURXO1v0XJJLXVggKtI3lpjbi2Tc7P +TMozI+gciKqdi0FuFskg5YmezTvacPd+mSYgFFQlq25zheabIZ0KbIIOqPjCDPoQ +HmyW74cNxA9hi63ugyuV+I6ShHI56yDqg+2DzZduCLzrTia2cyvk0/ZM/iZx4mER +dEr/VxqHD3VILs9RaRegAhJhldXRQLIQTO7ErBBDpqWeCtWVYpoNz4iCxTIM5Cuf +ReYNnyicsbkqWletNw+vHX/bvZ8= +-----END CERTIFICATE----- diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index 5ddcd2a..c5b5ee0 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -2,11 +2,40 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015-2018 Digi International Inc. + +import sys +import xml.etree.ElementTree as ET from devicecloud.apibase import APIBase -from devicecloud.util import iso8601_to_dt +from devicecloud.conditions import Attribute, Expression +from devicecloud.util import iso8601_to_dt, validate_type +import six +from xml.sax.saxutils import escape + + +dev_mac = Attribute('devMac') +group_id = Attribute('grpId') +group_path = Attribute('grpPath') +dev_connectware_id = Attribute('devConnectwareId') +# TODO: Can we support location based device lookups? (e.g. lat/long?) + + +ADD_GROUP_TEMPLATE = \ + """ + + {connectware_id} + {group_path} + +""" + +TAGS_TEMPLATE = \ + """ + + {connectware_id} + {tags} + +""" class DeviceCoreAPI(APIBase): @@ -16,24 +45,354 @@ def __init__(self, conn, sci): APIBase.__init__(self, conn) self._sci = sci - def list_devices(self): - """Retrieve a list of :class:`Device` objects for this device cloud account""" - devicecore_data = self._conn.get_json("/ws/DeviceCore") - json_dump = devicecore_data["items"] - devices = [] - for device_json in json_dump: - devices.append(Device(self._conn, self._sci, device_json)) - return devices + def get_devices(self, condition=None, page_size=1000): + """Iterates over each :class:`Device` for this device cloud account + + Examples:: + + # get a list of all devices + all_devices = list(dc.devicecore.get_devices()) + + # build a mapping of devices by their vendor id using a + # dict comprehension + devices = dc.devicecore.get_devices() # generator object + devs_by_vendor_id = {d.get_vendor_id(): d for d in devices} + + # iterate over all devices in 'minnesota' group and + # print the device mac and location + for device in dc.get_devices(group_path == 'minnesota'): + print "%s at %s" % (device.get_mac(), device.get_location()) + + :param condition: An :class:`.Expression` which defines the condition + which must be matched on the devicecore. If unspecified, + an iterator over all devices will be returned. + :param int page_size: The number of results to fetch in a + single page. In general, the default will suffice. + :returns: Iterator over each :class:`~Device` in this device cloud + account in the form of a generator object. + """ + + condition = validate_type(condition, type(None), Expression, *six.string_types) + page_size = validate_type(page_size, *six.integer_types) + + params = {"embed": "true"} + if condition is not None: + params["condition"] = condition.compile() + + for device_json in self._conn.iter_json_pages("/ws/DeviceCore", page_size=page_size, **params): + yield Device(self._conn, self._sci, device_json) + + def get_group_tree_root(self, page_size=1000): + r"""Return the root group for this accounts' group tree + + This will return the root group for this tree but with all links + between nodes (i.e. children starting from root) populated. + + Examples:: + + # print the group hierarchy to stdout + dc.devicecore.get_group_tree_root().print_subtree() + + # gather statistics about devices in each group including + # the count from its subgroups (recursively) + # + # This also shows how you can go from a group reference to devices + # for that particular group. + stats = {} # group -> devices count including children + def count_nodes(group): + count_for_this_node = \ + len(list(dc.devicecore.get_devices(group_path == group.get_path()))) + subnode_count = 0 + for child in group.get_children(): + subnode_count += count_nodes(child) + total = count_for_this_node + subnode_count + stats[group] = total + return total + count_nodes(dc.devicecore.get_group_tree_root()) + + :param int page_size: The number of results to fetch in a + single page. In general, the default will suffice. + :returns: The root group for this device cloud accounts group + hierarchy. + + """ + + # first pass, build mapping + group_map = {} # map id -> group + page_size = validate_type(page_size, *six.integer_types) + for group in self.get_groups(page_size=page_size): + group_map[group.get_id()] = group + + # second pass, find root and populate list of children for each node + root = None + for group_id, group in group_map.items(): + if group.is_root(): + root = group + else: + parent = group_map[group.get_parent_id()] + parent.add_child(group) + return root + + def get_groups(self, condition=None, page_size=1000): + """Return an iterator over all groups in this device cloud account + + Optionally, a condition can be specified to limit the number of + groups returned. + + Examples:: + + # Get all groups and print information about them + for group in dc.devicecore.get_groups(): + print group + + # Iterate over all devices which are in a group with a specific + # ID. + group = dc.devicore.get_groups(group_id == 123)[0] + for device in dc.devicecore.get_devices(group_path == group.get_path()): + print device.get_mac() + + :param condition: A condition to use when filtering the results set. If + unspecified, all groups will be returned. + :param int page_size: The number of results to fetch in a + single page. In general, the default will suffice. + :returns: Generator over the groups in this device cloud account. No + guarantees about the order of results is provided and child links + between nodes will not be populated. + + """ + query_kwargs = {} + if condition is not None: + query_kwargs["condition"] = condition.compile() + for group_data in self._conn.iter_json_pages("/ws/Group", page_size=page_size, **query_kwargs): + yield Group.from_json(group_data) + + def delete_device(self, dev): + """ Delete a from the cloud account associated with the handle. + + :raises DeviceCloudHttpException: If there is an unexpected error reported by Device Cloud. + :param dev: Device object of the device to delete. + :return: the Response from the delete request. + """ + return self._conn.delete('/ws/DeviceCore/%s' % dev.get_device_id()) + + def provision_device(self, **kwargs): + """Provision a single device with the specified information + + This API call provisions a new device on this cloud account. In order for this + to work, a `mac_address`, `device_id`, or `imei` must be provided. All + other parameters are optional and can be specified in addition to the primary + identifier for the device. + + This request will always return a dictionary unless the request fails altogether. The + dictionary returned will have the following form:: + + { + "error": , + "error_msg": , + "location": , + } + + :param str mac_address: The MAC address of the device being added as a string in + the form "00:00:00:00:00:00". This is one of the options for the required + primary id. + :param str device_id: The ID of the device to add. This is the 'devConnectwareId' + referenced in the API docs and should look like "00000000-00000000-000000FF-FF000000". + This is one of the options for the required primary id. + :param str imei: The IMEI of the device to be added (if no MAC/ID available). This is one + of the options for the required primary id. + :param str group_path: (optional) Path of group that this device should be added to. + :param str metadata: (optional) Arbitrary metadata to associate with this device. + :param float map_lat: (optional) Latitude of this device in degrees + :param float map_long: (optional) Longitude of this device in degrees + :param str contact: (optional) Contact associted with this device (or whatever you want). + :param str description: (optional) Textual description of this device. + + :raises DeviceCloudHttpException: If there is an unexpected error reported by Device Cloud. + :raises ValueError: If any input fields are known to have a bad form. + :return: A dictionary matching the format specified above. + + """ + # This snippet is from Device Cloud API Explorer and shows the pieces of + # information that may be specified when adding a device. + # + # + # + # + # + # + # + # + # + # + # + # + # + # + # + + results = self.provision_devices([kwargs, ]) + return results[0] + + def provision_devices(self, devices): + """Provision multiple devices with a single API call + + This method takes an iterable of dictionaries where the values in the dictionary are + expected to match the arguments of a call to :meth:`provision_device`. The + contents of each dictionary will be validated. + + :param list devices: An iterable of dictionaries each containing information about + a device to be provision. The form of the dictionary should match the keyword + arguments taken by :meth:`provision_device`. + :raises DeviceCloudHttpException: If there is an unexpected error reported by Device Cloud. + :raises ValueError: If any input fields are known to have a bad form. + :return: A list of dictionaries in the form described for :meth:`provision_device` in the + order matching the requested device list. Note that it is possible for there to + be mixed success and error when provisioning multiple devices. + + """ + # Validate all the input for each device provided + sio = six.StringIO() + + def write_tag(tag, val): + sio.write("<{tag}>{val}".format(tag=tag, val=val)) + + def maybe_write_element(tag, val): + if val is not None: + write_tag(tag, val) + return True + return False + + sio.write("") + for d in devices: + sio.write("") + + mac_address = d.get("mac_address") + device_id = d.get("device_id") + imei = d.get("imei") + if mac_address is not None: + write_tag("devMac", mac_address) + elif device_id is not None: + write_tag("devConnectwareId", device_id) + elif imei is not None: + write_tag("devCellularModemId", imei) + else: + raise ValueError("mac_address, device_id, or imei must be provided for device %r" % d) + + # Write optional elements if present. + maybe_write_element("grpPath", d.get("group_path")) + maybe_write_element("dpUserMetaData", d.get("metadata")) + maybe_write_element("dpTags", d.get("tags")) + maybe_write_element("dpMapLong", d.get("map_long")) + maybe_write_element("dpMapLat", d.get("map_lat")) + maybe_write_element("dpContact", d.get("contact")) + maybe_write_element("dpDescription", d.get("description")) + + sio.write("") + sio.write("") + + # Send the request, set the Accept XML as a nicety + results = [] + response = self._conn.post("/ws/DeviceCore", sio.getvalue(), headers={'Accept': 'application/xml'}) + root = ET.fromstring(response.content) # tag is root of response + for child in root: + if child.tag.lower() == "location": + results.append({ + "error": False, + "error_msg": None, + "location": child.text + }) + else: # we expect "error" but handle generically + results.append({ + "error": True, + "location": None, + "error_msg": child.text + }) + + return results + + +class Group(object): + """Provides access to information about a group in Device Cloud + + .. note:: + + This is primarily a container object and does not provide any functions itself at + this time. Information from here can be used along with other APIs to, for example, + get all devices with a given group path. This may change in the future. + + """ + + def __init__(self, group_id, name, description, path, parent_id): + self._id = group_id + self._name = name + self._description = description + self._path = path + self._parent_id = parent_id + self._children = [] + + @classmethod + def from_json(cls, json_data): + """Build and return a new Group object from json data (used internally)""" + # Example Data: + # { "grpId": "11817", "grpName": "7603_Digi", "grpDescription": "7603_Digi root group", + # "grpPath": "\/7603_Digi\/", "grpParentId": "1"} + return cls( + group_id=json_data["grpId"], + name=json_data["grpName"], + description=json_data.get("grpDescription", ""), + path=json_data["grpPath"], + parent_id=json_data["grpParentId"], + ) + + def __repr__(self): + return "Group(group_id={!r}, name={!r}, description{!r}, path={!r}, parent_id={!r})".format( + self._id, self._name, self._description, self._path, self._parent_id + ) + + def print_subtree(self, fobj=sys.stdout, level=0): + """Print this group node and the subtree rooted at it""" + fobj.write("{}{!r}\n".format(" " * (level * 2), self)) + for child in self.get_children(): + child.print_subtree(fobj, level + 1) + + def is_root(self): + """Return True if the group is the root for this account""" + return self.get_parent_id() == "1" + + def add_child(self, group): + """Add a child group reference to this one""" + self._children.append(group) + + def get_children(self): + """Return each child :class:`Group` of this one in a list""" + return self._children[:] + + def get_id(self): + """Get the ID of this group as as string""" + return self._id + + def get_name(self): + """Get the name of this group as a string""" + return self._name + + def get_description(self): + """Get the description of this group as a string""" + return self._description + + def get_path(self): + """Get the full path of this group as a string""" + return self._path + + def get_parent_id(self): + """Get the ID of this groups parent as a string""" + return self._parent_id class Device(object): - """Interface to a device in the device cloud""" + """Interface to a device in Device Cloud""" # TODO: provide ability to set/update available data items - # TODO: add/remove tags - # TODO: add device to group - # TODO: remove device from a group - # TODO: provision a new device (probably add top-level method for this) def __init__(self, conn, sci, device_json): self._conn = conn @@ -52,7 +411,8 @@ def get_device_json(self, use_cached=True): """ if not use_cached: - devicecore_data = self._conn.get_json("/ws/DeviceCore/{}".format(self.get_device_id())) + devicecore_data = self._conn.get_json( + "/ws/DeviceCore/{}".format(self.get_device_id())) self._device_json = devicecore_data["items"][0] # should only be 1 return self._device_json @@ -61,7 +421,7 @@ def get_tags(self, use_cached=True): device_json = self.get_device_json(use_cached) potential_tags = device_json.get("dpTags") if potential_tags: - return potential_tags.split(",") + return list(filter(None, potential_tags.split(","))) else: return [] @@ -102,7 +462,7 @@ def get_mac_last4(self, use_cached=True): return mac4.upper() def get_registration_dt(self, use_cached=True): - """Get the datetime of when this device was added to the device cloud""" + """Get the datetime of when this device was added to Device Cloud""" device_json = self.get_device_json(use_cached) start_date_iso8601 = device_json.get("devRecordStartDate") if start_date_iso8601: @@ -164,7 +524,7 @@ def get_global_ip(self, use_cached=True): return self.get_device_json(use_cached).get("dpGlobalIp") def get_last_connected_dt(self, use_cached=True): - """Get the datetime that the device last connected to the device cloud""" + """Get the datetime that the device last connected to Device Cloud""" return iso8601_to_dt(self.get_device_json(use_cached).get("dpLastConnectTime")) def get_contact(self, use_cached=True): @@ -195,7 +555,7 @@ def get_zb_pan_id(self, use_cached=True): """Get the Zigbee PAN ID from the device if present""" return self.get_device_json(use_cached).get("dpPanId") - def get_zb_extended_address(self, use_cached=True): + def get_zb_extended_address(self, use_cached=True): """Get the Zigbee extended address of this device if present""" return self.get_device_json(use_cached).get("xpExtAddr") @@ -212,3 +572,75 @@ def get_provision_id(self, use_cached=True): def get_current_connect_pw(self, use_cached=True): """Get the current connection password for this device""" return self.get_device_json(use_cached).get("dpCurrentConnectPw") + + def add_to_group(self, group_path): + """Add a device to a group, if the group doesn't exist it is created + + :param group_path: Path or "name" of the group + """ + + if self.get_group_path() != group_path: + post_data = ADD_GROUP_TEMPLATE.format(connectware_id=self.get_connectware_id(), + group_path=group_path) + self._conn.put('/ws/DeviceCore', post_data) + + # Invalidate cache + self._device_json = None + + def remove_from_group(self): + """Place a device back into the root group""" + + if self.get_group_path() != '': + post_data = ADD_GROUP_TEMPLATE.format(connectware_id=self.get_connectware_id(), + group_path='') + self._conn.put('/ws/DeviceCore', post_data) + + # Invalidate cache + self._device_json = None + + def add_tag(self, new_tags): + """Add a tag to existing device tags. This method will not add a duplicate, if already in the list. + + :param new_tags: the tag(s) to be added. new_tags can be a comma-separated string or list + """ + + tags = self.get_tags() + orig_tag_cnt = len(tags) + # print("self.get_tags() {}".format(tags)) + + if isinstance(new_tags, six.string_types): + new_tags = new_tags.split(',') + # print("spliting tags :: {}".format(new_tags)) + + for tag in new_tags: + if not tag in tags: + tags.append(tag.strip()) + + if len(tags) > orig_tag_cnt: + xml_tags = escape(",".join(tags)) + post_data = TAGS_TEMPLATE.format(connectware_id=self.get_connectware_id(), + tags=xml_tags) + self._conn.put('/ws/DeviceCore', post_data) + + # Invalidate cache + self._device_json = None + # else: + # print("skipping tag update") + + def remove_tag(self, tag): + """Remove tag from existing device tags + + :param tag: the tag to be removed from the list + + :raises ValueError: If tag does not exist in list + """ + + tags = self.get_tags() + tags.remove(tag) + + post_data = TAGS_TEMPLATE.format(connectware_id=self.get_connectware_id(), + tags=escape(",".join(tags))) + self._conn.put('/ws/DeviceCore', post_data) + + # Invalidate cache + self._device_json = None diff --git a/devicecloud/examples/__init__.py b/devicecloud/examples/__init__.py index 5cb2679..afdd51d 100644 --- a/devicecloud/examples/__init__.py +++ b/devicecloud/examples/__init__.py @@ -2,5 +2,4 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015-2018 Digi International Inc. diff --git a/devicecloud/examples/cookbook_streams.py b/devicecloud/examples/cookbook_streams.py index 2708d92..303813a 100644 --- a/devicecloud/examples/cookbook_streams.py +++ b/devicecloud/examples/cookbook_streams.py @@ -1,23 +1,12 @@ -from devicecloud.streams import STREAM_TYPE_STRING, DataPoint, STREAM_TYPE_INTEGER -from devicecloud import DeviceCloud -from getpass import getpass import datetime import pprint import random import time import json +from devicecloud.examples.example_helpers import get_authenticated_dc -def get_authenticated_dc(): - while True: - user = raw_input("username: ") - password = getpass("password: ") - dc = DeviceCloud(user, password) - if dc.has_valid_credentials(): - print ("Credentials accepted!") - return dc - else: - print ("Invalid username or password provided, try again") +from devicecloud.streams import STREAM_TYPE_STRING, DataPoint, STREAM_TYPE_INTEGER def get_or_create_classroom(datatype): @@ -76,7 +65,7 @@ def example_1(): classroom.bulk_write_datapoints(datapoints) most_recent_dp = classroom.get_current_value() - print json.loads(most_recent_dp.get_data())['name'] + print(json.loads(most_recent_dp.get_data())['name']) def example_2(): @@ -91,4 +80,4 @@ def example_2(): example_2() -print 'done.' \ No newline at end of file +print('done.') diff --git a/devicecloud/examples/devicecore_playground.py b/devicecloud/examples/devicecore_playground.py new file mode 100644 index 0000000..cdee37b --- /dev/null +++ b/devicecloud/examples/devicecore_playground.py @@ -0,0 +1,36 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. +from devicecloud.devicecore import dev_mac, group_path +from devicecloud.examples.example_helpers import get_authenticated_dc + + +def show_group_tree(dc): + stats = {} # group -> devices count including children + + def count_nodes(group): + count_for_this_node = \ + len(list(dc.devicecore.get_devices(group_path == group.get_path()))) + subnode_count = 0 + for child in group.get_children(): + subnode_count += count_nodes(child) + total = count_for_this_node + subnode_count + stats[group] = total + return total + + count_nodes(dc.devicecore.get_group_tree_root()) + print(stats) + dc.devicecore.get_group_tree_root().print_subtree() + + +if __name__ == '__main__': + dc = get_authenticated_dc() + devices = dc.devicecore.get_devices( + (dev_mac == '00:40:9D:50:B0:EA') + ) + for dev in devices: + print(dev) + + show_group_tree(dc) diff --git a/devicecloud/examples/example_helpers.py b/devicecloud/examples/example_helpers.py new file mode 100644 index 0000000..011c57b --- /dev/null +++ b/devicecloud/examples/example_helpers.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. +from getpass import getpass +import os +from six.moves import input +from devicecloud import DeviceCloud + + +def get_authenticated_dc(): + while True: + base_url = os.environ.get('DC_BASE_URL', 'https://devicecloud.digi.com') + + username = os.environ.get('DC_USERNAME', None) + if not username: + username = input("username: ") + + password = os.environ.get('DC_PASSWORD', None) + if not password: + password = getpass("password: ") + + dc = DeviceCloud(username, password, base_url=base_url) + if dc.has_valid_credentials(): + print("Credentials accepted!") + return dc + else: + print("Invalid username or password provided, try again") diff --git a/devicecloud/examples/file_system_service_playground.py b/devicecloud/examples/file_system_service_playground.py new file mode 100644 index 0000000..51aa1b3 --- /dev/null +++ b/devicecloud/examples/file_system_service_playground.py @@ -0,0 +1,114 @@ +import time +import six +from devicecloud.examples.example_helpers import get_authenticated_dc +from devicecloud.file_system_service import ErrorInfo, FileSystemServiceCommandBlock, LsCommand, PutCommand +from devicecloud.sci import DeviceTarget + + +def put_test_file(fssapi, target, tmp_file_path): + tmp_str = six.b('testing string') + + print("\nWriting test file {}".format(tmp_file_path)) + print(fssapi.put_file(target, tmp_file_path, file_data=tmp_str)) + + +def list_contents(fssapi, target, dev_dir): + print("\nList of files in {}".format(dev_dir)) + out_dict = fssapi.list_files(target, dev_dir) + print("list_files returned: {}".format(str(out_dict))) + print_list_contents(out_dict) + + +def print_list_contents(out_dict): + print("\nPrint each item from list_files:") + for device_id, device_data in six.iteritems(out_dict): + print("Items from device {}".format(device_id)) + if isinstance(device_data, ErrorInfo): + print(" ErrorInfo: {}".format(device_data)) + else: + (dirs, files) = device_data + if len(dirs) + len(files) == 0: + print " None" + for d in dirs: + print(" Directory: {}".format(str(d))) + for f in files: + print(" File: {}".format(str(f))) + + +def delete_test_file(fssapi, target, tmp_file_path): + print("\nDeleting test file: {}".format(tmp_file_path)) + print(fssapi.delete_file(target, tmp_file_path)) + + +def get_modified_files(fssapi, target, dev_dir, last_modified_cutoff): + print("\nGetting all files modified since {}".format(last_modified_cutoff)) + out_dict = fssapi.get_modified_items(target, dev_dir, last_modified_cutoff) + print_list_contents(out_dict) + + +def use_filesystem(dc, target, base_dir): + fssapi = dc.get_fss_api() + fd = dc.get_filedata_api() + + tmp_file = '{}/test.txt'.format(base_dir) + + put_test_file(fssapi, target, tmp_file) + + tmp_server_file = 'test_file.txt' + print("\nWriting temp file to server {}".format(tmp_server_file)) + fd.write_file("/~/test_dir/", "test_file.txt", six.b("Hello, world!"), "text/plain") + + tmp_server_device_file = '{}/{}'.format(base_dir, tmp_server_file) + print("\nWriting temp file from server {}".format(tmp_server_device_file)) + fssapi.put_file(target, tmp_server_device_file, server_file='/~/test_dir/{}'.format(tmp_server_file)) + + list_contents(fssapi, target, base_dir) + + print("\nUsing API to get file data") + print(fssapi.get_file(target, tmp_file)) + + print("\nUsing API to get other file data") + print(fssapi.get_file(target, tmp_server_device_file)) + + print("\nUsing API to get partial file data") + print(fssapi.get_file(target, tmp_file, offset=3, length=4)) + + print("\nUsing API to write part of a file") + fssapi.put_file(target, tmp_file, file_data=six.b("what"), offset=4) + print(fssapi.get_file(target, tmp_file)) + + print("\nUsing API to write part of a file and truncating") + fssapi.put_file(target, tmp_file, file_data=six.b("why"), offset=4, truncate=True) + print(fssapi.get_file(target, tmp_file)) + + delete_test_file(fssapi, target, tmp_file) + delete_test_file(fssapi, target, tmp_server_device_file) + + print("\nList of files in {}".format(base_dir)) + out_dict = fssapi.list_files(target, base_dir) + print(out_dict) + + +if __name__ == "__main__": + dc = get_authenticated_dc() + device_id = "your-device-id-here" + target = DeviceTarget(device_id) + base_dir = '/a/directory/on/your/device' + use_filesystem(dc, target, base_dir) + + fssapi = dc.get_fss_api() + tmp_file_path = "{}/{}".format(base_dir, 'test_file.txt') + put_test_file(fssapi, target, tmp_file_path) + cutoff_time = time.time() + get_modified_files(fssapi, target, base_dir, cutoff_time) + print("\nModifying file {}".format(tmp_file_path)) + fssapi.put_file(target, tmp_file_path, file_data=six.b("data"), offset=4) + time.sleep(5) + get_modified_files(fssapi, target, base_dir, cutoff_time) + delete_test_file(fssapi, target, tmp_file_path) + + command_block = FileSystemServiceCommandBlock() + command_block.add_command(LsCommand(base_dir)) + command_block.add_command(LsCommand('/another/directory/on/your/device')) + + print(fssapi.send_command_block(target, command_block)) diff --git a/devicecloud/examples/filedata_playground.py b/devicecloud/examples/filedata_playground.py index 5f333e5..3533a25 100644 --- a/devicecloud/examples/filedata_playground.py +++ b/devicecloud/examples/filedata_playground.py @@ -2,40 +2,21 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. -from getpass import getpass +# Copyright (c) 2015-2018 Digi International Inc. -from devicecloud import DeviceCloud -from devicecloud.filedata import fd_name, fd_size, fd_type +from devicecloud.examples.example_helpers import get_authenticated_dc +from devicecloud.filedata import fd_path import six - -def get_authenticated_dc(): - while True: - user = raw_input("username: ") - password = getpass("password: ") - dc = DeviceCloud(user, password, base_url="https://test-idigi-com-2v5p9uat81qu.runscope.net") - if dc.has_valid_credentials(): - print ("Credentials accepted!") - return dc - else: - print ("Invalid username or password provided, try again") - - if __name__ == '__main__': dc = get_authenticated_dc() dc.filedata.write_file("/~/test_dir/", "test_file.txt", six.b("Helllo, world!"), "text/plain") dc.filedata.write_file("/~/test_dir/", "test_file2.txt", six.b("Hello, again!")) - query = None # (fd_path == '/db/public/') - for dirpath, directories, files in dc.filedata.walk(): + for dirpath, directories, files in dc.filedata.walk("/"): for fd_file in files: - print fd_file + print(fd_file) - for fd in dc.filedata.get_filedata( - (fd_type == "file") & - (fd_size < 250) & - (fd_name.like('%.txt')), page_size=1): + for fd in dc.filedata.get_filedata(fd_path == "~/"): print (fd) diff --git a/devicecloud/examples/monitor_playground.py b/devicecloud/examples/monitor_playground.py new file mode 100644 index 0000000..eeadba5 --- /dev/null +++ b/devicecloud/examples/monitor_playground.py @@ -0,0 +1,65 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (C) 2015, Digi International, Inc.. +import pprint +import random +import time + +from devicecloud.examples.example_helpers import get_authenticated_dc + +from devicecloud.streams import DataPoint + +def test_tcp_monitor(dc): + # Create a fresh monitor over a pretty broad set of topics + topics = ['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint'] + mon = dc.monitor.get_monitor(topics) + if mon is not None: + mon.delete() + mon = dc.monitor.create_tcp_monitor(topics) + pprint.pprint(mon.get_metadata()) + + def listener(data): + pprint.pprint(data) + return True # we got it! + + mon.add_callback(listener) + + test_stream = dc.streams.get_stream("test") + try: + while True: + test_stream.write(DataPoint(random.random())) + time.sleep(3.14) + except KeyboardInterrupt: + print("Shutting down threads...") + + dc.monitor.stop_listeners() + +def test_http_monitor(dc): + # Create a fresh monitor over a pretty broad set of topics + topics = ['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint'] + mon = dc.monitor.get_monitor(topics) + if mon is not None: + mon.delete() + mon = dc.monitor.create_http_monitor(topics, 'http://digi.com', transport_token=None, transport_method='PUT', + connect_timeout=0, response_timeout=0, batch_size=1, batch_duration=0, + compression='none', format_type='json') + pprint.pprint(mon.get_metadata()) + + def listener(data): + pprint.pprint(data) + return True # we got it! + + test_stream = dc.streams.get_stream("test") + try: + while True: + test_stream.write(DataPoint(random.random())) + time.sleep(3.14) + except KeyboardInterrupt: + print("Shutting down threads...") + +if __name__ == '__main__': + dc = get_authenticated_dc() + test_http_monitor(dc) + diff --git a/devicecloud/examples/streams_playground.py b/devicecloud/examples/streams_playground.py index c479170..15f2678 100644 --- a/devicecloud/examples/streams_playground.py +++ b/devicecloud/examples/streams_playground.py @@ -2,27 +2,14 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. -from getpass import getpass +# Copyright (c) 2015-2018 Digi International Inc. from math import pi import pprint import time -from devicecloud import DeviceCloud -from devicecloud.streams import DataPoint, NoSuchStreamException, STREAM_TYPE_INTEGER +from devicecloud.examples.example_helpers import get_authenticated_dc - -def get_authenticated_dc(): - while True: - user = raw_input("username: ") - password = getpass("password: ") - dc = DeviceCloud(user, password, base_url="https://login-etherios-com-2v5p9uat81qu.runscope.net") - if dc.has_valid_credentials(): - print ("Credentials accepted!") - return dc - else: - print ("Invalid username or password provided, try again") +from devicecloud.streams import DataPoint, NoSuchStreamException, STREAM_TYPE_INTEGER, STREAM_TYPE_JSON def create_stream_and_delete(dc): @@ -156,8 +143,39 @@ def bulk_write_datapoints_multiple_streams(dc): stream.delete() +def create_and_use_json_stream(dc): + # get a test stream reference + test_stream = dc.streams.get_stream_if_exists("test-json") + + # we want a clean stream to work with. If the stream exists, nuke it + if test_stream is not None: + test_stream.delete() + + test_stream = dc.streams.create_stream( + stream_id="test-json", + data_type=STREAM_TYPE_JSON, + description='a stream used for testing json', + units='international json standard unit (IJSU)', + ) + + test_stream.write(DataPoint( + data_type=STREAM_TYPE_JSON, + data = {'key1': 'value1', + 2: 2, + 'key3': [1, 2, 3]}, + description="Some JSON data in IJSUs", + ) + ) + + time.sleep(5) + + print(test_stream.get_current_value()) + + test_stream.delete() + if __name__ == '__main__': dc = get_authenticated_dc() + create_and_use_json_stream(dc) create_stream_and_delete(dc) attempt_to_delete_non_existant(dc) write_points_and_delete_some(dc) diff --git a/devicecloud/file_system_service.py b/devicecloud/file_system_service.py new file mode 100644 index 0000000..5791c91 --- /dev/null +++ b/devicecloud/file_system_service.py @@ -0,0 +1,805 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. + +"""Provide access to Device Cloud file system service API""" +import base64 +from collections import namedtuple +import xml.etree.ElementTree as ET + +from devicecloud.sci import DeviceTarget +import six +from devicecloud.apibase import SCIAPIBase + + +class FileSystemServiceException(Exception): + """A file system service exception""" + + +class ResponseParseError(FileSystemServiceException): + """An Exception when receiving an unexpected SCI response format""" + + +def _parse_command_response(response): + """Parse an SCI command response into ElementTree XML + + This is a helper method that takes a Requests Response object + of an SCI command response and will parse it into an ElementTree Element + representing the root of the XML response. + + :param response: The requests response object + :return: An ElementTree Element that is the root of the response XML + :raises ResponseParseError: If the response XML is not well formed + """ + try: + root = ET.fromstring(response.text) + except ET.ParseError: + raise ResponseParseError( + "Unexpected response format, could not parse XML. Response: {}".format(response.text)) + + return root + + +def _parse_error_tree(error): + """Parse an error ElementTree Node to create an ErrorInfo object + + :param error: The ElementTree error node + :return: An ErrorInfo object containing the error ID and the message. + """ + errinf = ErrorInfo(error.get('id'), None) + if error.text is not None: + errinf.message = error.text + else: + desc = error.find('./desc') + if desc is not None: + errinf.message = desc.text + return errinf + + +class ErrorInfo(object): + """Represents an error response from the device or the devicecloud + + :ivar errno: The error number reported in the response + :ivar message: The error message reported in the response + """ + + def __init__(self, errno, message): + self.errno = int(errno) + self.message = message + + def __str__(self): + return "".format(errno=self.errno, message=self.message) + + +LsInfo = namedtuple('LsInfo', ['directories', 'files']) + + +class FileInfo(object): + """Represents a file from a device + + This stores the information about a file on the device returned from an ls command. + It also provides functionality to get the contents of the represented file and delete + it. However, writing to the file is not supported as it would invalidate the other + information stored in this object. + + + :param fssapi: A :class:`~.FileSystemServiceAPI` object to perform device file operations with + :type fssapi: :class:`~.FileSystemServiceAPI` + :param device_id: The Device ID of the device this file is on + :param path: The path to this file on the device + :param last_modified: The last time the file was modified + :type last_modified: int + :param size: The size of the file + :type size: int + :param hash: The files hash + :param hash_type: The method used to produce the hash + + :ivar device_id: The Device ID of the device this file is on + :ivar path: The path to this file on the device + :ivar last_modified: The last time the file was modified + :ivar size: The size of the file + :ivar hash: The files hash + :ivar hash_type: The method used to produce the hash + """ + + def __init__(self, fssapi, device_id, path, last_modified, size, hash, hash_type): + self._fssapi = fssapi + self.device_id = device_id + self.path = path + self.last_modified = last_modified + self.size = size + self.hash = hash + self.hash_type = hash_type + + def get_data(self): + """Get the contents of this file + + :return: The contents of this file + :rtype: six.binary_type + """ + + target = DeviceTarget(self.device_id) + return self._fssapi.get_file(target, self.path)[self.device_id] + + def delete(self): + """Delete this file from the device + + .. note:: + After deleting the file, this object will no longer contain valid information + and further calls to delete or get_data will return :class:`~.ErrorInfo` objects + """ + target = DeviceTarget(self.device_id) + return self._fssapi.delete_file(target, self.path)[self.device_id] + + def __str__(self): + return "".format( + device=self.device_id, + path=self.path + ) + + def __eq__(self, other): + return (self.device_id == other.device_id and + self.path == other.path and + self.last_modified == other.last_modified and + self.size == other.size and + self.hash == other.hash) + + +class DirectoryInfo(object): + """Represents a directory from a device + + This stores the information about a directory on the device returned from an ls command. + It also provides functionality to list the contents of the directory. + + :param fssapi: A :class:`~.FileSystemServiceAPI` object to perform device directory operations with + :type fssapi: :class:`~.FileSystemServiceAPI` + :param device_id: The Device ID of the device this file is on + :param path: The path to this file on the device + :param last_modified: The last time the file was modified + :type last_modified: int + + :ivar device_id: The Device ID of the device this file is on + :ivar path: The path to this file on the device + :ivar last_modified: the last time the file was modified + """ + + def __init__(self, fssapi, device_id, path, last_modified): + self._fssapi = fssapi + self.device_id = device_id + self.path = path + self.last_modified = last_modified + + def list_contents(self): + """List the contents of this directory + + :return: A LsInfo object that contains directories and files + :rtype: :class:`~.LsInfo` or :class:`~.ErrorInfo` + + Here is an example usage:: + + # let dirinfo be a DirectoryInfo object + ldata = dirinfo.list_contents() + if isinstance(ldata, ErrorInfo): + # Do some error handling + logger.warn("Error listing file info: (%s) %s", ldata.errno, ldata.message) + # It's of type LsInfo + else: + # Look at all the files + for finfo in ldata.files: + logger.info("Found file %s of size %s", finfo.path, finfo.size) + # Look at all the directories + for dinfo in ldata.directories: + logger.info("Found directory %s of last modified %s", dinfo.path, dinfo.last_modified) + """ + target = DeviceTarget(self.device_id) + return self._fssapi.list_files(target, self.path)[self.device_id] + + def __str__(self): + return "".format( + device=self.device_id, + path=self.path, + ) + + def __eq__(self, other): + return (self.device_id == other.device_id and + self.path == other.path and + self.last_modified == other.last_modified) + + +class FileSystemServiceCommandBlock(object): + + def __init__(self): + self.commands_node = ET.Element('commands') + + def add_command(self, command): + """Add a :class:`~.FileSystemServiceCommandABC` to this command block + + :param command: The command to include in this command block + :type command: :class:`~.FileSystemServiceCommandABC` + """ + self.commands_node.append(command.get_etree()) + + def get_command_string(self): + """Return this command block represented as a string + + :return: The string representation of this command block + """ + return ET.tostring(self.commands_node) + + def get_etree(self): + """Return the :class:`xml.etree.ElementTree.Element` representing the XML of this command block + + :return: The command block + :rtype: :class:`xml.etree.ElementTree.Element` + """ + return self.commands_node + + +class FileSystemServiceCommandABC(object): + """Abstract base class for FileSystemServiceCommands + + :cvar command_name: The file_system command name of this command + """ + + command_name = 'abstract' + + def get_etree(self): + """Return an :class:`xml.etree.ElementTree.Element` that is the root of the command XML for this command""" + raise NotImplementedError() + + @classmethod + def parse_response(cls, response, **kwargs): + """Parse the server response for this command type""" + raise NotImplementedError + + +class LsCommand(FileSystemServiceCommandABC): + """Class representing the ls command to list files on a device + + :param path: the path to the device directory to list + :param hash: the method to generate the file hashes + + :cvar command_name: The file_system command name of this command ("ls") + """ + + command_name = "ls" + + def __init__(self, path, hash='any'): + self.etree = ET.Element(self.command_name) + self.etree.set('path', path) + self.etree.set('hash', hash) + + def get_etree(self): + """Get the XML representation of this command + + :return: the :class:`xml.etree.ElementTree.Element` that is the root of the XML of the ls command + """ + return self.etree + + @classmethod + def parse_response(cls, response, device_id=None, fssapi=None, **kwargs): + """Parse the server response for this ls command + + This will parse xml of the following form:: + + + + ... + + ... + + + or with an error:: + + + + + + :param response: The XML root of the response for an ls command + :type response: :class:`xml.etree.ElementTree.Element` + :param device_id: The device id of the device this ls response came from + :param fssapi: A reference to a :class:`~FileSystemServiceAPI` for use with the + :class:`~FileInfo` and :class:`~DirectoryInfo` objects for future commands + :return: An :class:`~LsInfo` object containing the list of directories and files on + the device or an :class:`~ErrorInfo` if the xml contained an error + """ + if response.tag != cls.command_name: + raise ResponseParseError( + "Received response of type {}, LsCommand can only parse responses of type {}".format(response.tag, + cls.command_name)) + + if fssapi is None: + raise FileSystemServiceException("fssapi is required to parse an LsCommand response") + if device_id is None: + raise FileSystemServiceException("device_id is required to parse an LsCommand response") + + error = response.find('./error') + if error is not None: + return _parse_error_tree(error) + + hash_type = response.get('hash') + dirs = [] + files = [] + + # Get each file listed in this response + for myfile in response.findall('./file'): + fi = FileInfo(fssapi, + device_id, + myfile.get('path'), + int(myfile.get('last_modified')), + int(myfile.get('size')), + myfile.get('hash'), + hash_type) + files.append(fi) + # Get each directory listed for this device + for mydir in response.findall('./dir'): + di = DirectoryInfo(fssapi, + device_id, + mydir.get('path'), + int(mydir.get('last_modified'))) + dirs.append(di) + return LsInfo(directories=dirs, files=files) + + +class GetCommand(FileSystemServiceCommandABC): + """Class representing the get file command to get the contents of a file on a device + + :param path: the path to the file on the device to get the contents of + :param offset: the point in the file to start reading data from (if None, start at the beginning) + :param length: the length of data to retrieve (if None, read until the end) + + :cvar command_name: The file_system command name of this command ("get_file") + """ + command_name = 'get_file' + + def __init__(self, path, offset=None, length=None): + get_file_el = ET.Element(self.command_name) + get_file_el.set('path', path) + if offset is not None: + get_file_el.set('offset', str(offset)) + if length is not None: + get_file_el.set('length', str(length)) + + self.etree = get_file_el + + def get_etree(self): + """Get the XML representation of this get file command + + :return: the :class:`xml.etree.ElementTree.Element` that is the root of the XML of the get_file command + """ + return self.etree + + @classmethod + def parse_response(cls, response, **kwargs): + """Parse the server response for this get file command + + This will parse xml of the following form:: + + + + asdfasdfasdfasdfasf + + + + or with an error:: + + + + + + :param response: The XML root of the response for a get file command + :type response: :class:`xml.etree.ElementTree.Element` + :return: a six.binary_type string of the data of a file or an :class:`~ErrorInfo` if the xml contained an error + """ + if response.tag != cls.command_name: + raise ResponseParseError( + "Received response of type {}, GetCommand can only parse responses of type {}".format(response.tag, + cls.command_name)) + + error = response.find('./error') + if error is not None: + return _parse_error_tree(error) + + text = response.find('./data').text + if text: + return base64.b64decode(six.b(text)) + else: + return six.b('') + + +class PutCommand(FileSystemServiceCommandABC): + """Class representing the put file command to write contents to a file on a device + + :param path: The path on the target to the file to write to. If the file already exists it will be overwritten. + :param file_data: A `six.binary_type` containing the data to put into the file + :param server_file: The path to a file on the devicecloud server containing the data to put into the file on the + device + :param offset: Start writing bytes to the file at this position, if None start at the beginning + :param truncate: Boolean, if True after bytes are done being written end the file their even if previous data + exists beyond it. If False, leave any existing data in place. + + :cvar command_name: The file_system command name of this command ("put_file") + """ + command_name = 'put_file' + + def __init__(self, path, file_data=None, server_file=None, offset=None, truncate=False): + if file_data is not None and server_file is not None: + raise FileSystemServiceException("Can only specify one of server_file or file_data") + + put_file_el = ET.Element(self.command_name) + put_file_el.set('path', path) + put_file_el.set('truncate', 'true' if truncate else 'false') + + if offset is not None: + put_file_el.set('offset', str(offset)) + + if file_data is not None: + if not isinstance(file_data, six.binary_type): + raise TypeError("file_data must be of type {}".format(six.binary_type)) + data_el = ET.SubElement(put_file_el, 'data') + data_el.text = base64.b64encode(file_data).decode('ascii') + elif server_file is not None: + file_el = ET.SubElement(put_file_el, 'file') + file_el.text = server_file + else: + raise FileSystemServiceException("You must specify either file_data or server_file to put data into a file") + + self.etree = put_file_el + + def get_etree(self): + """Get the XML representation of this put file command + + :return: the :class:`xml.etree.ElementTree.Element` that is the root of the XML of the put file command + """ + return self.etree + + @classmethod + def parse_response(cls, response, **kwargs): + """Parse the server response for this put file command + + This will parse xml of the following form:: + + + + or with an error:: + + + + + + :param response: The XML root of the response for a put file command + :type response: :class:`xml.etree.ElementTree.Element` + :return: None if everything was ok or an :class:`~ErrorInfo` if the xml contained an error + """ + if response.tag != cls.command_name: + raise ResponseParseError( + "Received response of type {}, PutCommand can only parse responses of type {}".format(response.tag, + cls.command_name)) + error = response.find('./error') + if error is not None: + return _parse_error_tree(error) + + return None + + +class DeleteCommand(FileSystemServiceCommandABC): + """Class representing the delete file command on a device + + :param path: The path on the target to the file to delete. + + :cvar command_name: The file_system command name of this command ("rm") + """ + command_name = 'rm' + + def __init__(self, path): + rm_el = ET.Element(self.command_name) + rm_el.set('path', path) + self.etree = rm_el + + def get_etree(self): + """Get the XML representation of this delete file command + + :return: the :class:`xml.etree.ElementTree.Element` that is the root of the XML of the delete file command + """ + return self.etree + + @classmethod + def parse_response(cls, response, **kwargs): + """Parse the server response for this put file command + + This will parse xml of the following form:: + + + + or with an error:: + + + + + + :param response: The XML root of the response for a delete file command + :type response: :class:`xml.etree.ElementTree.Element` + :return: None if everything was ok or an :class:`~ErrorInfo` if the xml contained an error + """ + if response.tag != cls.command_name: + raise ResponseParseError( + "Received response of type {}, DeleteCommand can only parse responses of type {}".format(response.tag, + cls.command_name)) + error = response.find('./error') + if error is not None: + return _parse_error_tree(error) + return None + + +FILE_SYSTEM_COMMANDS = [LsCommand, GetCommand, PutCommand, DeleteCommand] + + +class FileSystemServiceAPI(SCIAPIBase): + """ Encapsulate the File System Service API """ + + def send_command_block(self, target, command_block): + """Send an arbitrary file system command block + + The primary use for this method is to send multiple file system commands with a single + web service request. This can help to avoid throttling. + + :param target: The device(s) to be targeted with this request + :type target: :class:`devicecloud.sci.TargetABC` or list of :class:`devicecloud.sci.TargetABC` instances + :param command_block: The block of commands to execute on the target + :type command_block: :class:`~FileSystemServiceCommandBlock` + :return: The response will be a dictionary where the keys are device_ids and the values are + the parsed responses of each command sent in the order listed in the command response for + that device. In practice it seems to be the same order as the commands were sent in, however, + Device Cloud documentation does not explicitly state anywhere that is the case so I cannot + guarantee it. This does mean that if you send different types of commands the response list + will be different types. Please see the commands parse_response functions for what those types + will be. (:meth:`LsCommand.parse_response`, :class:`GetCommand.parse_response`, + :class:`PutCommand.parse_response`, :class:`DeleteCommand.parse_response`) + """ + root = _parse_command_response( + self._sci_api.send_sci("file_system", target, command_block.get_command_string())) + + out_dict = {} + for device in root.findall('./file_system/device'): + device_id = device.get('id') + results = [] + for command in device.find('./commands'): + for command_class in FILE_SYSTEM_COMMANDS: + if command_class.command_name == command.tag.lower(): + results.append(command_class.parse_response(command, fssapi=self, device_id=device_id)) + out_dict[device_id] = results + return out_dict + + def list_files(self, target, path, hash='any'): + """List all files and directories in the path on the target + + :param target: The device(s) to be targeted with this request + :type target: :class:`devicecloud.sci.TargetABC` or list of :class:`devicecloud.sci.TargetABC` instances + :param path: The path on the target to list files and directories from + :param hash: an optional attribute which indicates a hash over the file contents should be retrieved. Values + include none, any, md5, and crc32. any is used to indicate the device should choose its best available hash. + :return: A dictionary with keys of device ids and values of :class:`~.LsInfo` objects containing the files and + directories or an :class:`~.ErrorInfo` object if there was an error response + :raises: :class:`~.ResponseParseError` If the SCI response has unrecognized formatting + + Here is an example usage:: + + # dc is a DeviceCloud instance + fssapi = dc.get_fss_api() + + target = AllTarget() + ls_dir = '/root/home/user/important_files/' + + ls_data = fssapi.list_files(target, ls_dir) + + # Loop over all device results + for device_id, device_data in ls_data.iteritems(): + # Check if it succeeded or was an error + if isinstance(device_data, ErrorInfo): + # Do some error handling + logger.warn("Error listing file info on device %s. errno: %s message:%s", + device_id, device_data.errno, device_data.message) + + # It's of type LsInfo + else: + # Look at all the files + for finfo in device_data.files: + logger.info("Found file %s of size %s on device %s", + finfo.path, finfo.size, device_id) + # Look at all the directories + for dinfo in device_data.directories: + logger.info("Found directory %s of last modified %s on device %s", + dinfo.path, dinfo.last_modified, device_id) + + """ + command_block = FileSystemServiceCommandBlock() + command_block.add_command(LsCommand(path, hash=hash)) + root = _parse_command_response( + self._sci_api.send_sci("file_system", target, command_block.get_command_string())) + + out_dict = {} + + # At this point the XML we have is of the form + # + # + # + # + # + # + # ... + # + # ... + # + # + # + # + # + # + # + # ... + # + # ... + # + # + # + # ... + # + # + + # Here we will get each of the XML trees rooted at the device nodes + for device in root.findall('./file_system/device'): + device_id = device.get('id') + error = device.find('./error') + if error is not None: + out_dict[device_id] = _parse_error_tree(error) + else: + linfo = LsCommand.parse_response(device.find('./commands/ls'), device_id=device_id, fssapi=self) + out_dict[device_id] = linfo + return out_dict + + def get_file(self, target, path, offset=None, length=None): + """Get the contents of a file on the device + + :param target: The device(s) to be targeted with this request + :type target: :class:`devicecloud.sci.TargetABC` or list of :class:`devicecloud.sci.TargetABC` instances + :param path: The path on the target to the file to retrieve + :param offset: Start retrieving data from this byte position in the file, if None start from the beginning + :param length: How many bytes to retrieve, if None retrieve until the end of the file + :return: A dictionary with keys of device ids and values of the bytes of the file (or partial file if offset + and/or length are specified) or an :class:`~.ErrorInfo` object if there was an error response + :raises: :class:`~.ResponseParseError` If the SCI response has unrecognized formatting + """ + command_block = FileSystemServiceCommandBlock() + command_block.add_command(GetCommand(path, offset, length)) + root = _parse_command_response( + self._sci_api.send_sci("file_system", target, command_block.get_command_string())) + out_dict = {} + for device in root.findall('./file_system/device'): + device_id = device.get('id') + error = device.find('./error') + if error is not None: + out_dict[device_id] = _parse_error_tree(error) + else: + data = GetCommand.parse_response(device.find('./commands/get_file')) + out_dict[device_id] = data + return out_dict + + def put_file(self, target, path, file_data=None, server_file=None, offset=None, truncate=False): + """Put data into a file on the device + + :param target: The device(s) to be targeted with this request + :type target: :class:`devicecloud.sci.TargetABC` or list of :class:`devicecloud.sci.TargetABC` instances + :param path: The path on the target to the file to write to. If the file already exists it will be overwritten. + :param file_data: A `six.binary_type` containing the data to put into the file + :param server_file: The path to a file on the devicecloud server containing the data to put into the file on the + device + :param offset: Start writing bytes to the file at this position, if None start at the beginning + :param truncate: Boolean, if True after bytes are done being written end the file their even if previous data + exists beyond it. If False, leave any existing data in place. + :return: A dictionary with keys being device ids and value being None if successful or an :class:`~.ErrorInfo` + if the operation failed on that device + :raises: :class:`~.FileSystemServiceException` if either both file_data and server_file are specified or + neither are specified + :raises: :class:`~.ResponseParseError` If the SCI response has unrecognized formatting + """ + + command_block = FileSystemServiceCommandBlock() + command_block.add_command(PutCommand(path, file_data, server_file, offset, truncate)) + + root = _parse_command_response(self._sci_api.send_sci("file_system", target, command_block.get_command_string())) + out_dict = {} + for device in root.findall('./file_system/device'): + device_id = device.get('id') + error = device.find('./error') + if error is not None: + out_dict[device_id] = _parse_error_tree(error) + else: + out_dict[device_id] = PutCommand.parse_response(device.find('./commands/put_file')) + + return out_dict + + def delete_file(self, target, path): + """Delete a file from a device + + :param target: The device(s) to be targeted with this request + :type target: :class:`devicecloud.sci.TargetABC` or list of :class:`devicecloud.sci.TargetABC` instances + :param path: The path on the target to the file to delete. + :return: A dictionary with keys being device ids and value being None if successful or an :class:`~.ErrorInfo` + if the operation failed on that device + :raises: :class:`~.ResponseParseError` If the SCI response has unrecognized formatting + """ + command_block = FileSystemServiceCommandBlock() + command_block.add_command(DeleteCommand(path)) + root = _parse_command_response(self._sci_api.send_sci("file_system", target, command_block.get_command_string())) + + out_dict = {} + for device in root.findall('./file_system/device'): + device_id = device.get('id') + error = device.find('./error') + if error is not None: + out_dict[device_id] = _parse_error_tree(error) + else: + out_dict[device_id] = DeleteCommand.parse_response(device.find('./commands/rm')) + return out_dict + + def get_modified_items(self, target, path, last_modified_cutoff): + """Get all files and directories from a path on the device modified since a given time + + :param target: The device(s) to be targeted with this request + :type target: :class:`devicecloud.sci.TargetABC` or list of :class:`devicecloud.sci.TargetABC` instances + :param path: The path on the target to the directory to check for modified files. + :param last_modified_cutoff: The time (as Unix epoch time) to get files modified since + :type last_modified_cutoff: int + :return: A dictionary where the key is a device id and the value is either an :class:`~.ErrorInfo` if there + was a problem with the operation or a :class:`~.LsInfo` with the items modified since the + specified date + """ + file_list = self.list_files(target, path) + out_dict = {} + for device_id, device_data in six.iteritems(file_list): + if isinstance(device_data, ErrorInfo): + out_dict[device_id] = device_data + else: + files = [] + dirs = [] + for cur_file in device_data.files: + if cur_file.last_modified > last_modified_cutoff: + files.append(cur_file) + + for cur_dir in device_data.directories: + if cur_dir.last_modified > last_modified_cutoff: + dirs.append(cur_dir) + out_dict[device_id] = LsInfo(directories=dirs, files=files) + + return out_dict + + def exists(self, target, path, path_sep="/"): + """Check if path refers to an existing path on the device + + :param target: The device(s) to be targeted with this request + :type target: :class:`devicecloud.sci.TargetABC` or list of :class:`devicecloud.sci.TargetABC` instances + :param path: The path on the target to check for existence. + :param path_sep: The path separator of the device + :return: A dictionary where the key is a device id and the value is either an :class:`~.ErrorInfo` if there + was a problem with the operation or a boolean with the existence status of the path on that device + """ + if path.endswith(path_sep): + path = path[:-len(path_sep)] + par_dir, filename = path.rsplit(path_sep, 1) + file_list = self.list_files(target, par_dir) + out_dict = {} + for device_id, device_data in six.iteritems(file_list): + if isinstance(device_data, ErrorInfo): + out_dict[device_id] = device_data + else: + out_dict[device_id] = False + for cur_file in device_data.files: + if cur_file.path == path: + out_dict[device_id] = True + for cur_dir in device_data.directories: + if cur_dir.path == path: + out_dict[device_id] = True + + return out_dict diff --git a/devicecloud/filedata.py b/devicecloud/filedata.py index 971e373..1e14243 100644 --- a/devicecloud/filedata.py +++ b/devicecloud/filedata.py @@ -2,10 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015-2018 Digi International Inc. -"""Provide access to the device cloud filedata API""" +"""Provide access to Device Cloud filedata API""" import base64 @@ -26,7 +25,7 @@ class FileDataAPI(APIBase): - """Encapsulate data and logic required to interact with the device cloud file data store""" + """Encapsulate data and logic required to interact with Device Cloud file data store""" def get_filedata(self, condition=None, page_size=1000): """Return a generator over all results matching the provided condition @@ -47,28 +46,15 @@ def get_filedata(self, condition=None, page_size=1000): condition = validate_type(condition, type(None), Expression, *six.string_types) page_size = validate_type(page_size, *six.integer_types) - offset = 0 - remaining_size = 1 # just needs to be non-zero - if condition is None: condition = (fd_path == "~/") # home directory - while remaining_size > 0: - response = self._conn.get_json( - "/ws/FileData?embed=true" - "&start={offset}" - "&size={page_size}" - "&condition={condition}".format( - condition=condition.compile(), - page_size=page_size, - offset=offset)) - - offset += page_size - remaining_size = int(response.get("remainingSize", "0")) - for fd_json in response.get("items", []): - yield FileDataObject.from_json(self, fd_json) - - def write_file(self, path, name, data, content_type=None, archive=False): + params = {"embed": "true", "condition": condition.compile()} + for fd_json in self._conn.iter_json_pages("/ws/FileData", page_size=page_size, **params): + yield FileDataObject.from_json(self, fd_json) + + def write_file(self, path, name, data, content_type=None, archive=False, + raw=False): """Write a file to the file data store at the given path :param str path: The path (directory) into which the file should be written. @@ -80,6 +66,7 @@ def write_file(self, path, name, data, content_type=None, archive=False): :type content_type: str or None :param bool archive: If true, history will be retained for various revisions of this file. If this is not required, leave as false. + :param bool raw: If true, skip the FileData XML headers (necessary for binary files) """ path = validate_type(path, *six.string_types) @@ -94,31 +81,50 @@ def write_file(self, path, name, data, content_type=None, archive=False): path += "/" name = name.lstrip("/") - if six.PY3: - base64_encoded_data = base64.encodebytes(data).decode('utf-8') - else: - base64_encoded_data = base64.encodestring(data) - sio = six.moves.StringIO() - sio.write("") - if content_type is not None: - sio.write("{}".format(content_type)) - sio.write("file") - sio.write("{}".format(base64_encoded_data)) - sio.write("{}".format(archive_str)) - sio.write("") + if not raw: + if six.PY3: + base64_encoded_data = base64.encodebytes(data).decode('utf-8') + else: + base64_encoded_data = base64.encodestring(data) + + sio.write("") + if content_type is not None: + sio.write("{}".format(content_type)) + sio.write("file") + sio.write("{}".format(base64_encoded_data)) + sio.write("{}".format(archive_str)) + sio.write("") + else: + sio.write(data) params = { "type": "file", "archive": archive_str } self._conn.put( - "/ws/FileData{path}{name}".format(path=path,name=name), + "/ws/FileData{path}{name}".format(path=path, name=name), sio.getvalue(), params=params) + def delete_file(self, path): + """Delete a file or directory from the filedata store + + This method removes a file or directory (recursively) from + the filedata store. + + :param path: The path of the file or directory to remove + from the file data store. + + """ + path = validate_type(path, *six.string_types) + if not path.startswith("/"): + path = "/" + path + + self._conn.delete("/ws/FileData{path}".format(path=path)) + def walk(self, root="~/"): - """Emulation of os.walk behavior against the device cloud filedata store + """Emulation of os.walk behavior against Device Cloud filedata store This method will yield tuples in the form ``(dirpath, FileDataDirectory's, FileData's)`` recursively in pre-order (depth first from top down). @@ -169,6 +175,10 @@ def __init__(self, fdapi, json_data): self._fdapi = fdapi self._json_data = json_data + def delete(self): + """Delete this file or directory""" + return self._fdapi.delete_file(self.get_full_path()) + def get_data(self): """Get the data associated with this filedata object diff --git a/devicecloud/monitor.py b/devicecloud/monitor.py new file mode 100644 index 0000000..6a7942f --- /dev/null +++ b/devicecloud/monitor.py @@ -0,0 +1,381 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. +# +# This code is originally from another Digi Open Source Library: +# https://github.com/digidotcom/idigi-python-monitor-api + +import xml.etree.ElementTree as ET +import logging +import textwrap +from devicecloud.apibase import APIBase +from devicecloud.conditions import Attribute +from devicecloud.monitor_tcp import TCPClientManager + +logger = logging.getLogger(__name__) + +#: System-generated identifier for the monitor. +MON_ID_ATTR = Attribute("monId") + +#: Device Cloud customer identifier. +MON_CST_ID_ATTR = Attribute("cstId") + +#: One or more topics to monitor separated by comma. See Device Cloud +#: documentation for more details +MON_TOPIC_ATTR = Attribute("monTopic") + +#: Format for delivered event data: xml, json +MON_FORMAT_TYPE_ATTR = Attribute("monFormatType") + +#: Transport method used to deliver push notifications to the +#: client application: tcp, http +MON_TRANSPORT_TYPE_ATTR = Attribute("monTransportType") + +#: For HTTP transport type only. URL of the customer web server. For http URLs, +#: the default listening port is 80; for https URLs, the default listening +#: port is 443. +MON_HTTP_TRANSPORT_URL_ATTR = Attribute("monTransportUrl") + +#: For HTTP transport type only. Credentials for basic authentication +#: in the following format: ``username:password`` +MON_HTTP_TRANSPORT_TOKEN_ATTR = Attribute("monTransportToken") + +#: For HTTP transport type only. HTTP method to use for sending +#: data: PUT or POST. The default is PUT. +MON_HTTP_TRANSPORT_METHOD_ATTR = Attribute("monTransportMethod") + +#: For HTTP transport type only. Time in milliseconds Device Cloud waits +#: when attempting to connect to the destination http server. A value of +#: 0 means use the system default of 5000 (5 seconds). Most monitors do +#: not need to configure this setting. +MON_HTTP_CONNECT_TIMEOUT_ATTR = Attribute("monConnectTimeout") + +#: For HTTP transport type only. Time in milliseconds Device Cloud waits +#: for a response for pushed events from the http server. A value of 0 means +#: use the system default of 5000 (5 seconds). Most monitors do not need to +#: configure this setting. +MON_HTTP_RESPONSE_TIMEOUT_ATTR = Attribute("monResponseTimeout") + +#: For TCP transport type only. Indicates whether the client will explicitly +#: acknowledge TCP push events or allow Device Cloud to automatically acknowledge +#: events when sent. Options include: explicit or off. The default is off. +MON_TCP_ACK_OPTION_ATTR = Attribute("monAckOption") + +#: Specifies an upper bound on how many messages are aggregated before sending +#: a batch. The default is 100. +MON_BATCH_SIZE_ATTR = Attribute("monBatchSize") + +#: Specifies an upper bound on the number of seconds messages are aggregated +#: before sending. The default is 10. +MON_BATCH_DURATION_ATTR = Attribute("monBatchDuration") + +#: Keyword that specifies the method used to compress messages. Options include: +#: zlib or none. The default is none. For zlib, the deflate algorithm is used to +#: compress the data; use inflate to decompress the data. +#: +#: Note: For backwards compatibility, gzip is accepted as a valid keyword. +#: Compression has always been done using the deflate algorithm. +MON_COMPRESSION_ATTR = Attribute("monCompression") + +#: Boolean value that specifies whether Device Cloud replays any missed +#: published events before any new published events are forwarded. True +#: indicates missed published events are replayed. False indicates missed +#: published events are not replayed. The default is false. +MON_AUTO_REPLAY_ON_CONNECT = Attribute("monAutoReplayOnConnect") + +#: Optional text field used to label or describe the monitor. +MON_DESCRIPTION_ATTR = Attribute("monDescription") + +#: Specifies last connection time to the client application. +MON_LAST_CONNECT_ATTR = Attribute("monLastConnect") + +#: Specifies the last message pushed to the client application +MON_LAST_SENT_ATTR = Attribute("monLastSent") + +#: Specifies the current connection status to the client application: +#: +#: - CONNECTING: For HTTP monitors only. Device Cloud is attempting +#: to connect to the configured HTTP server. Once connected, the state changes to ACTIVE. +#: - ACTIVE: Monitor is connected and publishing events. +#: - INACTIVE: Monitor is not connected and events are not published or recorded. +#: - SUSPENDED: For monitors with monAutoReplayOnConnect = True. Monitor has disconnected, +#: but publish events are recorded for later replay. +#: - DISABLED: For HTTP monitors only. If a monitor has not connected for 24 hours, +#: the state is set to DISABLED, and publish events are not recorded for replay. +#: A disabled monitor must be reconfigured via the Monitor web service. +MON_STATUS_ATTR = Attribute("monStatus") + + +class MonitorAPI(APIBase): + """Provide access to Device Cloud Monitor API for receiving push notifications + + The Monitor API in Device Cloud allows for the creation and destruction of + multiple "monitors." Each monitor is registered against one or more "topics" + which describe the data in which it is interested. + + There are, in turn, two main ways to receive data matching the topics for a + given monitor: + + 1. Stream: Device Cloud supports a protocol over TCP (optionally with SSL) over which + the batches of events will be sent when they are received. + 2. HTTP: When batches of events are received, a configured web + service endpoint will received a POST request with the new data. + + Currently, this library supports setting up both types of monitors, but there + is no special support provided for parsing HTTP postback requests. + + More information on the format for topic strings can be found in the `device + cloud documentation for monitors `_. + + Here's a quick example showing a typical pattern used for creating a push monitor + and associated listener that triggers a callback. Deletion of existing monitors + matching the same topics is not necessary but sometimes done in order to ensure + that changes to the monitor configuration in code always make it to the monitor + configuration in Device Cloud:: + + def monitor_callback(json_data): + print(json_data) + return True # message received + + # Listen for DataPoint updates + topics = ['DataPoint[U]'] + monitor = dc.monitor.get_monitor(topics) + if monitor: + monitor.delete() + monitor = dc.monitor.create_tcp_monitor(topics) + monitor.add_listener(monitor_callback) + + # later... + dc.monitor.stop_listeners() + + When updates to any DataPoint in Device Cloud occurs, the callback will be called + with a data structure like this one:: + + {'Document': {'Msg': {'DataPoint': {'cstId': 7603, + 'data': 0.411700824929, + 'description': '', + 'id': '684572e0-12c4-11e5-8507-fa163ed4cf14', + 'quality': 0, + 'serverTimestamp': 1434307047694, + 'streamId': 'test', + 'streamUnits': '', + 'timestamp': 1434307047694}, + 'group': '*', + 'operation': 'INSERTION', + 'timestamp': '2015-06-14T18:37:27.815Z', + 'topic': '7603/DataPoint/test'}}} + """ + + def __init__(self, conn): + APIBase.__init__(self, conn) + # TODO: determine best way to expose additional options + self._tcp_client_manager = TCPClientManager(self._conn, secure=True) + + def create_tcp_monitor(self, topics, batch_size=1, batch_duration=0, + compression='gzip', format_type='json'): + """Creates a TCP Monitor instance in Device Cloud for a given list of topics + + :param topics: a string list of topics (e.g. ['DeviceCore[U]', + 'FileDataCore']). + :param batch_size: How many Msgs received before sending data. + :param batch_duration: How long to wait before sending batch if it + does not exceed batch_size. + :param compression: Compression value (i.e. 'gzip'). + :param format_type: What format server should send data in (i.e. 'xml' or 'json'). + + Returns an object of the created Monitor + """ + + monitor_xml = """\ + + {topics} + {batch_size} + {format_type} + tcp + {compression} + + """.format( + topics=','.join(topics), + batch_size=batch_size, + batch_duration=batch_duration, + format_type=format_type, + compression=compression, + ) + monitor_xml = textwrap.dedent(monitor_xml) + + response = self._conn.post("/ws/Monitor", monitor_xml) + location = ET.fromstring(response.text).find('.//location').text + monitor_id = int(location.split('/')[-1]) + return TCPDeviceCloudMonitor(self._conn, monitor_id, self._tcp_client_manager) + + def create_http_monitor(self, topics, transport_url, transport_token=None, transport_method='PUT', connect_timeout=0, + response_timeout=0, batch_size=1, batch_duration=0, compression='none', format_type='json'): + """Creates a HTTP Monitor instance in Device Cloud for a given list of topics + + :param topics: a string list of topics (e.g. ['DeviceCore[U]', + 'FileDataCore']). + :param transport_url: URL of the customer web server. + :param transport_token: Credentials for basic authentication in the following format: username:password + :param transport_method: HTTP method to use for sending data: PUT or POST. The default is PUT. + :param connect_timeout: A value of 0 means use the system default of 5000 (5 seconds). + :param response_timeout: A value of 0 means use the system default of 5000 (5 seconds). + :param batch_size: How many Msgs received before sending data. + :param batch_duration: How long to wait before sending batch if it + does not exceed batch_size. + :param compression: Compression value (i.e. 'gzip'). + :param format_type: What format server should send data in (i.e. 'xml' or 'json'). + + Returns an object of the created Monitor + """ + + monitor_xml = """\ + + {topics} + {batch_size} + {format_type} + http + {transport_url} + {transport_token} + {transport_method} + {connect_timeout} + {response_timeout} + {compression} + + """.format( + topics=','.join(topics), + transport_url=transport_url, + transport_token=transport_token, + transport_method=transport_method, + connect_timeout=connect_timeout, + response_timeout=response_timeout, + batch_size=batch_size, + batch_duration=batch_duration, + format_type=format_type, + compression=compression, + ) + monitor_xml = textwrap.dedent(monitor_xml) + + response = self._conn.post("/ws/Monitor", monitor_xml) + location = ET.fromstring(response.text).find('.//location').text + monitor_id = int(location.split('/')[-1]) + return HTTPDeviceCloudMonitor(self._conn, monitor_id) + + def get_monitors(self, condition=None, page_size=1000): + """Return an iterator over all monitors matching the provided condition + + Get all inactive monitors and print id:: + + for mon in dc.monitor.get_monitors(MON_STATUS_ATTR == "DISABLED"): + print(mon.get_id()) + + Get all the HTTP monitors and print id:: + + for mon in dc.monitor.get_monitors(MON_TRANSPORT_TYPE_ATTR == "http"): + print(mon.get_id()) + + Many other possibilities exist. See the :mod:`devicecloud.condition` documention + for additional details on building compound expressions. + + :param condition: An :class:`.Expression` which defines the condition + which must be matched on the monitor that will be retrieved from + Device Cloud. If a condition is unspecified, an iterator over + all monitors for this account will be returned. + :type condition: :class:`.Expression` or None + :param int page_size: The number of results to fetch in a single page. + :return: Generator yielding :class:`.DeviceCloudMonitor` instances matching the + provided conditions. + """ + req_kwargs = {} + if condition: + req_kwargs['condition'] = condition.compile() + for monitor_data in self._conn.iter_json_pages("/ws/Monitor", **req_kwargs): + yield DeviceCloudMonitor.from_json(self._conn, monitor_data, self._tcp_client_manager) + + def get_monitor(self, topics): + """Attempts to find a Monitor in device cloud that matches the provided topics + + :param topics: a string list of topics (e.g. ``['DeviceCore[U]', 'FileDataCore'])``) + + Returns a :class:`DeviceCloudMonitor` if found, otherwise None. + """ + for monitor in self.get_monitors(MON_TOPIC_ATTR == ",".join(topics)): + return monitor # return the first one, even if there are multiple + return None + + def stop_listeners(self): + """Stop any listener threads that may be running and join on them""" + self._tcp_client_manager.stop() + + +class DeviceCloudMonitor(object): + """Provides access to a single monitor instance on Device Cloud + + This is a base class that should not be instantiated directly. + + :type _tcp_client_manager: devicecloud.monitor_tcp.TCPClientManager + :type _conn: devicecloud.DeviceCloudConnection + """ + + # TODO: consider adding getters/setters for each metadata + + @classmethod + def from_json(cls, conn, monitor_data, tcp_client_manager): + monitor_id = int(monitor_data['monId']) + transport_type = monitor_data['monTransportType'] + kls = { + "tcp": TCPDeviceCloudMonitor, + "http": HTTPDeviceCloudMonitor, + }.get(transport_type.lower()) + if kls is None: + raise ValueError("Unexpected monTransportType %r" % transport_type) + return kls(conn, monitor_id, tcp_client_manager) + + def __init__(self, conn, monitor_id, *args, **kwargs): + self._conn = conn + self._id = monitor_id + + def get_id(self): + """Get the ID of this monitor as an integer""" + return self._id + + def get_metadata(self): + """Get additional information about this monitor + + This method returns a dictionary where the keys contain information about the + monitor. The returned data will look something like this:: + + { + 'cstId': '7603', + 'monBatchDuration': '10', + 'monBatchSize': '1', + 'monCompression': 'zlib', + 'monFormatType': 'json', + 'monId': '178023', + 'monStatus': 'INACTIVE', + 'monTopic': 'DeviceCore,FileDataCore,FileData,DataPoint', + 'monTransportType': 'tcp' + } + """ + return self._conn.get_json("/ws/Monitor/{id}".format(id=self._id))["items"][0] + + def delete(self): + """Delete this monitor form Device Cloud""" + self._conn.delete("/ws/Monitor/{id}".format(id=self._id)) + + +class HTTPDeviceCloudMonitor(DeviceCloudMonitor): + """Device Cloud Monitor with HTTP transport type""" + + +class TCPDeviceCloudMonitor(DeviceCloudMonitor): + """Device Cloud Monitor with TCP transport type""" + + def __init__(self, conn, monitor_id, tcp_client_manager): + DeviceCloudMonitor.__init__(self, conn, monitor_id) + self._tcp_client_manager = tcp_client_manager + + def add_callback(self, callback): + """Create a secure SSL/TCP listen session to Device Cloud""" + self._tcp_client_manager.create_session(callback, self._id) diff --git a/devicecloud/monitor_tcp.py b/devicecloud/monitor_tcp.py new file mode 100644 index 0000000..bf3d75c --- /dev/null +++ b/devicecloud/monitor_tcp.py @@ -0,0 +1,579 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. +# +# This code is originally from another Digi Open Source Library: +# https://github.com/digidotcom/idigi-python-monitor-api + +import json +import logging +import socket +import struct +from threading import Thread +import errno +import select +import zlib +import ssl + +import pkg_resources +from six.moves.queue import Queue, Empty +import six + +DEFAULT_CRT_NAME = "devicecloud.crt" + +# Push Opcodes. +CONNECTION_REQUEST = 0x01 +CONNECTION_RESPONSE = 0x02 +PUBLISH_MESSAGE = 0x03 +PUBLISH_MESSAGE_RECEIVED = 0x04 + +# Data has not been completely read. +INCOMPLETE = -1 +# No Data Received on Socket. +NO_DATA = -2 + +# Possible Responses from iDigi with respect to Push. +STATUS_OK = 200 +STATUS_UNAUTHORIZED = 403 +STATUS_BAD_REQUEST = 400 + +# Ports to Connect on for Push. +PUSH_OPEN_PORT = 3200 +PUSH_SECURE_PORT = 3201 + + +def _read_msg_header(session): + """ + Perform a read on input socket to consume headers and then return + a tuple of message type, message length. + + :param session: Push Session to read data for. + + Returns response type (i.e. PUBLISH_MESSAGE) if header was completely + read, otherwise None if header was not completely read. + """ + try: + data = session.socket.recv(6 - len(session.data)) + if len(data) == 0: # No Data on Socket. Likely closed. + return NO_DATA + session.data += data + # Data still not completely read. + if len(session.data) < 6: + return INCOMPLETE + + except ssl.SSLError: + # This can happen when select gets triggered + # for an SSL socket and data has not yet been + # read. + return INCOMPLETE + + session.message_length = struct.unpack('!i', session.data[2:6])[0] + response_type = struct.unpack('!H', session.data[0:2])[0] + + # Clear out session data as header is consumed. + session.data = six.b("") + return response_type + + +def _read_msg(session): + """ + Perform a read on input socket to consume message and then return the + payload and block_id in a tuple. + + :param session: Push Session to read data for. + """ + if len(session.data) == session.message_length: + # Data Already completely read. Return + return True + + try: + data = session.socket.recv(session.message_length - len(session.data)) + if len(data) == 0: + raise PushException("No Data on Socket!") + session.data += data + except ssl.SSLError: + # This can happen when select gets triggered + # for an SSL socket and data has not yet been + # read. Wait for it to get triggered again. + return False + + # Whether or not all data was read. + return len(session.data) == session.message_length + + +class PushException(Exception): + """Indicates an issue interacting with Push Functionality.""" + + +class PushSession(object): + """ + A PushSession is responsible for establishing a socket connection + with iDigi to receive events generated by Devices connected to + iDigi. + """ + + def __init__(self, callback, monitor_id, client): + """Creates a PushSession for use with Device Cloud + + :param callback: The callback function to invoke when data received. + Must have 1 required parameter that will contain the payload. + :param monitor_id: The id of the Monitor to observe. + :param client: The client object this session is derived from. + """ + self.callback = callback + self.monitor_id = monitor_id + self.client = client + self.socket = None + self.log = logging.getLogger("%s.push_session.%s" % (__name__, monitor_id)) + + # Received protocol data holders. + self.data = six.b("") + self.message_length = 0 + + def send_connection_request(self): + """ + Sends a ConnectionRequest to the iDigi server using the credentials + established with the id of the monitor as defined in the monitor + member. + """ + try: + self.log.info("Sending ConnectionRequest for Monitor %s." + % self.monitor_id) + # Send connection request and perform a receive to ensure + # request is authenticated. + # Protocol Version = 1. + payload = struct.pack('!H', 0x01) + # Username Length. + payload += struct.pack('!H', len(self.client.username)) + # Username. + payload += six.b(self.client.username) + # Password Length. + payload += struct.pack('!H', len(self.client.password)) + # Password. + payload += six.b(self.client.password) + # Monitor ID. + payload += struct.pack('!L', int(self.monitor_id)) + + # Header 6 Bytes : Type [2 bytes] & Length [4 Bytes] + # ConnectionRequest is Type 0x01. + data = struct.pack("!HL", CONNECTION_REQUEST, len(payload)) + + # The full payload. + data += payload + + # Send Connection Request. + self.socket.send(data) + + # Set a 60 second blocking on recv, if we don't get any data + # within 60 seconds, timeout which will throw an exception. + self.socket.settimeout(60) + + # Should receive 10 bytes with ConnectionResponse. + response = self.socket.recv(10) + + # Make socket blocking. + self.socket.settimeout(0) + + if len(response) != 10: + raise PushException("Length of Connection Request Response " + "(%d) is not 10." % len(response)) + + # Type + response_type = int(struct.unpack("!H", response[0:2])[0]) + if response_type != CONNECTION_RESPONSE: + raise PushException( + "Connection Response Type (%d) is not " + "ConnectionResponse Type (%d)." % (response_type, CONNECTION_RESPONSE)) + + status_code = struct.unpack("!H", response[6:8])[0] + self.log.info("Got ConnectionResponse for Monitor %s. Status %s." + % (self.monitor_id, status_code)) + if status_code != STATUS_OK: + raise PushException("Connection Response Status Code (%d) is " + "not STATUS_OK (%d)." % (status_code, STATUS_OK)) + except Exception as exception: + # TODO(posborne): This is bad! It isn't necessarily a socket exception! + # Likely a socket exception, close it and raise an exception. + self.socket.close() + self.socket = None + raise exception + + def start(self): + """Creates a TCP connection to Device Cloud and sends a ConnectionRequest message""" + self.log.info("Starting Insecure Session for Monitor %s" % self.monitor_id) + if self.socket is not None: + raise Exception("Socket already established for %s." % self) + + try: + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.socket.connect((self.client.hostname, PUSH_OPEN_PORT)) + self.socket.setblocking(0) + except socket.error as exception: + self.socket.close() + self.socket = None + raise + + self.send_connection_request() + + def stop(self): + """Stop/Close this session + + Close the socket associated with this session and puts Session + into a state such that it can be re-established later. + """ + if self.socket is not None: + self.socket.close() + self.socket = None + self.data = None + + +class SecurePushSession(PushSession): + """ + SecurePushSession extends PushSession by wrapping the socket connection + in SSL. It expects the certificate to match any of those in the passed + in ca_certs member file. + """ + + def __init__(self, callback, monitor_id, client, ca_certs=None): + """ + Creates a PushSession wrapped in SSL for use with interacting with + Device Cloud push functionality. + + :param callback: The callback function to invoke when data is received. + Must have 1 required parameter that will contain the + payload. + :param monitor_id: The id of the Monitor to observe. + :param client: The client object this session is derived from. + :param ca_certs: Path to a file containing Certificates. + If not provided, the devicecloud.crt file provided with the module will + be used. In most cases, the devicecloud.crt file should be acceptable. + """ + PushSession.__init__(self, callback, monitor_id, client) + # Fall back on devicecloud.crt in the same path as this module if not + # specified. + if ca_certs is None: + ca_certs = pkg_resources.resource_filename("devicecloud.data", DEFAULT_CRT_NAME) + self.ca_certs = ca_certs + + def start(self): + """ + Creates a SSL connection to the iDigi Server and sends a + ConnectionRequest message. + """ + self.log.info("Starting SSL Session for Monitor %s." + % self.monitor_id) + if self.socket is not None: + raise Exception("Socket already established for %s." % self) + + try: + # Create socket, wrap in SSL and connect. + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # Validate that certificate server uses matches what we expect. + if self.ca_certs is not None: + self.socket = ssl.wrap_socket(self.socket, + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=self.ca_certs) + else: + self.socket = ssl.wrap_socket(self.socket) + + self.socket.connect((self.client.hostname, PUSH_SECURE_PORT)) + self.socket.setblocking(0) + except Exception as exception: + self.socket.close() + self.socket = None + raise exception + + self.send_connection_request() + + +class CallbackWorkerPool(object): + """ + A Worker Pool implementation that creates a number of predefined threads + used for invoking Session callbacks. + """ + + def __init__(self, write_queue=None, size=1): + """ + Creates a Callback Worker Pool for use in invoking Session Callbacks + when data is received by a push client. + + :param write_queue: Queue used for queueing up socket write events + for when a payload message is received and processed. + :param size: The number of worker threads to invoke callbacks. + """ + # Used to queue up PublishMessageReceived events to be sent back to + # the iDigi server. + self._write_queue = write_queue + # Used to queue up sessions and data to callback with. + self._queue = Queue(size) + # Number of workers to create. + self.size = size + self.log = logging.getLogger('{}.callback_worker_pool'.format(__name__)) + + for _ in range(size): + worker = Thread(target=self._consume_queue) + worker.daemon = True + worker.start() + + def _consume_queue(self): + """ + Continually blocks until data is on the internal queue, then calls + the session's registered callback and sends a PublishMessageReceived + if callback returned True. + """ + while True: + session, block_id, raw_data = self._queue.get() + data = json.loads(raw_data.decode('utf-8')) # decode as JSON + try: + result = session.callback(data) + if result is None: + self.log.warn("Callback %r returned None, expected boolean. Messages " + "are not marked as received unless True is returned", session.callback) + elif result: + # Send a Successful PublishMessageReceived with the + # block id sent in request + if self._write_queue is not None: + response_message = struct.pack('!HHH', + PUBLISH_MESSAGE_RECEIVED, + block_id, 200) + self._write_queue.put((session.socket, response_message)) + except Exception as exception: + self.log.exception(exception) + + self._queue.task_done() + + def queue_callback(self, session, block_id, data): + """ + Queues up a callback event to occur for a session with the given + payload data. Will block if the queue is full. + + :param session: the session with a defined callback function to call. + :param block_id: the block_id of the message received. + :param data: the data payload of the message received. + """ + self._queue.put((session, block_id, data)) + + +class TCPClientManager(object): + """A Client for the 'Push' feature in Device Cloud""" + + def __init__(self, conn, secure=True, ca_certs=None, workers=1): + """ + Arbitrator for multiple TCP Client Sessions + + :param conn: The :class:`devicecloud.DeviceCloudConnection` to use + :param secure: Whether or not to create a secure SSL wrapped session. + :param ca_certs: Path to a file containing Certificates. + If not provided, the devicecloud.crt file provided with the module will + be used. In most cases, the devicecloud.crt file should be acceptable. + :param workers: Number of workers threads to process callback calls. + """ + self._conn = conn + self._secure = secure + self._ca_certs = ca_certs + + # A dict mapping Sockets to their PushSessions + self.sessions = {} + # IO thread is used monitor sockets and consume data. + self._io_thread = None + # Writer thread is used to send data on sockets. + self._writer_thread = None + # Write queue is used to queue up data to write to sockets. + self._write_queue = Queue() + # A pool that monitors callback events and invokes them. + self._callback_pool = CallbackWorkerPool(self._write_queue, size=workers) + + self.closed = False + self.log = logging.getLogger(__name__) + + @property + def hostname(self): + return self._conn.hostname + + @property + def username(self): + return self._conn.username + + @property + def password(self): + return self._conn.password + + def _restart_session(self, session): + """Restarts and re-establishes session + + :param session: The session to restart + """ + # remove old session key, if socket is None, that means the + # session was closed by user and there is no need to restart. + if session.socket is not None: + self.log.info("Attempting restart session for Monitor Id %s." + % session.monitor_id) + del self.sessions[session.socket.fileno()] + session.stop() + session.start() + self.sessions[session.socket.fileno()] = session + + def _writer(self): + """ + Indefinitely checks the writer queue for data to write + to socket. + """ + while not self.closed: + try: + sock, data = self._write_queue.get(timeout=0.1) + self._write_queue.task_done() + sock.send(data) + except Empty: + pass # nothing to write after timeout + except socket.error as err: + if err.errno == errno.EBADF: + self._clean_dead_sessions() + + def _clean_dead_sessions(self): + """ + Traverses sessions to determine if any sockets + were removed (indicates a stopped session). + In these cases, remove the session. + """ + for sck in list(self.sessions.keys()): + session = self.sessions[sck] + if session.socket is None: + del self.sessions[sck] + + def _select(self): + """ + While the client is not marked as closed, performs a socket select + on all PushSession sockets. If any data is received, parses and + forwards it on to the callback function. If the callback is + successful, a PublishMessageReceived message is sent. + """ + try: + while not self.closed: + try: + inputready = select.select(self.sessions.keys(), [], [], 0.1)[0] + for sock in inputready: + session = self.sessions[sock] + sck = session.socket + + if sck is None: + # Socket has since been deleted, continue + continue + + # If no defined message length, nothing has been + # consumed yet, parse the header. + if session.message_length == 0: + # Read header information before receiving rest of + # message. + response_type = _read_msg_header(session) + if response_type == NO_DATA: + # No data could be read, assume socket closed. + if session.socket is not None: + self.log.error("Socket closed for Monitor %s." % session.monitor_id) + self._restart_session(session) + continue + elif response_type == INCOMPLETE: + # More Data to be read. Continue. + continue + elif response_type != PUBLISH_MESSAGE: + self.log.warn("Response Type (%x) does not match PublishMessage (%x)" + % (response_type, PUBLISH_MESSAGE)) + continue + + try: + if not _read_msg(session): + # Data not completely read, continue. + continue + except PushException as err: + # If Socket is None, it was closed, + # otherwise it was closed when it shouldn't + # have been restart it. + session.data = six.b("") + session.message_length = 0 + + if session.socket is None: + del self.sessions[sck] + else: + self.log.exception(err) + self._restart_session(session) + continue + + # We received full payload, + # clear session data and parse it. + data = session.data + session.data = six.b("") + session.message_length = 0 + block_id = struct.unpack('!H', data[0:2])[0] + compression = struct.unpack('!B', data[4:5])[0] + payload = data[10:] + + if compression == 0x01: + # Data is compressed, uncompress it. + payload = zlib.decompress(payload) + + # Enqueue payload into a callback queue to be + # invoked + self._callback_pool.queue_callback(session, block_id, payload) + except select.error as err: + # Evaluate sessions if we get a bad file descriptor, if + # socket is gone, delete the session. + if err.args[0] == errno.EBADF: + self._clean_dead_sessions() + except Exception as err: + self.log.exception(err) + finally: + for session in self.sessions.values(): + if session is not None: + session.stop() + + def _init_threads(self): + """Initializes the IO and Writer threads""" + if self._io_thread is None: + self._io_thread = Thread(target=self._select) + self._io_thread.start() + + if self._writer_thread is None: + self._writer_thread = Thread(target=self._writer) + self._writer_thread.start() + + def create_session(self, callback, monitor_id): + """ + Creates and Returns a PushSession instance based on the input monitor + and callback. When data is received, callback will be invoked. + If neither monitor or monitor_id are specified, throws an Exception. + + :param callback: Callback function to call when PublishMessage + messages are received. Expects 1 argument which will contain the + payload of the pushed message. Additionally, expects + function to return True if callback was able to process + the message, False or None otherwise. + :param monitor_id: The id of the Monitor, will be queried + to understand parameters of the monitor. + """ + self.log.info("Creating Session for Monitor %s." % monitor_id) + session = SecurePushSession(callback, monitor_id, self, self._ca_certs) \ + if self._secure else PushSession(callback, monitor_id, self) + + session.start() + self.sessions[session.socket.fileno()] = session + + self._init_threads() + return session + + def stop(self): + """Stops all session activity. + + Blocks until io and writer thread dies + """ + if self._io_thread is not None: + self.log.info("Waiting for I/O thread to stop...") + self.closed = True + self._io_thread.join() + + if self._writer_thread is not None: + self.log.info("Waiting for Writer Thread to stop...") + self.closed = True + self._writer_thread.join() + + self.log.info("All worker threads stopped.") diff --git a/devicecloud/sci.py b/devicecloud/sci.py index 80f0d84..8baf9af 100644 --- a/devicecloud/sci.py +++ b/devicecloud/sci.py @@ -2,17 +2,17 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015-2018 Digi International Inc. """Server Command Interface functionality""" from devicecloud.apibase import APIBase +from xml.etree import ElementTree as ET import six SCI_TEMPLATE = """\ - <{operation}{synchronous}{cache}{sync_timeout}{allow_offline}{wait_for_reconnect}> + <{operation}{attribute}{reply}{synchronous}{cache}{sync_timeout}{allow_offline}{wait_for_reconnect}> {targets} @@ -63,14 +63,75 @@ def __init__(self, group): self._group = group def to_xml(self): - return ''.format(self._group) + return ''.format(self._group) + + +class AsyncRequestProxy(object): + """An object representing an asynychronous SCI request. + + Can be used for polling the status of the corresponding request. + + :ivar job_id: the ID in device cloud of the job + :ivar response: the response to the request if completed + :ivar completed: True if the request has completed, False otherwise; queries on read + """ + def __init__(self, job_id, conn): + self.job_id = job_id + self._conn = conn + self.response = None + + @property + def completed(self): + if self.response is not None: + return True + resp = self._conn.get('/ws/sci/{0}'.format(self.job_id)) + dom = ET.fromstring(resp.content) + status = dom.find('.//status') + if status is not None and status.text == 'complete': + self.response = resp.content + return True + else: + return False class ServerCommandInterfaceAPI(APIBase): """Encapsulate Server Command Interface API""" + def get_async_job(self, job_id): + """Query an asynchronous SCI job by ID + + This is useful if the job was not created with send_sci_async(). + + :param int job_id: The job ID to query + :returns: The SCI response from GETting the job information + """ + uri = "/ws/sci/{0}".format(job_id) + # TODO: do parsing here? + return self._conn.get(uri) + + def send_sci_async(self, operation, target, payload, **sci_options): + """Send an asynchronous SCI request, and wraps the job in an object + to manage it + + :param str operation: The operation is one of {send_message, update_firmware, disconnect, query_firmware_targets, + file_system, data_service, and reboot} + :param target: The device(s) to be targeted with this request + :type target: :class:`~.TargetABC` or list of :class:`~.TargetABC` instances + + TODO: document other params + + """ + sci_options['synchronous'] = False + resp = self.send_sci(operation, target, payload, **sci_options) + dom = ET.fromstring(resp.content) + job_element = dom.find('.//jobId') + if job_element is None: + return + job_id = int(job_element.text) + return AsyncRequestProxy(job_id, self._conn) + def send_sci(self, operation, target, payload, reply=None, synchronous=None, sync_timeout=None, - cache=None, allow_offline=None, wait_for_reconnect=None): + cache=None, allow_offline=None, wait_for_reconnect=None, attribute=None): """Send SCI request to 1 or more targets :param str operation: The operation is one of {send_message, update_firmware, disconnect, query_firmware_targets, @@ -81,8 +142,8 @@ def send_sci(self, operation, target, payload, reply=None, synchronous=None, syn TODO: document other params """ - if not isinstance(payload, six.string_types): - raise TypeError("payload is required to be string") + if not isinstance(payload, six.string_types) and not isinstance(payload, six.binary_type): + raise TypeError("payload is required to be a string or bytes") # validate targets and bulid targets xml section try: @@ -112,7 +173,7 @@ def send_sci(self, operation, target, payload, reply=None, synchronous=None, syn # sync_timeout argument # TODO: What units is syncTimeout in? seconds? - if not sync_timeout is None or isinstance(sync_timeout, six.integer_types): + if sync_timeout is not None and not isinstance(sync_timeout, six.integer_types): raise TypeError("sync_timeout expected to either be None or a number") if sync_timeout is not None: sync_timeout_xml = ' syncTimeout="{}"'.format(sync_timeout) @@ -143,6 +204,11 @@ def send_sci(self, operation, target, payload, reply=None, synchronous=None, syn else: wait_for_reconnect_xml = '' + if attribute is not None: + operation_attribute = ' ' + attribute + else: + operation_attribute = '' + full_request = SCI_TEMPLATE.format( operation=operation, targets=targets_xml, @@ -152,7 +218,8 @@ def send_sci(self, operation, target, payload, reply=None, synchronous=None, syn cache=cache_xml, allow_offline=allow_offline_xml, wait_for_reconnect=wait_for_reconnect_xml, - payload=payload + payload=payload, + attribute=operation_attribute ) # TODO: do parsing here? diff --git a/devicecloud/streams.py b/devicecloud/streams.py index 2c4fc77..83693c1 100644 --- a/devicecloud/streams.py +++ b/devicecloud/streams.py @@ -2,11 +2,10 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015-2018 Digi International Inc. r"""Module providing classes for interacting with device cloud data streams""" - +import json import logging import datetime @@ -26,6 +25,8 @@ STREAM_TYPE_DOUBLE = "DOUBLE" STREAM_TYPE_STRING = "STRING" STREAM_TYPE_BINARY = "BINARY" +STREAM_TYPE_JSON = "JSON" +STREAM_TYPE_GEOJSON = "GEOJSON" STREAM_TYPE_UNKNOWN = "UNKNOWN" ROLLUP_INTERVAL_HALF = "half" @@ -54,6 +55,8 @@ STREAM_TYPE_STRING: (str, str), STREAM_TYPE_BINARY: (str, str), STREAM_TYPE_UNKNOWN: (str, str), + STREAM_TYPE_JSON: (json.loads, json.dumps), + STREAM_TYPE_GEOJSON: (json.loads, json.dumps) } @@ -62,6 +65,34 @@ logger = logging.getLogger("devicecloud.streams") +def _get_encoder_method(stream_type): + """A function to get the python type to device cloud type converter function. + + :param stream_type: The streams data type + :return: A function that when called with the python object will return the serializable + type for sending to the cloud. If there is no function for the given type, or the `stream_type` + is `None` the returned function will simply return the object unchanged. + """ + if stream_type is not None: + return DSTREAM_TYPE_MAP.get(stream_type.upper(), (lambda x: x, lambda x: x))[1] + else: + return lambda x: x + + +def _get_decoder_method(stream_type): + """ A function to get Device Cloud type to python type converter function. + + :param stream_type: The streams data type + :return: A function that when called with Device Cloud object will return the python + native type. If there is no function for the given type, or the `stream_type` is `None` + the returned function will simply return the object unchanged. + """ + if stream_type is not None: + return DSTREAM_TYPE_MAP.get(stream_type.upper(), (lambda x: x, lambda x: x))[0] + else: + return lambda x: x + + class StreamException(DeviceCloudException): """Base class for stream related exceptions""" @@ -84,11 +115,15 @@ class StreamsAPI(APIBase): def __init__(self, *args, **kwargs): APIBase.__init__(self, *args, **kwargs) - def _get_streams(self): + def _get_streams(self, uri_suffix=None): """Clear and update internal cache of stream objects""" # TODO: handle paging, perhaps change this to be a generator + if uri_suffix is not None and not uri_suffix.startswith('/'): + uri_suffix = '/' + uri_suffix + elif uri_suffix is None: + uri_suffix = "" streams = {} - response = self._conn.get_json("/ws/DataStream") + response = self._conn.get_json("/ws/DataStream{}".format(uri_suffix)) for stream_data in response["items"]: stream_id = stream_data["streamId"] stream = DataStream(self._conn, stream_id, stream_data) @@ -97,12 +132,12 @@ def _get_streams(self): def create_stream(self, stream_id, data_type, description=None, data_ttl=None, rollup_ttl=None, units=None): - """Create a new data stream on the device cloud + """Create a new data stream on Device Cloud - This method will attempt to create a new data stream on the device cloud. + This method will attempt to create a new data stream on Device Cloud. This method will only succeed if the stream does not already exist. - :param str stream_id: The path/id of the stream being created on the device cloud. + :param str stream_id: The path/id of the stream being created on Device Cloud. :param str data_type: The type of this stream. This must be in the set `{ INTEGER, LONG, FLOAT, DOUBLE, STRING, BINARY, UNKNOWN }`. These values are available in constants like :attr:`~STREAM_TYPE_INTEGER`. @@ -139,14 +174,16 @@ def create_stream(self, stream_id, data_type, description=None, data_ttl=None, stream = DataStream(self._conn, stream_id) return stream - def get_streams(self): - """Return the iterator over all streams present on the device cloud + def get_streams(self, stream_prefix=None): + """Return the iterator over streams preset on device cloud. - :return: iterator over all :class:`.DataStream` instances on the device cloud + :param stream_prefix: An optional prefix to limit the iterator to; all streams are returned if it is not specified. + + :return: iterator over all :class:`.DataStream` instances on Device Cloud """ # TODO: deal with paging. We now return a generator, so the interface should look the same - return iter(self._get_streams().values()) + return iter(self._get_streams(stream_prefix).values()) def get_stream(self, stream_id): """Return a reference to a stream with the given ``stream_id`` @@ -156,7 +193,7 @@ def get_stream(self, stream_id): stream exists, one can use :py:meth:`get_stream_if_exists` which will return None if the stream is not already created. - :param stream_id: The path of the stream on the device cloud + :param stream_id: The path of the stream on Device Cloud :raises TypeError: if the stream_id provided is the wrong type :raises ValueError: if the stream_id is not properly formed :return: datastream instance with the provided stream_id @@ -171,7 +208,7 @@ def get_stream_if_exists(self, stream_id): This works similar to :py:meth:`get_stream` but will return None if the stream is not already created. - :param stream_id: The path of the stream on the device cloud + :param stream_id: The path of the stream on Device Cloud :raises TypeError: if the stream_id provided is the wrong type :raises ValueError: if the stream_id is not properly formed :return: :class:`.DataStream` instance with the provided stream_id @@ -190,7 +227,7 @@ def bulk_write_datapoints(self, datapoints): """Perform a bulk write (or set of writes) of a collection of data points This method takes a list (or other iterable) of datapoints and writes them - to the device cloud in an efficient manner, minimizing the number of HTTP + to Device Cloud in an efficient manner, minimizing the number of HTTP requests that need to be made. As this call is performed from outside the context of any particular stream, @@ -213,14 +250,14 @@ def bulk_write_datapoints(self, datapoints): dc.streams.bulk_write_datapoints(datapoints) Depending on the size of the list of datapoints provided, this method may - need to make multiple calls to the device cloud (in chunks of 250). + need to make multiple calls to Device Cloud (in chunks of 250). - :param list datapoints: a list of datapoints to be written to the device cloud + :param list datapoints: a list of datapoints to be written to Device Cloud :raises TypeError: if a list of datapoints is not provided :raises ValueError: if any of the provided data points do not have all required information (such as information about the stream) :raises DeviceCloudHttpException: in the case of an unexpected error in communicating - with the device cloud. + with Device Cloud. """ datapoints = list(datapoints) # effectively performs validation that we have the right type @@ -253,7 +290,7 @@ class DataPoint(object): This class encapsulates the data required for both pushing data points to the device cloud as well as for storing and provding methods to access data from - streams that has been retrieved from the device cloud. + streams that has been retrieved from Device Cloud. """ @@ -262,11 +299,13 @@ def from_json(cls, stream, json_data): """Create a new DataPoint object from device cloud JSON data :param DataStream stream: The :class:`~DataStream` out of which this data is coming - :param dict json_data: Deserialized JSON data from the device cloud about this device + :param dict json_data: Deserialized JSON data from Device Cloud about this device :raises ValueError: if the data is malformed :return: (:class:`~DataPoint`) newly created :class:`~DataPoint` """ + type_converter = _get_decoder_method(stream.get_data_type()) + data = type_converter(json_data.get("data")) return cls( # these are actually properties of the stream, not the data point stream_id=stream.get_stream_id(), @@ -274,7 +313,7 @@ def from_json(cls, stream, json_data): units=stream.get_units(), # and these are part of the data point itself - data=json_data.get("data"), + data=data, description=json_data.get("description"), timestamp=json_data.get("timestampISO"), server_timestamp=json_data.get("serverTimestampISO"), @@ -288,7 +327,7 @@ def from_rollup_json(cls, stream, json_data): """Rollup json data from the server looks slightly different :param DataStream stream: The :class:`~DataStream` out of which this data is coming - :param dict json_data: Deserialized JSON data from the device cloud about this device + :param dict json_data: Deserialized JSON data from Device Cloud about this device :raises ValueError: if the data is malformed :return: (:class:`~DataPoint`) newly created :class:`~DataPoint` """ @@ -298,8 +337,8 @@ def from_rollup_json(cls, stream, json_data): timestamp = isoformat(dc_utc_timestamp_to_dt(int(json_data.get("timestamp")))) # Special handling for data, all rollup data is float type - type_converter = DSTREAM_TYPE_MAP[dp.get_data_type()] - data = type_converter[0](float(json_data.get("data"))) + type_converter = _get_decoder_method(stream.get_data_type()) + data = type_converter(float(json_data.get("data"))) # Update the special fields dp.set_timestamp(timestamp) @@ -374,12 +413,7 @@ def get_id(self): def get_data(self): """Get the actual data value associated with this data point""" - data = self._data - if self._data_type is not None: - type_converters = DSTREAM_TYPE_MAP.get(self._data_type.upper()) - if type_converters: - data = type_converters[0](self._data) - return data + return self._data def set_data(self, data): """Set the data for this data point @@ -443,7 +477,7 @@ def get_quality(self): def set_quality(self, quality): """Set the quality for this sample - Quality is stored on the device cloud as a 32-bit integer, so the input + Quality is stored on Device Cloud as a 32-bit integer, so the input to this function should be either None, an integer, or a string that can be converted to an integer. @@ -545,10 +579,14 @@ def to_xml(self): set on this datapoint. Values not set (e.g. quality) will be ommitted. """ + type_converter = _get_encoder_method(self._data_type) + # Convert from python native to device cloud + encoded_data = type_converter(self._data) + out = StringIO() out.write("") out.write("{}".format(self.get_stream_id())) - out.write("{}".format(self.get_data())) + out.write("{}".format(encoded_data)) conditional_write(out, "{}", self.get_description()) if self.get_timestamp() is not None: out.write("{}".format(isoformat(self.get_timestamp()))) @@ -602,13 +640,13 @@ def __repr__(self): )) def _get_stream_metadata(self, use_cached): - """Retrieve metadata about this stream from the device cloud""" + """Retrieve metadata about this stream from Device Cloud""" if self._cached_data is None or not use_cached: try: self._cached_data = self._conn.get_json("/ws/DataStream/%s" % self._stream_id)["items"][0] except DeviceCloudHttpException as http_exception: if http_exception.response.status_code == 404: - raise NoSuchStreamException("Stream with id %r has not been created", self._stream_id) + raise NoSuchStreamException("Stream with id %r has not been created" % self._stream_id) raise http_exception return self._cached_data @@ -638,7 +676,7 @@ def get_data_type(self, use_cached=True): * BINARY - Data with this type map to a python string. * UNKNOWN - Data with this type map to a python string. - :param bool use_cached: If False, the function will always request the latest from the device cloud. + :param bool use_cached: If False, the function will always request the latest from Device Cloud. If True, the device will not make a request if it already has cached data. :return: The data type of this stream as a string :rtype: str @@ -654,7 +692,7 @@ def get_units(self, use_cached=True): Units are a user-defined field stored as a string - :param bool use_cached: If False, the function will always request the latest from the device cloud. + :param bool use_cached: If False, the function will always request the latest from Device Cloud. If True, the device will not make a request if it already has cached data. :return: The unit of this stream as a string :rtype: str or None @@ -665,7 +703,7 @@ def get_units(self, use_cached=True): def get_description(self, use_cached=True): """Get the description associated with this data stream - :param bool use_cached: If False, the function will always request the latest from the device cloud. + :param bool use_cached: If False, the function will always request the latest from Device Cloud. If True, the device will not make a request if it already has cached data. :raises devicecloud.DeviceCloudHttpException: in the case of an unexpected http error :raises devicecloud.streams.NoSuchStreamException: if this stream has not yet been created @@ -681,7 +719,7 @@ def get_data_ttl(self, use_cached=True): The dataTtl is the time to live (TTL) in seconds for data points stored in the data stream. A data point expires after the configured amount of time and is automatically deleted. - :param bool use_cached: If False, the function will always request the latest from the device cloud. + :param bool use_cached: If False, the function will always request the latest from Device Cloud. If True, the device will not make a request if it already has cached data. :raises devicecloud.DeviceCloudHttpException: in the case of an unexpected http error :raises devicecloud.streams.NoSuchStreamException: if this stream has not yet been created @@ -700,7 +738,7 @@ def get_rollup_ttl(self, use_cached=True): stored in the stream. A roll-up expires after the configured amount of time and is automatically deleted. - :param bool use_cached: If False, the function will always request the latest from the device cloud. + :param bool use_cached: If False, the function will always request the latest from Device Cloud. If True, the device will not make a request if it already has cached data. :raises devicecloud.DeviceCloudHttpException: in the case of an unexpected http error :raises devicecloud.streams.NoSuchStreamException: if this stream has not yet been created @@ -716,7 +754,7 @@ def get_current_value(self, use_cached=False): The current value is the last recorded data point for this stream. - :param bool use_cached: If False, the function will always request the latest from the device cloud. + :param bool use_cached: If False, the function will always request the latest from Device Cloud. If True, the device will not make a request if it already has cached data. :raises devicecloud.DeviceCloudHttpException: in the case of an unexpected http error :raises devicecloud.streams.NoSuchStreamException: if this stream has not yet been created @@ -731,7 +769,7 @@ def get_current_value(self, use_cached=False): return None def delete(self): - """Delete this stream from the device cloud along with its history + """Delete this stream from Device Cloud along with its history This call will return None on success and raise an exception in the event of an error performing the deletion. @@ -836,7 +874,7 @@ def write(self, datapoint): Values already set on the datapoint will not be overridden (except for path) - :param DataPoint datapoint: The :class:`.DataPoint` that should be written to the device cloud + :param DataPoint datapoint: The :class:`.DataPoint` that should be written to Device Cloud """ if not isinstance(datapoint, DataPoint): @@ -853,7 +891,7 @@ def read(self, start_time=None, end_time=None, use_client_timeline=True, newest_ """Read one or more DataPoints from a stream .. warning:: - The data points from the device cloud is a paged data set. When iterating over the + The data points from Device Cloud is a paged data set. When iterating over the result set there could be delays when we hit the end of a page. If this is undesirable, the caller should collect all results into a data structure first before iterating over the result set. diff --git a/devicecloud/test/__init__.py b/devicecloud/test/__init__.py index 5cb2679..afdd51d 100644 --- a/devicecloud/test/__init__.py +++ b/devicecloud/test/__init__.py @@ -2,5 +2,4 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015-2018 Digi International Inc. diff --git a/devicecloud/test/integration/__init__.py b/devicecloud/test/integration/__init__.py new file mode 100644 index 0000000..14b3231 --- /dev/null +++ b/devicecloud/test/integration/__init__.py @@ -0,0 +1,5 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. diff --git a/devicecloud/test/integration/inttest_monitor_tcp.py b/devicecloud/test/integration/inttest_monitor_tcp.py new file mode 100644 index 0000000..466bd7d --- /dev/null +++ b/devicecloud/test/integration/inttest_monitor_tcp.py @@ -0,0 +1,65 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. +import pprint +import time +from devicecloud.streams import DataPoint + +from devicecloud.test.integration.inttest_utilities import DeviceCloudIntegrationTestCase +import six + + +class StreamsIntegrationTestCase(DeviceCloudIntegrationTestCase): + + def test_event_reception(self): + rx = [] + + def receive_notification(notification): + rx.append(notification) + return True + + topics = ['DataPoint', 'FileData'] + monitor = self._dc.monitor.get_monitor(topics) + if monitor: + monitor.delete() + monitor = self._dc.monitor.create_tcp_monitor(topics) + monitor.add_callback(receive_notification) + + self._dc.filedata.write_file("/~/inttest/monitor_tcp/", "test_file.txt", six.b("Hello, world!"), "text/plain") + self._dc.streams.get_stream("inttest/monitor_tcp").write(DataPoint(10)) + + # Wait for the evenets to come in from the cloud + time.sleep(3) + self._dc.monitor.stop_listeners() + + try: + fd_push_seen = False + dp_push_seen = False + for rec in rx: + msg = rec['Document']['Msg'] + fd = msg.get('FileData', None) + if fd and 'id' in fd: + if (fd['id']['fdName'] == 'test_file.txt' and + fd['id']['fdPath'] == '/db/7603_Digi/inttest/monitor_tcp/'): + fd_push_seen = True + # else: + # print('id not in test_event_reception/fd: {}'.format(rx)) + dp = msg.get('DataPoint') + if dp and 'streamId' in dp: + print('test_event_reception/dp: {}'.format(dp)) + if dp['streamId'] == 'inttest/monitor_tcp': + dp_push_seen = True + # else: + # print('streamId not in test_event_reception/dp: {}'.format(rx)) + self.assertTrue(fd_push_seen) + self.assertTrue(dp_push_seen) + except: + # add some additional debugging information + pprint.pprint(rx) + raise + +if __name__ == '__main__': + import unittest + unittest.main() diff --git a/devicecloud/test/integration/inttest_streams.py b/devicecloud/test/integration/inttest_streams.py new file mode 100644 index 0000000..4a3f30e --- /dev/null +++ b/devicecloud/test/integration/inttest_streams.py @@ -0,0 +1,127 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. + +"""Integration tests for streams functionality + +These tests test that the streams functionality actually works against the device +cloud itself. + +""" +from math import pi +import datetime + +from devicecloud.streams import DataPoint, STREAM_TYPE_INTEGER +from devicecloud.test.integration.inttest_utilities import DeviceCloudIntegrationTestCase, dc_inttest_main +import six + + +class StreamsIntegrationTestCase(DeviceCloudIntegrationTestCase): + + def test_basic_nonbulk_stream_operations(self): + # + # This test verifiest that we can perform a number of simple operations on + # a data stream (non-bulk). The ops are create, write, read, and delete + # + SID = "pythondc-inttest/test_basic_nonbulk_stream_operations" + + # get a test stream reference + test_stream = self._dc.streams.get_stream_if_exists(SID) + + # we want a clean stream to work with. If the stream exists, nuke it + if test_stream is not None: + test_stream.delete() + + test_stream = self._dc.streams.create_stream( + stream_id=SID, + data_type='float', + description='a stream used for testing', + units='some-unit', + ) + + for i in range(5): + test_stream.write(DataPoint( + data=i * pi, + description="This is {} * pi".format(i) + )) + + for i, dp in enumerate(test_stream.read(newest_first=False)): + self.assertAlmostEqual(dp.get_data(), i * pi) + + # now cleanup by deleting the stream + test_stream.delete() + + def test_bulk_write_datapoints_multiple_streams(self): + # + # This test verifies that we can write in bulk a bunch of datapoints to several + # datastreams and read them back. + # + SID_FMT = "pythondc-inttest/test_bulk_write_datapoints_multiple_streams-{}" + datapoints = [] + dt = datetime.datetime.now() + for i in range(300): + datapoints.append(DataPoint( + stream_id=SID_FMT.format(i % 3), + data_type=STREAM_TYPE_INTEGER, + units="meters", + timestamp=dt - datetime.timedelta(seconds=300 - i), + data=i, + )) + + # remove any existing data before starting out + for i in range(3): + s = self._dc.streams.get_stream_if_exists(SID_FMT.format(i % 3)) + if s: + s.delete() + + self._dc.streams.bulk_write_datapoints(datapoints) + + for i in range(3): + stream = self._dc.streams.get_stream(SID_FMT.format(i)) + for j, dp in enumerate(stream.read(newest_first=False)): + self.assertEqual(dp.get_data(), j * 3 + i) + stream.delete() + + def test_bulk_write_datapoints_single_stream(self): + # + # This test verifies that we can write in bulk a bunch of datapoints to a single + # stream and read them back. + # + datapoints = [] + dt = datetime.datetime.now() + for i in range(300): + datapoints.append(DataPoint( + data_type=STREAM_TYPE_INTEGER, + units="meters", + timestamp=dt - datetime.timedelta(seconds=300 - i), + data=i, + )) + + stream = self._dc.streams.get_stream_if_exists("pythondc-inttest/test_bulk_write_datapoints_single_stream") + if stream: + stream.delete() + + stream = self._dc.streams.get_stream("pythondc-inttest/test_bulk_write_datapoints_single_stream") + stream.bulk_write_datapoints(datapoints) + stream_contents_asc = list(stream.read(newest_first=False)) + self.assertEqual(len(stream_contents_asc), 300) + for i, dp in enumerate(stream_contents_asc): + self.assertEqual(dp.get_units(), "meters") + self.assertEqual(dp.get_data_type(), STREAM_TYPE_INTEGER) + self.assertEqual(dp.get_data(), i) + self.assertEqual(dp.get_stream_id(), "pythondc-inttest/test_bulk_write_datapoints_single_stream") + self.assertEqual(dp.get_location(), None) + self.assertEqual(dp.get_description(), "") + self.assertIsInstance(dp.get_server_timestamp(), datetime.datetime) + self.assertIsInstance(dp.get_id(), *six.string_types) + self.assertEqual(dp.get_quality(), 0) + self.assertIsInstance(dp.get_timestamp(), datetime.datetime) + + # Cleanup by deleting the stream + stream.delete() + + +if __name__ == '__main__': + dc_inttest_main() diff --git a/devicecloud/test/integration/inttest_utilities.py b/devicecloud/test/integration/inttest_utilities.py new file mode 100644 index 0000000..4ec10fd --- /dev/null +++ b/devicecloud/test/integration/inttest_utilities.py @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. + +from getpass import getpass +import unittest + +from devicecloud import DeviceCloud +import os +from six.moves import input + + +class DeviceCloudIntegrationTestCase(unittest.TestCase): + + def setUp(self): + if not os.environ.get("RUN_INTEGRATION_TESTS", False): + self.skipTest("Not performing integration tests") + else: + self._username = os.environ.get("DC_USERNAME", None) + self._password = os.environ.get("DC_PASSWORD", None) + self._base_url = os.environ.get("DC_URL", None) # will use default if unspecified + if not self._username or not self._password: + self.fail("DC_USERNAME and DC_PASSWORD must be set for integration tests to run") + self._dc = DeviceCloud(self._username, self._password, base_url=self._base_url) + + +def dc_inttest_main(): + """Helper method for kicking off integration tests in a module + + This is used in the same way that one might use 'unittest.main()' in a normal + function. + """ + os.environ["RUN_INTEGRATION_TESTS"] = "yes" + if not os.environ.get("DC_USERNAME"): + os.environ["DC_USERNAME"] = input("username: ") + if not os.environ.get("DC_PASSWORD"): + os.environ["DC_PASSWORD"] = getpass("password: ") + unittest.main() diff --git a/devicecloud/test/test_devicecore.py b/devicecloud/test/test_devicecore.py deleted file mode 100644 index 97c1415..0000000 --- a/devicecloud/test/test_devicecore.py +++ /dev/null @@ -1,154 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. -import copy -import datetime -import unittest - -from dateutil.tz import tzutc -from devicecloud.test.test_utilities import HttpTestBase - - -EXAMPLE_GET_DEVICES = { - "resultTotalRows": "2", - "requestedStartRow": "0", - "resultSize": "2", - "requestedSize": "1000", - "remainingSize": "0", - "items": [ - { - "id": { - "devId": "702077", - "devVersion": "6" - }, - "devRecordStartDate": "2013-02-28T19:54:00.000Z", - "devMac": "00:40:9D:58:17:5B", - "devCellularModemId": "354374042391400", - "devConnectwareId": "00000000-00000000-00409DFF-FF58175B", - "cstId": "1872", - "grpId": "2331", - "devEffectiveStartDate": "2013-02-28T19:53:00.000Z", - "devTerminated": "false", - "dvVendorId": "4261412864", - "dpDeviceType": "ConnectPort X5 R", - "dpFirmwareLevel": "34537482", - "dpFirmwareLevelDesc": "2.15.0.10", - "dpRestrictedStatus": "0", - "dpLastKnownIp": "10.35.1.107", - "dpGlobalIp": "204.182.3.237", - "dpConnectionStatus": "0", - "dpLastConnectTime": "2013-04-08T04:01:20.633Z", - "dpContact": "", - "dpDescription": "", - "dpLocation": "", - "dpMapLat": "34.964465", - "dpMapLong": "40.268198", - "dpServerId": "", - "dpZigbeeCapabilities": "0", - "dpCapabilities": "6707", - "grpPath": "", - "dpLastDisconnectTime": "2013-04-16T19:46:06.557Z" - }, - { - "id": { - "devId": "714038", - "devVersion": "7" - }, - "devRecordStartDate": "2013-07-16T18:05:00.000Z", - "devMac": "00:1d:09:2b:7d:8c", - "devConnectwareId": "00000000-00000000-001D09FF-FF2B7D8C", - "cstId": "1872", - "grpId": "2331", - "devEffectiveStartDate": "2013-07-16T18:05:00.000Z", - "devTerminated": "false", - "dvVendorId": "50331982", - "dpDeviceType": "IntelligentSystem", - "dpFirmwareLevel": "0", - "dpRestrictedStatus": "0", - "dpLastKnownIp": "10.35.1.113", - "dpGlobalIp": "204.182.3.238", - "dpConnectionStatus": "0", - "dpLastConnectTime": "2013-07-24T00:40:20.363Z", - "dpServerId": "", - "dpCapabilities": "66114", - "grpPath": "", - "dpLastDisconnectTime": "2013-07-24T00:40:36.537Z" - } - ] -} - - -class TestDeviceCore(HttpTestBase): - def _get_device(self, mac): - devices = self.dc.devicecore.list_devices() - self.assertEqual(len(devices), 2) - - # get a ref to device with mac "00:40:9D:58:17:5B" - for device in devices: - if device.get_mac() == mac: - break - else: - self.fail("No device with expected MAC address") - - return device - - def test_dc_get_devices(self): - self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - devices = self.dc.devicecore.list_devices() - self.assertEqual(len(devices), 2) - - self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - dev1 = self._get_device("00:40:9D:58:17:5B") - - self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - dev2 = self._get_device("00:1d:09:2b:7d:8c") - - self.assertEqual(dev1.get_mac(), "00:40:9D:58:17:5B") - self.assertEqual(dev1.get_mac_last4(), "175B") - self.assertEqual(dev1.get_device_id(), "702077") - self.assertEqual(dev1.get_connectware_id(), "00000000-00000000-00409DFF-FF58175B") - self.assertEqual(dev1.get_ip(), "10.35.1.107") - self.assertEqual(dev1.get_tags(), []) - self.assertEqual(dev1.get_registration_dt(), - datetime.datetime(2013, 2, 28, 19, 54, tzinfo=tzutc())) - self.assertEqual(dev1.get_meid(), '354374042391400') - self.assertEqual(dev1.get_customer_id(), '1872') - self.assertEqual(dev1.get_group_id(), '2331') - self.assertEqual(dev1.get_group_path(), '') - self.assertEqual(dev1.get_vendor_id(), '4261412864') - self.assertEqual(dev1.get_device_type(), 'ConnectPort X5 R') - self.assertEqual(dev1.get_firmware_level(), '34537482') - self.assertEqual(dev1.get_firmware_level_description(), '2.15.0.10') - self.assertEqual(dev1.get_restricted_status(), "0") - self.assertEqual(dev1.get_last_known_ip(), '10.35.1.107') - self.assertEqual(dev1.get_global_ip(), '204.182.3.237') - self.assertEqual(dev1.get_last_connected_dt(), - datetime.datetime(2013, 4, 8, 4, 1, 20, 633, tzinfo=tzutc())) - self.assertEqual(dev1.get_contact(), '') - self.assertEqual(dev1.get_description(), '') - self.assertEqual(dev1.get_location(), '') - self.assertEqual(dev1.get_latlon(), (34.964465, 40.268198)) - self.assertEqual(dev1.get_user_metadata(), None) - self.assertEqual(dev1.get_zb_pan_id(), None) - self.assertEqual(dev1.get_zb_extended_address(), None) - self.assertEqual(dev1.get_server_id(), '') - self.assertEqual(dev1.get_provision_id(), None) - self.assertEqual(dev1.get_current_connect_pw(), None) - - def test_refresh_from_cache(self): - get_devices_update = copy.deepcopy(EXAMPLE_GET_DEVICES) - get_devices_update["items"][0]["dpDeviceType"] = "Turboencabulator" - del get_devices_update["items"][1] # remove the other item... close enough - self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - device = self._get_device("00:40:9D:58:17:5B") - self.prepare_json_response("GET", "/ws/DeviceCore/702077", get_devices_update) - self.assertEqual(device.get_device_type(), "ConnectPort X5 R") - self.assertEqual(device.get_device_type(False), "Turboencabulator") - self.assertEqual(device.get_device_type(), "Turboencabulator") # make sure cache updated - - -if __name__ == '__main__': - unittest.main() diff --git a/devicecloud/test/test_sci.py b/devicecloud/test/test_sci.py deleted file mode 100644 index 8099152..0000000 --- a/devicecloud/test/test_sci.py +++ /dev/null @@ -1,44 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. - -import unittest - -from devicecloud.sci import DeviceTarget -from devicecloud.test.test_utilities import HttpTestBase -import httpretty -import six - - -EXAMPLE_SCI_DEVICE_NOT_CONNECTED = """\ -Device Not Connected -""" - - -class TestSCI(HttpTestBase): - def _prepare_sci_response(self, response, status=200): - self.prepare_response("POST", "/ws/sci", response, status) - - def test_sci_successful_error(self): - self._prepare_sci_response(EXAMPLE_SCI_DEVICE_NOT_CONNECTED) - self.dc.get_sci_api().send_sci( - operation="send_message", - target=DeviceTarget("00000000-00000000-00409DFF-FF58175B"), - payload="") - self.assertEqual(httpretty.last_request().body, - six.b('' - '' - '' - '' - '' - '' - '' - '')) - - - -if __name__ == "__main__": - unittest.main() diff --git a/devicecloud/test/unit/__init__.py b/devicecloud/test/unit/__init__.py new file mode 100644 index 0000000..14b3231 --- /dev/null +++ b/devicecloud/test/unit/__init__.py @@ -0,0 +1,5 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. diff --git a/devicecloud/test/test_conditions.py b/devicecloud/test/unit/test_conditions.py similarity index 67% rename from devicecloud/test/test_conditions.py rename to devicecloud/test/unit/test_conditions.py index d36df29..3f1afff 100644 --- a/devicecloud/test/test_conditions.py +++ b/devicecloud/test/unit/test_conditions.py @@ -1,3 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. + import unittest import datetime @@ -16,11 +22,11 @@ def test_lt(self): def test_eq(self): a = Attribute("a") - self.assertEqual((a == "a string").compile(), "a='a%20string'") + self.assertEqual((a == "a string").compile(), "a='a string'") def test_like(self): a = Attribute("a") - self.assertEqual(a.like(r"%.txt").compile(), "a like '%25.txt'") + self.assertEqual(a.like(r"%.txt").compile(), "a like '%.txt'") def test_and(self): a = Attribute("a") @@ -32,17 +38,18 @@ def test_or(self): a = Attribute("a") b = Attribute("b") expr = (a.like("%.csv")) | (b < 1024) - self.assertEqual(expr.compile(), "a like '%25.csv' or b<'1024'") + self.assertEqual(expr.compile(), "a like '%.csv' or b<'1024'") def test_datacmp(self): a = Attribute("a") self.assertEqual((a < datetime.datetime(2014, 7, 7)).compile(), - "a<'2014-07-07T00%3A00%3A00Z'") + "a<'2014-07-07T00:00:00Z'") def test_multi_combination(self): a = Attribute("a") self.assertEqual(((a > 1) & (a > 2) & (a > 3)).compile(), "a>'1' and a>'2' and a>'3'") + if __name__ == '__main__': unittest.main() diff --git a/devicecloud/test/unit/test_core.py b/devicecloud/test/unit/test_core.py new file mode 100644 index 0000000..aa13f33 --- /dev/null +++ b/devicecloud/test/unit/test_core.py @@ -0,0 +1,109 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. +import unittest + +from devicecloud import DeviceCloudHttpException +from devicecloud.test.unit.test_utilities import HttpTestBase +from mock import patch, call +import six + + +TEST_BASIC_RESPONSE = """\ +{ + "resultTotalRows": "2", + "requestedStartRow": "0", + "resultSize": "2", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { "id": 1, "name": "bob" }, + { "id": 2, "name": "tim" } + ] +} +""" + +TEST_PAGED_RESPONSE_PAGE1 = """\ +{ + "resultTotalRows": "2", + "requestedStartRow": "0", + "resultSize": "1", + "requestedSize": "1", + "remainingSize": "1", + "items": [ + { "id": 1, "name": "bob" } + ] +} +""" + +TEST_PAGED_RESPONSE_PAGE2 = """\ +{ + "resultTotalRows": "2", + "requestedStartRow": "1", + "resultSize": "1", + "requestedSize": "1", + "remainingSize": "0", + "items": [ + { "id": 2, "name": "tim" } + ] +} +""" + +TEST_ERROR_RESPONSE = six.b("""\ +\ +Invalid target. Device not found.\ +Invalid SCI request. No valid targets found.\ +""") + + +class TestDeviceCloudConnection(HttpTestBase): + + @patch("time.sleep", return_value=None) + def test_throttle_retries(self, patched_time_sleep): + self.prepare_response("GET", "/test/path", "", status=429) + self.assertRaises(DeviceCloudHttpException, self.dc.get_connection().get, "/test/path", retries=5) + patched_time_sleep.assert_has_calls([ + call(1.5 ** 0), + call(1.5 ** 1), + call(1.5 ** 2), + call(1.5 ** 3), + call(1.5 ** 4), + ]) + + def test_iter_json_with_params(self): + it = self.dc.get_connection().iter_json_pages("/test/path", foo="bar", key="value") + self.prepare_response("GET", "/test/path", TEST_BASIC_RESPONSE) + self.assertEqual(len(list(it)), 2) + self.assertDictEqual(self._get_last_request_params(), { + "start": "0", + "foo": "bar", + "key": "value", + "size": "1000", + }) + + def test_iter_json_pages_paged_noparams(self): + it = self.dc.get_connection().iter_json_pages("/test/path", page_size=1) + self.prepare_response("GET", "/test/path", TEST_PAGED_RESPONSE_PAGE1) + self.assertEqual(six.next(it)["id"], 1) + self.prepare_response("GET", "/test/path", TEST_PAGED_RESPONSE_PAGE2) + self.assertEqual(six.next(it)["id"], 2) + self.assertDictEqual(self._get_last_request_params(), { + "size": "1", + "start": "1" + }) + + def test_http_exception(self): + self.prepare_response("POST", "/test/path", TEST_ERROR_RESPONSE, status=400) + try: + self.dc.get_connection().post("/test/path", "bad data") + except DeviceCloudHttpException as e: + str(e) # ensure this does not stack trace at least + self.assertEqual(e.response.status_code, 400) + self.assertEqual(e.response.content, TEST_ERROR_RESPONSE) + else: + self.fail("DeviceCloudHttpException not raised") + +if __name__ == "__main__": + unittest.main() diff --git a/devicecloud/test/unit/test_devicecore.py b/devicecloud/test/unit/test_devicecore.py new file mode 100644 index 0000000..8ab1fcd --- /dev/null +++ b/devicecloud/test/unit/test_devicecore.py @@ -0,0 +1,545 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. + +import copy +import datetime +import unittest + +from dateutil.tz import tzutc +from devicecloud import DeviceCloudHttpException +from devicecloud.devicecore import dev_mac, group_id +from devicecloud.test.unit.test_utilities import HttpTestBase +import httpretty +from devicecloud.devicecore import ADD_GROUP_TEMPLATE, TAGS_TEMPLATE +import six +import mock +from xml.sax.saxutils import escape + + +EXAMPLE_GET_DEVICES = { + "resultTotalRows": "2", + "requestedStartRow": "0", + "resultSize": "2", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { + "id": { + "devId": "702077", + "devVersion": "6" + }, + "devRecordStartDate": "2013-02-28T19:54:00.000Z", + "devMac": "00:40:9D:58:17:5B", + "devCellularModemId": "354374042391400", + "devConnectwareId": "00000000-00000000-00409DFF-FF58175B", + "cstId": "1872", + "grpId": "2331", + "devEffectiveStartDate": "2013-02-28T19:53:00.000Z", + "devTerminated": "false", + "dvVendorId": "4261412864", + "dpDeviceType": "ConnectPort X5 R", + "dpFirmwareLevel": "34537482", + "dpFirmwareLevelDesc": "2.15.0.10", + "dpRestrictedStatus": "0", + "dpLastKnownIp": "10.35.1.107", + "dpGlobalIp": "204.182.3.237", + "dpConnectionStatus": "0", + "dpLastConnectTime": "2013-04-08T04:01:20.633Z", + "dpContact": "", + "dpDescription": "", + "dpLocation": "", + "dpMapLat": "34.964465", + "dpMapLong": "40.268198", + "dpServerId": "", + "dpZigbeeCapabilities": "0", + "dpCapabilities": "6707", + "grpPath": "", + "dpLastDisconnectTime": "2013-04-16T19:46:06.557Z" + }, + { + "id": { + "devId": "714038", + "devVersion": "7" + }, + "devRecordStartDate": "2013-07-16T18:05:00.000Z", + "devMac": "00:1d:09:2b:7d:8c", + "devConnectwareId": "00000000-00000000-001D09FF-FF2B7D8C", + "cstId": "1872", + "grpId": "2331", + "devEffectiveStartDate": "2013-07-16T18:05:00.000Z", + "devTerminated": "false", + "dvVendorId": "50331982", + "dpDeviceType": "IntelligentSystem", + "dpFirmwareLevel": "0", + "dpRestrictedStatus": "0", + "dpLastKnownIp": "10.35.1.113", + "dpGlobalIp": "204.182.3.238", + "dpConnectionStatus": "0", + "dpLastConnectTime": "2013-07-24T00:40:20.363Z", + "dpServerId": "", + "dpCapabilities": "66114", + "grpPath": "", + "dpLastDisconnectTime": "2013-07-24T00:40:36.537Z" + } + ] +} + + +GET_DEVICES_PAGE1 = """\ +{ + "resultTotalRows": "2", + "requestedStartRow": "0", + "resultSize": "1", + "requestedSize": "1", + "remainingSize": "1", + "items": [ + {"id": {"devId": "702077","devVersion": "6"},"devRecordStartDate": "2013-02-28T19:54:00.000Z","devMac": "00:40:9D:58:17:5B","devCellularModemId": "354374042391400","devConnectwareId": "00000000-00000000-00409DFF-FF58175B","cstId": "1872","grpId": "2331","devEffectiveStartDate": "2013-02-28T19:53:00.000Z","devTerminated": "false","dvVendorId": "4261412864","dpDeviceType": "ConnectPort X5 R","dpFirmwareLevel": "34537482","dpFirmwareLevelDesc": "2.15.0.10","dpRestrictedStatus": "0","dpLastKnownIp": "10.35.1.107","dpGlobalIp": "204.182.3.237","dpConnectionStatus": "0","dpLastConnectTime": "2013-04-08T04:01:20.633Z","dpContact": "","dpDescription": "","dpLocation": "","dpMapLat": "34.964465","dpMapLong": "40.268198","dpServerId": "","dpZigbeeCapabilities": "0","dpCapabilities": "6707","grpPath": "","dpLastDisconnectTime": "2013-04-16T19:46:06.557Z"} + ] + } +""" + +GET_DEVICES_PAGE2 = """\ +{ + "resultTotalRows": "2", + "requestedStartRow": "1", + "resultSize": "1", + "requestedSize": "1", + "remainingSize": "0", + "items": [ + {"id": {"devId": "702078","devVersion": "6"},"devRecordStartDate": "2013-02-28T19:54:00.000Z","devMac": "00:40:9D:58:17:5B","devCellularModemId": "354374042391400","devConnectwareId": "00000000-00000000-00409DFF-FF58175B","cstId": "1872","grpId": "2331","devEffectiveStartDate": "2013-02-28T19:53:00.000Z","devTerminated": "false","dvVendorId": "4261412864","dpDeviceType": "ConnectPort X5 R","dpFirmwareLevel": "34537482","dpFirmwareLevelDesc": "2.15.0.10","dpRestrictedStatus": "0","dpLastKnownIp": "10.35.1.107","dpGlobalIp": "204.182.3.237","dpConnectionStatus": "0","dpLastConnectTime": "2013-04-08T04:01:20.633Z","dpContact": "","dpDescription": "","dpLocation": "","dpMapLat": "34.964465","dpMapLong": "40.268198","dpServerId": "","dpZigbeeCapabilities": "0","dpCapabilities": "6707","grpPath": "","dpLastDisconnectTime": "2013-04-16T19:46:06.557Z"} + ] + } +""" + +EXAMPLE_GET_GROUPS = """\ +{ + "resultTotalRows": "2", + "requestedStartRow": "0", + "resultSize": "2", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { "grpId": "11817", "grpName": "7603_Digi", "grpDescription": "7603_Digi root group", "grpPath": "\/7603_Digi\/", "grpParentId": "1"}, + { "grpId": "13542", "grpName": "Demo", "grpPath": "\/7603_Digi\/Demo\/", "grpParentId": "11817"} + ] +} +""" + +EXAMPLE_GET_GROUPS_EXTENDED = """\ +{ + "resultTotalRows": "4", + "requestedStartRow": "0", + "resultSize": "4", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { "grpId": "11817", "grpName": "7603_Digi", "grpDescription": "7603_Digi root group", "grpPath": "\/7603_Digi\/", "grpParentId": "1"}, + { "grpId": "13542", "grpName": "Demo", "grpPath": "\/7603_Digi\/Demo\/", "grpParentId": "11817"}, + { "grpId": "13544", "grpName": "SubDir2", "grpPath": "\/7603_Digi\/Demo\/SubDir2\/", "grpParentId": "13542"}, + { "grpId": "13545", "grpName": "Another Second Level", "grpDescription": "Another Second Level", "grpPath": "\/7603_Digi\/Another Second Level\/", "grpParentId": "11817"} + ] +} +""" + +PROVISION_SUCCESS_RESPONSE1 = """\ + + + DeviceCore/946246/0 + +""" + +PROVISION_MULTIPLE_SUCCESS_RESPONSE1 = """\ + + + DeviceCore/1397876/0 + DeviceCore/946246/0 + +""" + +PROVISION_ERROR1 = """\ + + + The device 00000000-00000000-BC5FF4FF-FFF7908A is already provisioned. + +""" + +PROVISION_MIXED_RESULT_RESPONSE = """\ + + + DeviceCore/1397876/0 + The device 00000000-00000000-D48564FF-FF9D4FEE is already provisioned. + +""" + + +class TestDeviceCoreGroups(HttpTestBase): + + def test_get_groups(self): + self.prepare_response("GET", "/ws/Group", EXAMPLE_GET_GROUPS) + it = self.dc.devicecore.get_groups() + + grp = six.next(it) + self.assertEqual(grp.is_root(), True) + self.assertEqual(grp.get_id(), "11817") + self.assertEqual(grp.get_name(), "7603_Digi") + self.assertEqual(grp.get_description(), "7603_Digi root group") + self.assertEqual(grp.get_path(), "/7603_Digi/") + self.assertEqual(grp.get_parent_id(), "1") + + grp = six.next(it) + self.assertEqual(grp.is_root(), False) + self.assertEqual(grp.get_id(), "13542") + self.assertEqual(grp.get_name(), "Demo") + self.assertEqual(grp.get_description(), "") + self.assertEqual(grp.get_path(), "/7603_Digi/Demo/") + self.assertEqual(grp.get_parent_id(), "11817") + + def test_repr_and_tree_print(self): + self.prepare_response("GET", "/ws/Group", EXAMPLE_GET_GROUPS_EXTENDED) + fobj = six.StringIO() + root = self.dc.devicecore.get_group_tree_root() + root.print_subtree(fobj) # the order of the traversal can vary, so just assert on the length + if six.PY2: + self.assertEqual(len(fobj.getvalue()), 489) + elif six.PY3: + self.assertEqual(len(fobj.getvalue()), 471) # no u'' on repr for strings + + def test_get_groups_condition(self): + self.prepare_response("GET", "/ws/Group", EXAMPLE_GET_GROUPS) + list(self.dc.devicecore.get_groups(group_id == "123")) + params = self._get_last_request_params() + self.assertEqual(params["condition"], "grpId='123'") + + +class TestDeviceCoreProvisioning(HttpTestBase): + + def test_provision_one_simple_device_id(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_SUCCESS_RESPONSE1, status=207) + res = self.dc.devicecore.provision_device(device_id='00000000-00000000-0000DEFF-FFADBEEFF') + req = self._get_last_request() + self.assertEqual(req.body, six.b( + "" + "" + "00000000-00000000-0000DEFF-FFADBEEFF" + "" + "")) + self.assertDictEqual(res, {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) + + def test_provision_one_simple_mac(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_SUCCESS_RESPONSE1, status=207) + res = self.dc.devicecore.provision_device(mac_address="DE:AD:BE:EF:00:00") + req = self._get_last_request() + self.assertEqual(req.body, six.b( + "" + "" + "DE:AD:BE:EF:00:00" + "" + "")) + self.assertDictEqual(res, {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) + + def test_provision_imei(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_SUCCESS_RESPONSE1, status=207) + res = self.dc.devicecore.provision_device(imei="990000862471854") + req = self._get_last_request() + self.assertEqual(req.body, six.b( + "" + "" + "990000862471854" + "" + "")) + self.assertDictEqual(res, {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) + + def test_provision_all_the_fixins(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_SUCCESS_RESPONSE1, status=207) + res = self.dc.devicecore.provision_device( + mac_address="DE:AD:BE:EF:00:00", + group_path="/group/path", + metadata="Sweet, sweet metadata", + map_lat=44.9807496, + map_long=-93.1397815, + contact="Saint Paul Parks Department", + description="Buried Treasure", + ) + req = self._get_last_request() + self.assertEqual(req.body, six.b( + '' + '' + 'DE:AD:BE:EF:00:00' + '/group/path' + 'Sweet, sweet metadata' + '-93.1397815' + '44.9807496' + 'Saint Paul Parks Department' + 'Buried Treasure' + '' + '')) + self.assertDictEqual(res, {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) + + def test_provision_multiple_simple(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_MULTIPLE_SUCCESS_RESPONSE1, status=207) + res = self.dc.devicecore.provision_devices([ + {'device_id': "00000000-00000000-0000DEFF-FFADBEEFF"}, + {'mac_address': 'DE:AD:BE:EF:00:00'} + ]) + req = self._get_last_request() + self.assertEqual(req.body, six.b( + '' + '' + '00000000-00000000-0000DEFF-FFADBEEFF' + '' + '' + '' + 'DE:AD:BE:EF:00:00' + '' + '')) + self.assertTrue(len(res), 2) + self.assertDictEqual(res[0], {"error": False, "error_msg": None, "location": "DeviceCore/1397876/0"}) + self.assertDictEqual(res[1], {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) + + def test_without_required_param(self): + self.assertRaises(ValueError, self.dc.devicecore.provision_device, description="I should not work") + + def test_bad_request_400(self): + self.prepare_response('POST', '/ws/DeviceCore', 'Bad Request', status=400) + self.assertRaises(DeviceCloudHttpException, + self.dc.devicecore.provision_device, mac_address="DE:AD:BE:EF:00:00") + + def test_bad_request_500(self): + self.prepare_response('POST', '/ws/DeviceCore', 'Bad Request', status=500) + self.assertRaises(DeviceCloudHttpException, + self.dc.devicecore.provision_device, mac_address="DE:AD:BE:EF:00:00") + + def test_error_response(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_ERROR1, status=207) + res = self.dc.devicecore.provision_device(imei="990000862471854") + self.assertDictEqual(res, { + "error": True, + "error_msg": 'The device 00000000-00000000-BC5FF4FF-FFF7908A is already provisioned.', + "location": None} + ) + + def test_mixed_error_success_response(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_MIXED_RESULT_RESPONSE, status=207) + res = self.dc.devicecore.provision_devices([ + {'device_id': "00000000-00000000-0000DEFF-FFADBEEFF"}, + {'mac_address': 'DE:AD:BE:EF:00:00'} + ]) + self.assertTrue(len(res), 2) + self.assertDictEqual(res[0], { + 'error': False, + 'error_msg': None, + 'location': 'DeviceCore/1397876/0', + }) + self.assertDictEqual(res[1], { + 'error': True, + 'error_msg': 'The device 00000000-00000000-D48564FF-FF9D4FEE is already provisioned.', + 'location': None, + }) + + +class TestDeviceCoreDeleting(HttpTestBase): + + def test_delete_device_good(self): + fake_device = mock.MagicMock() + fake_device.get_device_id.return_value = '1234' + self.prepare_response("DELETE", "/ws/DeviceCore/1234", "1 items deleted", status=200) + self.dc.devicecore.delete_device(fake_device) + req = self._get_last_request() + self.assertEqual(req.path, "/ws/DeviceCore/1234") + + def test_delete_device_not_exist(self): + fake_device = mock.MagicMock() + fake_device.get_device_id.return_value = '1234' + self.prepare_response("DELETE", "/ws/DeviceCore/1234", "0 items deleted", status=200) + self.dc.devicecore.delete_device(fake_device) + req = self._get_last_request() + self.assertEqual(req.path, "/ws/DeviceCore/1234") + + def test_delete_device_bad_status(self): + fake_device = mock.MagicMock() + fake_device.get_device_id.return_value = '1234' + self.prepare_response("DELETE", "/ws/DeviceCore/1234", "I pity da foo' who don' know about API changes.", status=400) + try: + self.dc.devicecore.delete_device(fake_device) + except DeviceCloudHttpException: + pass + else: + assert False, "should have thrown exception" + req = self._get_last_request() + self.assertEqual(req.path, "/ws/DeviceCore/1234") + + +class TestDeviceCoreDevices(HttpTestBase): + + def test_dc_get_devices(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + devices = self.dc.devicecore.get_devices() + dev1 = six.next(devices) + dev2 = six.next(devices) + self.assertRaises(StopIteration, six.next, devices) + + self.assertEqual(dev1.get_mac(), "00:40:9D:58:17:5B") + self.assertEqual(dev1.get_mac_last4(), "175B") + self.assertEqual(dev1.get_device_id(), "702077") + self.assertEqual(dev1.get_connectware_id(), "00000000-00000000-00409DFF-FF58175B") + self.assertEqual(dev1.get_ip(), "10.35.1.107") + self.assertEqual(dev1.get_tags(), []) + self.assertEqual(dev1.get_registration_dt(), + datetime.datetime(2013, 2, 28, 19, 54, tzinfo=tzutc())) + self.assertEqual(dev1.get_meid(), '354374042391400') + self.assertEqual(dev1.get_customer_id(), '1872') + self.assertEqual(dev1.get_group_id(), '2331') + self.assertEqual(dev1.get_group_path(), '') + self.assertEqual(dev1.get_vendor_id(), '4261412864') + self.assertEqual(dev1.get_device_type(), 'ConnectPort X5 R') + self.assertEqual(dev1.get_firmware_level(), '34537482') + self.assertEqual(dev1.get_firmware_level_description(), '2.15.0.10') + self.assertEqual(dev1.get_restricted_status(), "0") + self.assertEqual(dev1.get_last_known_ip(), '10.35.1.107') + self.assertEqual(dev1.get_global_ip(), '204.182.3.237') + self.assertEqual(dev1.get_last_connected_dt(), + datetime.datetime(2013, 4, 8, 4, 1, 20, 633000, tzinfo=tzutc())) + self.assertEqual(dev1.get_contact(), '') + self.assertEqual(dev1.get_description(), '') + self.assertEqual(dev1.get_location(), '') + self.assertEqual(dev1.get_latlon(), (34.964465, 40.268198)) + self.assertEqual(dev1.get_user_metadata(), None) + self.assertEqual(dev1.get_zb_pan_id(), None) + self.assertEqual(dev1.get_zb_extended_address(), None) + self.assertEqual(dev1.get_server_id(), '') + self.assertEqual(dev1.get_provision_id(), None) + self.assertEqual(dev1.get_current_connect_pw(), None) + + def test_dc_get_devices_paged(self): + self.prepare_response("GET", "/ws/DeviceCore", GET_DEVICES_PAGE1) + gen = self.dc.devicecore.get_devices(page_size=1) + dev1 = six.next(gen) + self.prepare_response("GET", "/ws/DeviceCore", GET_DEVICES_PAGE2) + dev2 = six.next(gen) + self.assertRaises(StopIteration, six.next, gen) + self.assertEqual(dev1.get_device_id(), '702077') + self.assertEqual(dev2.get_device_id(), '702078') + + def test_dc_get_devices_with_condition(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + gen = self.dc.devicecore.get_devices(dev_mac == 'xx:xx:xx:xx:xx', page_size=1) + six.next(gen) + qs = httpretty.last_request().querystring + self.assertEqual(qs['condition'][0], "devMac='xx:xx:xx:xx:xx'") + self.assertEqual(qs['size'][0], "1") + self.assertEqual(qs['embed'][0], "true") + self.assertEqual(qs['start'][0], "0") + + def test_refresh_from_cache(self): + get_devices_update = copy.deepcopy(EXAMPLE_GET_DEVICES) + get_devices_update["items"][0]["dpDeviceType"] = "Turboencabulator" + del get_devices_update["items"][1] # remove the other item... close enough + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + devices = self.dc.devicecore.get_devices() + device = six.next(devices) + self.prepare_json_response("GET", "/ws/DeviceCore/702077", get_devices_update) + self.assertEqual(device.get_device_type(), "ConnectPort X5 R") + self.assertEqual(device.get_device_type(False), "Turboencabulator") + self.assertEqual(device.get_device_type(), "Turboencabulator") # make sure cache updated + + def test_add_device_to_group(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + expected = ADD_GROUP_TEMPLATE.format(connectware_id=dev.get_connectware_id(), + group_path='testgrp') + dev.add_to_group('testgrp') + self.assertIsNone(dev._device_json) + self.assertEqual(six.b(expected), httpretty.last_request().body) + + def test_remove_device_from_group(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + dev.get_group_path = lambda: 'something other than empty string' + expected = ADD_GROUP_TEMPLATE.format(connectware_id=dev.get_connectware_id(), + group_path='') + dev.remove_from_group() + self.assertIsNone(dev._device_json) + self.assertEqual(six.b(expected), httpretty.last_request().body) + + def test_add_device_tag(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + expected = TAGS_TEMPLATE.format(connectware_id=dev.get_connectware_id(), + tags='test') + dev.add_tag('test') + self.assertIsNone(dev._device_json) + self.assertEqual(six.b(expected), httpretty.last_request().body) + + def test_add_multiple_tags(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + expected = TAGS_TEMPLATE.format(connectware_id=dev.get_connectware_id(), + tags='test,test2,test3') + dev.add_tag('test,test2,test3') + self.assertIsNone(dev._device_json) + self.assertEqual(six.b(expected), httpretty.last_request().body) + + def test_add_tag_list(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + tags = ['test', 'test2', 'test3'] + expected = TAGS_TEMPLATE.format(connectware_id=dev.get_connectware_id(), + tags="{}".format(",".join(tags))) + dev.add_tag(tags) + self.assertIsNone(dev._device_json) + self.assertEqual(six.b(expected), httpretty.last_request().body) + + def test_add_tags_with_spaces(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + tags = 'test, test2, test3, compound tag' + clean_tags = [t.strip() for t in tags.split(',')] + expected = TAGS_TEMPLATE.format(connectware_id=dev.get_connectware_id(), + tags="{}".format(",".join(clean_tags))) + dev.add_tag(tags) + self.assertIsNone(dev._device_json) + self.assertEqual(six.b(expected), httpretty.last_request().body) + + def test_add_tags_with_special_chars(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + tags = 'test, test2, test3, this & that, < more >' + clean_tags = [t.strip() for t in tags.split(',')] + expected = TAGS_TEMPLATE.format(connectware_id=dev.get_connectware_id(), + tags=escape("{}".format(",".join(clean_tags)))) + dev.add_tag(tags) + self.assertIsNone(dev._device_json) + self.assertEqual(six.b(expected), httpretty.last_request().body) + + def test_remove_device_tag(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + try: + dev.remove_tag('test') + except ValueError: + pass + else: + assert False, "should have thrown exception" + + +if __name__ == '__main__': + unittest.main() diff --git a/devicecloud/test/unit/test_file_system_service.py b/devicecloud/test/unit/test_file_system_service.py new file mode 100644 index 0000000..a4a7f5a --- /dev/null +++ b/devicecloud/test/unit/test_file_system_service.py @@ -0,0 +1,859 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. + +import base64 +import unittest +from xml.etree import ElementTree as ET + +from devicecloud.file_system_service import FileInfo, DirectoryInfo, FileSystemServiceException, \ + _parse_command_response, ResponseParseError, \ + ErrorInfo, LsInfo, _parse_error_tree, LsCommand, GetCommand, PutCommand, DeleteCommand, \ + FileSystemServiceCommandBlock +from devicecloud.sci import AllTarget +from devicecloud.test.unit.test_utilities import HttpTestBase +import mock +import six + + +class FileSystemResponse(object): + def __init__(self): + self.string_start = "" + self.end_string = "" + self.block = "" + + def add_device_block(self, dev_id, command_block): + block = "{command_block}" + self.block += block.format(dev_id=dev_id, command_block=command_block) + + def get_string(self): + return self.string_start + self.block + self.end_string + + @property + def text(self): + return self.string_start + self.block + self.end_string + + +LS_BLOCK = """\ + + + + + +""" + +ERROR_BLOCK = """\ +<{command}> +{errtext} + +""" + +GET_FILE_BLOCK = """\ + +{data} + +""" + +GENERIC_COMMAND_BLOCK = """<{command}>""" + +PUT_FILE_DATA_COMMAND = """\ +\ +\ +{data}\ +\ +\ +""" + +PUT_FILE_FILE_COMMAND = """\ +\ +\ +{server_file}\ +\ +\ +""" + +DELETE_FILE_COMMAND = """\ +\ +\ +\ +""" + + +class TestFileInfo(unittest.TestCase): + def setUp(self): + self.fss_api = mock.Mock() + self.dev_id = '00000000-00000000-18A905FF-FF2F1BBD' + + def test_eq_not_eq(self): + file1 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + file2 = FileInfo(self.fss_api, self.dev_id, '/a/path/file2.py', 1434377919, 181, + "DEA17715739E46079C1A6DDCB38344DF", 'md5') + self.assertNotEqual(file1, file2) + + def test_eq(self): + file1 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + file2 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + self.assertEqual(file1, file2) + self.assertFalse(file1 is file2) + + def test_get_data(self): + self.fss_api.get_file.side_effect = ({self.dev_id: "some file data"},) + file1 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + self.assertEqual("some file data", file1.get_data()) + self.assertEqual(1, self.fss_api.get_file.call_count) + call_name, call_args, call_kwargs = self.fss_api.get_file.mock_calls[0] + self.assertEqual(call_args[0]._device_id, self.dev_id) + self.assertEqual(call_args[1], '/a/path/file1.txt') + + def test_get_data_error(self): + error = ErrorInfo(1, "error message") + self.fss_api.get_file.side_effect = ({self.dev_id: error},) + file1 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + self.assertEqual(error, file1.get_data()) + self.assertEqual(1, self.fss_api.get_file.call_count) + call_name, call_args, call_kwargs = self.fss_api.get_file.mock_calls[0] + self.assertEqual(call_args[0]._device_id, self.dev_id) + self.assertEqual(call_args[1], '/a/path/file1.txt') + + def test_delete(self): + self.fss_api.delete_file.side_effect = ({self.dev_id: None},) + file1 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + self.assertIsNone(file1.delete()) + self.assertEqual(1, self.fss_api.delete_file.call_count) + call_name, call_args, call_kwargs = self.fss_api.delete_file.mock_calls[0] + self.assertEqual(call_args[0]._device_id, self.dev_id) + self.assertEqual(call_args[1], '/a/path/file1.txt') + + def test_delete_error(self): + error = ErrorInfo(1, "error message") + self.fss_api.delete_file.side_effect = ({self.dev_id: error},) + file1 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + ret_err = file1.delete() + self.assertEqual(error.errno, ret_err.errno) + self.assertEqual(error.message, ret_err.message) + self.assertEqual(1, self.fss_api.delete_file.call_count) + call_name, call_args, call_kwargs = self.fss_api.delete_file.mock_calls[0] + self.assertEqual(call_args[0]._device_id, self.dev_id) + self.assertEqual(call_args[1], '/a/path/file1.txt') + + +class TestDirectoryInfo(unittest.TestCase): + def setUp(self): + self.fss_api = mock.Mock() + self.dev_id = '00000000-00000000-18A905FF-FF2F1BBD' + + def test_eq_not_eq(self): + dir1 = DirectoryInfo(self.fss_api, self.dev_id, '/a/path/dir1', 1436276773) + dir2 = DirectoryInfo(self.fss_api, self.dev_id, '/a/path/dir2', 1434377919) + self.assertNotEqual(dir1, dir2) + + def test_eq(self): + dir1 = DirectoryInfo(self.fss_api, self.dev_id, '/a/path/dir1', 1436276773) + dir2 = DirectoryInfo(self.fss_api, self.dev_id, '/a/path/dir1', 1436276773) + self.assertEqual(dir1, dir2) + self.assertFalse(dir1 is dir2) + + def test_list_contents(self): + file1 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + file2 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + self.fss_api.list_files.side_effect = ({self.dev_id: LsInfo([], [file1, file2])},) + dir1 = DirectoryInfo(self.fss_api, self.dev_id, '/a/path/dir1', 1436276773) + dirs, files = dir1.list_contents() + self.assertEqual(0, len(dirs)) + self.assertListEqual([file1, file2], files) + self.assertEqual(1, self.fss_api.list_files.call_count) + call_name, call_args, call_kwargs = self.fss_api.list_files.mock_calls[0] + self.assertEqual(call_args[0]._device_id, self.dev_id) + self.assertEqual(call_args[1], '/a/path/dir1') + + +class TestLsCommand(unittest.TestCase): + def test_init(self): + lscommand = LsCommand(path='/a/path/here') + et = lscommand.get_etree() + self.assertEqual(et.tag, 'ls') + self.assertEqual('/a/path/here', et.get('path')) + self.assertEqual('any', et.get('hash')) + self.assertEqual(0, len(list(et))) + self.assertEqual(None, et.text) + + def test_parse(self): + fss_api = mock.Mock() + dev_id = 'my_dev_id' + file1 = FileInfo(fss_api, dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + file2 = FileInfo(fss_api, dev_id, '/a/path/file2.py', 1434377919, 181, + "DEA17715739E46079C1A6DDCB38344DF", 'md5') + dir1 = DirectoryInfo(fss_api, dev_id, '/a/path/dir', 1436203917) + linfo = LsCommand.parse_response(ET.fromstring(LS_BLOCK), device_id=dev_id, fssapi=fss_api) + self.assertEqual(linfo, LsInfo(directories=[dir1], files=[file1, file2])) + + def test_parse_error(self): + fss_api = mock.Mock() + dev_id = 'my_dev_id' + errinfo = LsCommand.parse_response( + ET.fromstring(ERROR_BLOCK.format(command='ls', errno=1, errtext="error text")), + device_id=dev_id, fssapi=fss_api) + + self.assertEqual(errinfo.errno, 1) + self.assertEqual(errinfo.message, "error text") + + def test_parse_wrong_response(self): + self.assertRaises(ResponseParseError, LsCommand.parse_response, ET.fromstring('')) + + +class TestGetCommand(unittest.TestCase): + def test_init_defaults(self): + getcommand = GetCommand(path='/a/path/here') + et = getcommand.get_etree() + self.assertEqual(et.tag, 'get_file') + self.assertEqual('/a/path/here', et.get('path')) + self.assertEqual(None, et.get('offset', None)) + self.assertEqual(None, et.get('length', None)) + self.assertEqual(0, len(list(et))) + self.assertEqual(None, et.text) + + def test_init_values(self): + getcommand = GetCommand(path='/a/path/here', offset=5, length=10) + et = getcommand.get_etree() + self.assertEqual(et.tag, 'get_file') + self.assertEqual('/a/path/here', et.get('path')) + self.assertEqual('5', et.get('offset', None)) + self.assertEqual('10', et.get('length', None)) + self.assertEqual(0, len(list(et))) + self.assertEqual(None, et.text) + + def test_parse_empty(self): + data = GetCommand.parse_response(ET.fromstring(GET_FILE_BLOCK.format(data=''))) + self.assertEqual(six.b(""), data) + + def test_parse(self): + data_str = base64.b64encode(six.b("File Data")).decode('ascii') + data = GetCommand.parse_response(ET.fromstring(GET_FILE_BLOCK.format(data=data_str))) + self.assertEqual(six.b("File Data"), data) + + def test_parse_error(self): + errinfo = GetCommand.parse_response( + ET.fromstring(ERROR_BLOCK.format(command='get_file', errno=1, errtext="error text"))) + + self.assertEqual(errinfo.errno, 1) + self.assertEqual(errinfo.message, "error text") + + def test_parse_wrong_response(self): + self.assertRaises(ResponseParseError, GetCommand.parse_response, ET.fromstring('')) + + +class TestPutCommand(unittest.TestCase): + def test_init_defaults(self): + putcommand = PutCommand(path='/a/path/here', file_data=six.b("some file data")) + et = putcommand.get_etree() + self.assertEqual(et.tag, 'put_file') + self.assertEqual('/a/path/here', et.get('path')) + self.assertEqual(None, et.get('offset', None)) + self.assertEqual('false', et.get('truncate')) + self.assertEqual(1, len(list(et))) + data = et.find('./data') + self.assertEqual(base64.b64encode(six.b("some file data")), six.b(data.text)) + + def test_init_server_file(self): + putcommand = PutCommand(path='/a/path/here', server_file='/a/file/on/server') + et = putcommand.get_etree() + self.assertEqual(et.tag, 'put_file') + self.assertEqual('/a/path/here', et.get('path')) + self.assertEqual(None, et.get('offset', None)) + self.assertEqual('false', et.get('truncate')) + self.assertEqual(1, len(list(et))) + server_file = et.find('./file') + self.assertEqual('/a/file/on/server', server_file.text) + + def test_init_values(self): + putcommand = PutCommand(path='/a/path/here', file_data=six.b("some file data"), offset=5, truncate=True) + et = putcommand.get_etree() + self.assertEqual(et.tag, 'put_file') + self.assertEqual('/a/path/here', et.get('path')) + self.assertEqual('5', et.get('offset', None)) + self.assertEqual('true', et.get('truncate')) + self.assertEqual(1, len(list(et))) + data = et.find('./data') + self.assertEqual(base64.b64encode(six.b("some file data")), six.b(data.text)) + + def test_put_command_no_data(self): + self.assertRaises(FileSystemServiceException, PutCommand, path='/a/path/here') + + def test_put_command_both_data(self): + self.assertRaises(FileSystemServiceException, PutCommand, path='/a/path/here', + file_data=six.b("some file data"), server_file='/a/file/on/server') + + def test_parse(self): + self.assertIsNone(PutCommand.parse_response(ET.fromstring(''))) + + def test_parse_error(self): + errinfo = PutCommand.parse_response( + ET.fromstring(ERROR_BLOCK.format(command='put_file', errno=1, errtext="error text"))) + + self.assertEqual(errinfo.errno, 1) + self.assertEqual(errinfo.message, "error text") + + def test_parse_wrong_response(self): + self.assertRaises(ResponseParseError, PutCommand.parse_response, ET.fromstring('')) + + +class TestDeleteCommand(unittest.TestCase): + def test_init(self): + delcommand = DeleteCommand(path='/a/path/here') + et = delcommand.get_etree() + self.assertEqual(et.tag, 'rm') + self.assertEqual('/a/path/here', et.get('path')) + self.assertEqual(0, len(list(et))) + self.assertEqual(None, et.text) + + def test_parse(self): + self.assertIsNone(DeleteCommand.parse_response(ET.fromstring(''))) + + def test_parse_error(self): + errinfo = DeleteCommand.parse_response( + ET.fromstring(ERROR_BLOCK.format(command='rm', errno=1, errtext="error text"))) + + self.assertEqual(errinfo.errno, 1) + self.assertEqual(errinfo.message, "error text") + + def test_parse_wrong_response(self): + self.assertRaises(ResponseParseError, DeleteCommand.parse_response, ET.fromstring('')) + + +class TestCommandBlock(unittest.TestCase): + def test_init(self): + command_block = FileSystemServiceCommandBlock() + et = command_block.get_etree() + self.assertEqual(et.tag, 'commands') + self.assertEqual(0, len(list(et))) + self.assertEqual(0, len(et.keys())) + + def test_add_command(self): + command_block = FileSystemServiceCommandBlock() + command_block.add_command(DeleteCommand(path='/a/path')) + et = command_block.get_etree() + self.assertEqual(et.tag, 'commands') + self.assertEqual(1, len(list(et))) + self.assertEqual(0, len(et.keys())) + self.assertIsNotNone(et.find('./{}'.format(DeleteCommand.command_name))) + + def test_get_command_string(self): + command_block = FileSystemServiceCommandBlock() + self.assertEqual(six.b(''), command_block.get_command_string()) + command_block.add_command(DeleteCommand(path='/a/path')) + self.assertEqual(six.b(''), command_block.get_command_string()) + + +class TestFileSystemServiceAPI(HttpTestBase): + def setUp(self): + HttpTestBase.setUp(self) + self.fss_api = self.dc.get_fss_api() + self.sci_api = mock.Mock() + self.fss_api._sci_api = self.sci_api + self.target = AllTarget() + self.dev1_id = '00000000-00000000-18A905FF-FF2F1BBD' + self.dev2_id = '00000000-00000000-18A905FF-FF2F1BBE' + + # Create some file, directory, and error info objects to use in tests + self.file1 = FileInfo(self.fss_api, self.dev1_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + self.file2 = FileInfo(self.fss_api, self.dev1_id, '/a/path/file2.py', 1434377919, 181, + "DEA17715739E46079C1A6DDCB38344DF", 'md5') + self.dir1 = DirectoryInfo(self.fss_api, self.dev1_id, '/a/path/dir', 1436203917) + self.errinfo = ErrorInfo(errno=1, message="an error message") + + def clone_file(self, orig_file): + return FileInfo(orig_file._fssapi, orig_file.device_id, orig_file.path, + orig_file.last_modified, orig_file.size, orig_file.hash, orig_file.hash_type) + + def clone_dir(self, orig_dir): + return DirectoryInfo(orig_dir._fssapi, orig_dir.device_id, orig_dir.path, orig_dir.last_modified) + + def prep_sci_response(self, response): + self.sci_api.send_sci.side_effect = (response,) + + def test_parse_command_response_bad_xml_response(self): + fsr = FileSystemResponse() + fsr.add_device_block('asdf', '<> some_garbage_data') + self.assertRaises(ResponseParseError, _parse_command_response, fsr) + + def test_parse_command_response_good_data(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, "") + root = _parse_command_response(fsr) + self.assertIsNotNone(root.find('.//some_command')) + + def test_parse_error_tree_text(self): + command = ET.fromstring(ERROR_BLOCK.format(command='command', errno=1, errtext="some text")) + error = command.find('./error') + errinfo = _parse_error_tree(error) + self.assertEqual(errinfo.errno, 1) + self.assertEqual(errinfo.message, 'some text') + + def test_parse_error_tree_desc_node(self): + command = ET.fromstring(ERROR_BLOCK.format(command='command', errno=1, errtext="some text")) + error = command.find('./error') + errinfo = _parse_error_tree(error) + self.assertEqual(errinfo.errno, 1) + self.assertEqual(errinfo.message, 'some text') + + def test_list_dir(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, LS_BLOCK) + fsr.add_device_block(self.dev2_id, LS_BLOCK) + self.prep_sci_response(fsr) + list_dict = self.fss_api.list_files(self.target, '/a/path/') + + files_dev1 = [self.file1, self.file2] + dirs_dev1 = [self.dir1] + + file1 = self.clone_file(self.file1) + file1.device_id = self.dev2_id + file2 = self.clone_file(self.file2) + file2.device_id = self.dev2_id + + files_dev2 = [file1, file2] + dir1 = self.clone_dir(self.dir1) + dir1.device_id = self.dev2_id + dirs_dev2 = [dir1] + + expected_dict = {self.dev1_id: LsInfo(dirs_dev1, files_dev1), + self.dev2_id: LsInfo(dirs_dev2, files_dev2)} + + self.assertDictEqual(expected_dict, list_dict) + + def test_list_nonexistent_dir(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, + ERROR_BLOCK.format(command="ls", errno=1, errtext="No such file or directory")) + self.prep_sci_response(fsr) + out_dict = self.fss_api.list_files(self.target, '/a/nonexistent/path') + self.assertEqual(1, len(out_dict.keys())) + self.assertTrue(self.dev1_id in out_dict.keys()) + error = out_dict[self.dev1_id] + self.assertEqual(1, error.errno) + self.assertEqual("No such file or directory", error.message) + + def test_get_entire_file(self): + data_string = base64.b64encode(six.b('testing string')).decode('ascii') + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GET_FILE_BLOCK.format(data=data_string)) + fsr.add_device_block(self.dev2_id, GET_FILE_BLOCK.format(data=data_string)) + self.prep_sci_response(fsr) + get_file_data = self.fss_api.get_file(self.target, '/a/path/file1.txt') + expected_dict = { + self.dev1_id: six.b('testing string'), + self.dev2_id: six.b('testing string'), + } + self.assertDictEqual(expected_dict, get_file_data) + + def test_get_partial_file(self): + data_string = base64.b64encode(six.b('ting')).decode('ascii') + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GET_FILE_BLOCK.format(data=data_string)) + fsr.add_device_block(self.dev2_id, GET_FILE_BLOCK.format(data=data_string)) + self.prep_sci_response(fsr) + get_file_data = self.fss_api.get_file(self.target, '/a/path/file1.txt', offset=2, length=4) + expected_dict = { + self.dev1_id: six.b('ting'), + self.dev2_id: six.b('ting'), + } + self.assertDictEqual(expected_dict, get_file_data) + + def test_get_nonexistent_file(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, + ERROR_BLOCK.format(command='get_file', errno=1, errtext="No such file or directory")) + self.prep_sci_response(fsr) + out_dict = self.fss_api.get_file(self.target, '/a/nonexistent/path') + self.assertEqual(1, len(out_dict.keys())) + self.assertTrue(self.dev1_id in out_dict.keys()) + error = out_dict[self.dev1_id] + self.assertEqual(1, error.errno) + self.assertEqual("No such file or directory", error.message) + + def test_get_file_some_error(self): + data_string = base64.b64encode(six.b('testing string')).decode('ascii') + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, + ERROR_BLOCK.format(command='get_file', errno=1, errtext="No such file or directory")) + fsr.add_device_block(self.dev2_id, GET_FILE_BLOCK.format(data=data_string)) + self.prep_sci_response(fsr) + out_dict = self.fss_api.get_file(self.target, '/a/nonexistent/path') + self.assertEqual(2, len(out_dict.keys())) + self.assertTrue(self.dev1_id in out_dict.keys()) + self.assertTrue(self.dev2_id in out_dict.keys()) + + # Verify error info + error = out_dict[self.dev1_id] + self.assertEqual(1, error.errno) + self.assertEqual("No such file or directory", error.message) + + # Verify OK data + self.assertEqual(six.b('testing string'), out_dict[self.dev2_id]) + + def test_put_complete_file(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + fsr.add_device_block(self.dev2_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + out_dict = self.fss_api.put_file(self.target, file_path, file_data=six.b('testing string')) + + expected_dict = { + self.dev1_id: None, + self.dev2_id: None + } + self.assertDictEqual(expected_dict, out_dict) + + self.sci_api.send_sci.assert_called_once_with('file_system', self.target, six.b(PUT_FILE_DATA_COMMAND.format( + path=file_path, + data=base64.b64encode(six.b('testing string')).decode('ascii'), + offset="", + truncate='false'))) + + def test_put_partial_file(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + fsr.add_device_block(self.dev2_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + out_dict = self.fss_api.put_file(self.target, file_path, file_data=six.b('testing string'), offset=5) + expected_dict = { + self.dev1_id: None, + self.dev2_id: None + } + self.assertDictEqual(expected_dict, out_dict) + + self.sci_api.send_sci.assert_called_once_with('file_system', self.target, six.b(PUT_FILE_DATA_COMMAND.format( + path=file_path, + data=base64.b64encode(six.b('testing string')).decode('ascii'), + offset="offset=\"5\" ", + truncate='false'))) + + def test_put_partial_file_truncate(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + fsr.add_device_block(self.dev2_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + out_dict = self.fss_api.put_file(self.target, file_path, file_data=six.b('testing string'), offset=5, + truncate=True) + expected_dict = { + self.dev1_id: None, + self.dev2_id: None + } + self.assertDictEqual(expected_dict, out_dict) + + self.sci_api.send_sci.assert_called_once_with('file_system', self.target, six.b(PUT_FILE_DATA_COMMAND.format( + path=file_path, + data=base64.b64encode(six.b('testing string')).decode('ascii'), + offset="offset=\"5\" ", + truncate='true'))) + + def test_put_file_both_data_args(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + fsr.add_device_block(self.dev2_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + self.assertRaises(FileSystemServiceException, self.fss_api.put_file, self.target, file_path, + file_data=six.b('testing string'), server_file='/a/path/file2.txt') + + def test_put_file_both_neither_args(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + fsr.add_device_block(self.dev2_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + self.assertRaises(FileSystemServiceException, self.fss_api.put_file, self.target, file_path) + + def test_put_file_server_file(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + fsr.add_device_block(self.dev2_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + server_file = '/a/path/file2.txt' + out_dict = self.fss_api.put_file(self.target, file_path, server_file=server_file) + expected_dict = { + self.dev1_id: None, + self.dev2_id: None + } + self.assertDictEqual(expected_dict, out_dict) + + self.sci_api.send_sci.assert_called_once_with('file_system', self.target, six.b(PUT_FILE_FILE_COMMAND.format( + path=file_path, + server_file=server_file, + offset="", + truncate='false'))) + + def test_put_file_some_error(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + fsr.add_device_block(self.dev2_id, + ERROR_BLOCK.format(command='put_file', errno='1', errtext='something went wrong')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + server_file = '/a/path/file2.txt' + out_dict = self.fss_api.put_file(self.target, file_path, server_file=server_file) + self.assertIsNone(out_dict[self.dev1_id]) + error = out_dict[self.dev2_id] + self.assertEqual(1, error.errno) + self.assertEqual('something went wrong', error.message) + + def test_delete_file(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='rm')) + fsr.add_device_block(self.dev2_id, GENERIC_COMMAND_BLOCK.format(command='rm')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + out_dict = self.fss_api.delete_file(self.target, file_path) + expected_dict = { + self.dev1_id: None, + self.dev2_id: None + } + self.assertDictEqual(expected_dict, out_dict) + + self.sci_api.send_sci.assert_called_once_with('file_system', self.target, six.b(DELETE_FILE_COMMAND.format( + path=file_path))) + + def test_delete_file_some_error(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='rm')) + fsr.add_device_block(self.dev2_id, ERROR_BLOCK.format(command='rm', errno='1', errtext='something went wrong')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + out_dict = self.fss_api.delete_file(self.target, file_path) + self.assertIsNone(out_dict[self.dev1_id]) + error = out_dict[self.dev2_id] + self.assertEqual(1, error.errno) + self.assertEqual('something went wrong', error.message) + + def test_get_modified_items(self): + + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[], files=[self.file1, self.file2]) + out_dict = {self.dev1_id: linfo, self.dev2_id: LsInfo(directories=[], files=[])} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.get_modified_items(self.target, '/a/path/', self.file1.last_modified - 1) + + expected_out_dict = { + self.dev1_id: LsInfo(files=[self.file1], directories=[]), + self.dev2_id: LsInfo([], []) + } + + self.assertDictEqual(expected_out_dict, out_dict) + + def test_get_modified_items_errinfo(self): + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[], files=[self.file1, self.file2]) + out_dict = {self.dev1_id: linfo, self.dev2_id: self.errinfo} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.get_modified_items(self.target, '/a/path/', self.file1.last_modified - 1) + + expected_out_dict = { + self.dev1_id: LsInfo(files=[self.file1], directories=[]), + self.dev2_id: self.errinfo + } + + self.assertDictEqual(expected_out_dict, out_dict) + + def test_get_modified_items_no_results(self): + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[], files=[self.file1, self.file2]) + out_dict = {self.dev1_id: linfo, self.dev2_id: linfo} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.get_modified_items(self.target, '/a/path/', self.file1.last_modified + self.file2.last_modified) + + expected_out_dict = { + self.dev1_id: LsInfo([], []), + self.dev2_id: LsInfo([], []) + } + + self.assertDictEqual(expected_out_dict, out_dict) + + def test_get_modified_items_mult_dev_results(self): + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[], files=[self.file1, self.file2]) + out_dict = {self.dev1_id: linfo, self.dev2_id: linfo} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.get_modified_items(self.target, '/a/path/', self.file1.last_modified - 1) + + expected_out_dict = { + self.dev1_id: LsInfo(files=[self.file1], directories=[]), + self.dev2_id: LsInfo(files=[self.file1], directories=[]) + } + + self.assertDictEqual(expected_out_dict, out_dict) + + def test_get_modified_directory_results(self): + dir2 = DirectoryInfo(self.fss_api, self.dev1_id, '/a/path/dir2', self.dir1.last_modified - 2) + + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[self.dir1, dir2], files=[]) + out_dict = {self.dev1_id: linfo, self.dev2_id: LsInfo([], [])} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.get_modified_items(self.target, '/a/path/', self.dir1.last_modified - 1) + + expected_out_dict = { + self.dev1_id: LsInfo(files=[], directories=[self.dir1]), + self.dev2_id: LsInfo([], []) + } + + self.assertDictEqual(expected_out_dict, out_dict) + + def test_exists_file(self): + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[], files=[self.file1]) + out_dict = {self.dev1_id: linfo, self.dev2_id: LsInfo([], [])} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.exists(self.target, self.file1.path) + + expected_out_dict = { + self.dev1_id: True, + self.dev2_id: False + } + + self.assertDictEqual(expected_out_dict, out_dict) + + def test_exists_dir(self): + + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[self.dir1], files=[]) + out_dict = {self.dev1_id: linfo, self.dev2_id: LsInfo([], [])} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.exists(self.target, self.dir1.path) + + expected_out_dict = { + self.dev1_id: True, + self.dev2_id: False + } + + self.assertDictEqual(expected_out_dict, out_dict) + + def test_exists_dir_trailing_slash(self): + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[self.dir1], files=[]) + out_dict = {self.dev1_id: linfo, self.dev2_id: LsInfo([], [])} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.exists(self.target, self.dir1.path + '/') + + expected_out_dict = { + self.dev1_id: True, + self.dev2_id: False + } + + self.assertDictEqual(expected_out_dict, out_dict) + + def test_exists_errinfo(self): + + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[], files=[self.file1]) + out_dict = {self.dev1_id: linfo, self.dev2_id: self.errinfo} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.exists(self.target, self.file1.path) + + expected_out_dict = { + self.dev1_id: True, + self.dev2_id: self.errinfo + } + + self.assertDictEqual(expected_out_dict, out_dict) + + def test_send_command_block_all_ok(self): + command_block = FileSystemServiceCommandBlock() + command_block.add_command(DeleteCommand(path='/a/path/1.txt')) + command_block.add_command(DeleteCommand(path='/a/path/2.txt')) + + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, (GENERIC_COMMAND_BLOCK.format(command='rm') + GENERIC_COMMAND_BLOCK.format(command='rm'))) + fsr.add_device_block(self.dev2_id, (GENERIC_COMMAND_BLOCK.format(command='rm') + GENERIC_COMMAND_BLOCK.format(command='rm'))) + self.prep_sci_response(fsr) + + out_dict = self.fss_api.send_command_block(self.target, command_block) + + # Should just be Nones because they are delete commands with no errors + expected_dict = { + self.dev1_id: [None, None], + self.dev2_id: [None, None], + } + + self.assertDictEqual(expected_dict, out_dict) + + def test_send_command_block_mixed_commands(self): + command_block = FileSystemServiceCommandBlock() + command_block.add_command(DeleteCommand(path='/a/path/1.txt')) + command_block.add_command(GetCommand(path='/a/path/2.txt')) + + data_string = base64.b64encode(six.b('testing string')).decode('ascii') + + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, (GENERIC_COMMAND_BLOCK.format(command='rm') + GET_FILE_BLOCK.format(data=data_string))) + fsr.add_device_block(self.dev2_id, (GENERIC_COMMAND_BLOCK.format(command='rm') + GET_FILE_BLOCK.format(data=data_string))) + self.prep_sci_response(fsr) + + out_dict = self.fss_api.send_command_block(self.target, command_block) + + expected_dict = { + self.dev1_id: [None, six.b('testing string')], + self.dev2_id: [None, six.b('testing string')], + } + + self.assertDictEqual(expected_dict, out_dict) + + def test_send_command_block_errors(self): + command_block = FileSystemServiceCommandBlock() + command_block.add_command(DeleteCommand(path='/a/path/1.txt')) + command_block.add_command(GetCommand(path='/a/path/2.txt')) + + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, (GENERIC_COMMAND_BLOCK.format(command='rm') + ERROR_BLOCK.format(command='get_file', errno=1, errtext="an error message"))) + fsr.add_device_block(self.dev2_id, (GENERIC_COMMAND_BLOCK.format(command='rm') + ERROR_BLOCK.format(command='get_file', errno=1, errtext="an error message"))) + self.prep_sci_response(fsr) + + out_dict = self.fss_api.send_command_block(self.target, command_block) + + # Check empty DeleteCommand responses + self.assertIsNone(out_dict[self.dev1_id][0]) + self.assertIsNone(out_dict[self.dev2_id][0]) + + # Check error info values + self.assertEqual(1, out_dict[self.dev1_id][1].errno) + self.assertEqual(1, out_dict[self.dev2_id][1].errno) + self.assertEqual("an error message", out_dict[self.dev1_id][1].message) + self.assertEqual("an error message", out_dict[self.dev2_id][1].message) + + +if __name__ == '__main__': + unittest.main() diff --git a/devicecloud/test/test_filedata.py b/devicecloud/test/unit/test_filedata.py similarity index 89% rename from devicecloud/test/test_filedata.py rename to devicecloud/test/unit/test_filedata.py index 90319ef..16da0c2 100644 --- a/devicecloud/test/test_filedata.py +++ b/devicecloud/test/unit/test_filedata.py @@ -1,10 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. + import base64 import unittest from xml.etree import ElementTree import datetime from dateutil.tz import tzutc -from devicecloud.test.test_utilities import HttpTestBase +from devicecloud.test.unit.test_utilities import HttpTestBase import six @@ -121,6 +127,13 @@ def test_write_file_simple(self): self.assertEqual(base64.decodestring(six.b(fd_data)), data) self.assertEqual(root.find("fdArchive").text, "true") + def test_delete_path(self): + self.prepare_response("DELETE", "/ws/FileData/test", "") + self.dc.filedata.delete_file("/test") + req = self._get_last_request() + self.assertEqual(req.method, "DELETE") + self.assertEqual(req.path, "/ws/FileData/test") + def test_walk(self): self.prepare_response("GET", "/ws/FileData", GET_HOME_RESULT) gen = self.dc.filedata.walk() @@ -161,6 +174,20 @@ def test_walk(self): class TestFileDataObject(HttpTestBase): + + def test_file_delete(self): + self.prepare_response("GET", "/ws/FileData", GET_FILEDATA_SIMPLE) + objects = list(self.dc.filedata.get_filedata()) + self.assertEqual(len(objects), 2) + obj = objects[0] + self.assertEqual(obj.get_full_path(), "/db/blah/test.txt") + + self.prepare_response("DELETE", "/ws/FileData/db/blah/test.txt", "") + obj.delete() + req = self._get_last_request() + self.assertEqual(req.method, "DELETE") + self.assertEqual(req.path, "/ws/FileData/db/blah/test.txt") + def test_file_metadata_access(self): self.prepare_response("GET", "/ws/FileData", GET_FILEDATA_SIMPLE) objects = list(self.dc.filedata.get_filedata()) @@ -171,9 +198,9 @@ def test_file_metadata_access(self): self.assertEqual(obj.get_type(), "file") self.assertEqual(obj.get_content_type(), "application/binary") self.assertEqual(obj.get_last_modified_date(), - datetime.datetime(2014, 7, 20, 18, 46, 45, 123, tzinfo=tzutc())) + datetime.datetime(2014, 7, 20, 18, 46, 45, 123000, tzinfo=tzutc())) self.assertEqual(obj.get_created_date(), - datetime.datetime(2014, 7, 20, 18, 46, 45, 123, tzinfo=tzutc())) + datetime.datetime(2014, 7, 20, 18, 46, 45, 123000, tzinfo=tzutc())) self.assertEqual(obj.get_customer_id(), "1234") self.assertEqual(obj.get_full_path(), "/db/blah/test.txt") self.assertEqual(obj.get_size(), 1234) @@ -189,9 +216,9 @@ def test_directory_metadata_access(self): self.assertEqual(obj.get_type(), "directory") self.assertEqual(obj.get_content_type(), "application/xml") self.assertEqual(obj.get_last_modified_date(), - datetime.datetime(2014, 7, 20, 18, 46, 45, 123, tzinfo=tzutc())) + datetime.datetime(2014, 7, 20, 18, 46, 45, 123000, tzinfo=tzutc())) self.assertEqual(obj.get_created_date(), - datetime.datetime(2014, 7, 20, 18, 46, 45, 123, tzinfo=tzutc())) + datetime.datetime(2014, 7, 20, 18, 46, 45, 123000, tzinfo=tzutc())) self.assertEqual(obj.get_customer_id(), "1234") self.assertEqual(obj.get_full_path(), "/db/blah/") self.assertEqual(obj.get_size(), 0) diff --git a/devicecloud/test/unit/test_monitor.py b/devicecloud/test/unit/test_monitor.py new file mode 100644 index 0000000..dd0a6bb --- /dev/null +++ b/devicecloud/test/unit/test_monitor.py @@ -0,0 +1,304 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. + +from devicecloud.monitor import MON_TOPIC_ATTR, MON_TRANSPORT_TYPE_ATTR +from devicecloud.test.unit.test_utilities import HttpTestBase +import six + +CREATE_TCP_MONITOR_GOOD_REQUEST = """\ + + topA,topB + 10 + json + tcp + gzip + +""" + +CREATE_HTTP_MONITOR_GOOD_REQUEST = """\ + + topA,topB + 1 + json + http + http://digi.com + None + PUT + 0 + 0 + none + +""" + +CREATE_MONITOR_GOOD_RESPONSE = """\ + + + Monitor/178008 + +""" + +GET_TCP_MONITOR_SINGLE_FOUND = """\ +{ + "resultTotalRows": "1", + "requestedStartRow": "0", + "resultSize": "1", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { + "monId": "178007", + "cstId": "7603", + "monTopic": "DeviceCore,FileDataCore,FileData,DataPoint", + "monTransportType": "tcp", + "monFormatType": "json", + "monBatchSize": "1", + "monCompression": "zlib", + "monStatus": "INACTIVE", + "monBatchDuration": "10" + } + ] +} +""" + +GET_HTTP_MONITOR_SINGLE_FOUND = """\ +{ + "resultTotalRows": "1", + "requestedStartRow": "0", + "resultSize": "1", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { + "monId": "178007", + "cstId": "7603", + "monTopic": "DeviceCore,FileDataCore,FileData,DataPoint", + "monTransportType": "http", + "monFormatType": "json", + "monBatchSize": "1", + "monCompression": "none", + "monStatus": "INACTIVE", + "monBatchDuration": "0" + } + ] +} +""" + +GET_TCP_MONITOR_METADTATA = """\ +{ + "resultTotalRows": "1", + "requestedStartRow": "0", + "resultSize": "1", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { + "monId": "178007", + "cstId": "7603", + "monTopic": "DeviceCore,FileDataCore,FileData,DataPoint", + "monTransportType": "tcp", + "monFormatType": "json", + "monBatchSize": "1", + "monCompression": "zlib", + "monStatus": "INACTIVE", + "monBatchDuration": "10" + } + ] +} +""" + +GET_HTTP_MONITOR_METADTATA = """\ +{ + "resultTotalRows": "1", + "requestedStartRow": "0", + "resultSize": "1", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { + "monId": "178007", + "cstId": "7603", + "monTopic": "DeviceCore,FileDataCore,FileData,DataPoint", + "monTransportType": "http", + "monFormatType": "json", + "monBatchSize": "1", + "monCompression": "none", + "monStatus": "INACTIVE", + "monBatchDuration": "0" + } + ] +} +""" + +GET_MONITOR_MULTIPLE_FOUND = """\ +{ + "resultTotalRows": "2", + "requestedStartRow": "0", + "resultSize": "2", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { + "monId": "178007", + "cstId": "7603", + "monTopic": "DeviceCore,FileDataCore,FileData,DataPoint", + "monTransportType": "tcp", + "monFormatType": "json", + "monBatchSize": "1", + "monCompression": "zlib", + "monStatus": "INACTIVE", + "monBatchDuration": "10" + }, + { + "monId": "178007", + "cstId": "7603", + "monTopic": "DeviceCore,FileDataCore,FileData,DataPoint", + "monTransportType": "tcp", + "monFormatType": "json", + "monBatchSize": "1", + "monCompression": "zlib", + "monStatus": "INACTIVE", + "monBatchDuration": "10" + }, + { + "monId": "178007", + "cstId": "7603", + "monTopic": "DeviceCore,FileDataCore,FileData,DataPoint", + "monTransportType": "http", + "monFormatType": "json", + "monBatchSize": "1", + "monCompression": "none", + "monStatus": "INACTIVE", + "monBatchDuration": "0" + } + ] +} +""" + +GET_MONITOR_NONE_FOUND = """\ +{ + "resultTotalRows": "0", + "requestedStartRow": "0", + "resultSize": "0", + "requestedSize": "1000", + "remainingSize": "0", + "items": [] +} +""" + + +class TestMonitorAPI(HttpTestBase): + + def test_create_tcp_monitor(self): + self.prepare_response("POST", "/ws/Monitor", data=CREATE_MONITOR_GOOD_RESPONSE) + mon = self.dc.monitor.create_tcp_monitor(['topA', 'topB'], batch_size=10, batch_duration=0, + compression='gzip', format_type='json') + self.assertEqual(self._get_last_request().body, six.b(CREATE_TCP_MONITOR_GOOD_REQUEST)) + self.assertEqual(mon.get_id(), 178008) + + def test_create_http_monitor(self): + self.prepare_response("POST", "/ws/Monitor", data=CREATE_MONITOR_GOOD_RESPONSE) + mon = self.dc.monitor.create_http_monitor(['topA', 'topB'], 'http://digi.com', transport_token=None, + transport_method='PUT', connect_timeout=0, response_timeout=0, + batch_size=1, batch_duration=0, compression='none', + format_type='json') + self.assertEqual(self._get_last_request().body, six.b(CREATE_HTTP_MONITOR_GOOD_REQUEST)) + self.assertEqual(mon.get_id(), 178008) + + def test_get_tcp_monitors(self): + self.prepare_response("GET", "/ws/Monitor", data=GET_TCP_MONITOR_SINGLE_FOUND) + mons = list(self.dc.monitor.get_monitors((MON_TOPIC_ATTR == "DeviceCore") & + (MON_TRANSPORT_TYPE_ATTR == "tcp"))) + self.assertEqual(len(mons), 1) + mon = mons[0] + self.assertEqual(mon.get_id(), 178007) + self.assertEqual(self._get_last_request_params(), { + 'condition': "monTopic='DeviceCore' and monTransportType='tcp'", + 'start': '0', + 'size': '1000' + }) + + def test_get_http_monitors(self): + self.prepare_response("GET", "/ws/Monitor", data=GET_TCP_MONITOR_SINGLE_FOUND) + mons = list(self.dc.monitor.get_monitors((MON_TOPIC_ATTR == "DeviceCore") & + (MON_TRANSPORT_TYPE_ATTR == "http"))) + self.assertEqual(len(mons), 1) + mon = mons[0] + self.assertEqual(mon.get_id(), 178007) + self.assertEqual(self._get_last_request_params(), { + 'condition': "monTopic='DeviceCore' and monTransportType='http'", + 'start': '0', + 'size': '1000' + }) + + def test_tcp_get_monitor_present(self): + self.prepare_response("GET", "/ws/Monitor", data=GET_TCP_MONITOR_SINGLE_FOUND) + mon = self.dc.monitor.get_monitor(['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint']) + self.assertEqual(mon.get_id(), 178007) + self.assertEqual(self._get_last_request_params(), { + 'condition': "monTopic='DeviceCore,FileDataCore,FileData,DataPoint'", + 'start': '0', + 'size': '1000' + }) + + def test_http_get_monitor_present(self): + self.prepare_response("GET", "/ws/Monitor", data=GET_HTTP_MONITOR_SINGLE_FOUND) + mon = self.dc.monitor.get_monitor(['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint']) + self.assertEqual(mon.get_id(), 178007) + self.assertEqual(self._get_last_request_params(), { + 'condition': "monTopic='DeviceCore,FileDataCore,FileData,DataPoint'", + 'start': '0', + 'size': '1000' + }) + + def test_get_monitor_multiple(self): + # Should just pick the first result (currently), so results are the same as ever + self.prepare_response("GET", "/ws/Monitor", data=GET_MONITOR_MULTIPLE_FOUND) + mon = self.dc.monitor.get_monitor(['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint']) + self.assertEqual(mon.get_id(), 178007) + self.assertEqual(self._get_last_request_params(), { + 'condition': "monTopic='DeviceCore,FileDataCore,FileData,DataPoint'", + 'start': '0', + 'size': '1000' + }) + + def test_get_monitor_does_not_exist(self): + # Should just pick the first result (currently), so results are the same as ever + self.prepare_response("GET", "/ws/Monitor", data=GET_MONITOR_NONE_FOUND) + mon = self.dc.monitor.get_monitor(['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint']) + self.assertEqual(mon, None) + + +class TestDeviceCloudMonitor(HttpTestBase): + + def setUp(self): + HttpTestBase.setUp(self) + self.prepare_response("GET", "/ws/Monitor", data=GET_TCP_MONITOR_SINGLE_FOUND) + mon = self.dc.monitor.get_monitor(['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint']) + self.mon = mon + + def test_get_tcp_metadata(self): + self.prepare_response("GET", "/ws/Monitor/178007", data=GET_TCP_MONITOR_METADTATA) + self.assertEqual(self.mon.get_metadata(), { + "monId": "178007", + "cstId": "7603", + "monTopic": "DeviceCore,FileDataCore,FileData,DataPoint", + "monTransportType": "tcp", + "monFormatType": "json", + "monBatchSize": "1", + "monCompression": "zlib", + "monStatus": "INACTIVE", + "monBatchDuration": "10" + }) + + def test_delete(self): + self.prepare_response("DELETE", "/ws/Monitor/178007") + self.mon.delete() + req = self._get_last_request() + self.assertEqual(req.method, "DELETE") + self.assertEqual(req.path, "/ws/Monitor/178007") + + def test_get_id(self): + self.assertEqual(self.mon.get_id(), 178007) diff --git a/devicecloud/test/unit/test_monitor_tcp.py b/devicecloud/test/unit/test_monitor_tcp.py new file mode 100644 index 0000000..1293273 --- /dev/null +++ b/devicecloud/test/unit/test_monitor_tcp.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. + +from devicecloud.monitor_tcp import TCPClientManager +from devicecloud.test.unit.test_utilities import HttpTestBase + + +class TestTCPClientManager(HttpTestBase): + + # NOTE: currently only integration tests exist to test several parts of + # the basic device cloud push client functionality for historical reasons. + # In the future, it would be nice to extended the unit test coverage + # for this code. + + def setUp(self): + HttpTestBase.setUp(self) + self.client_manager = TCPClientManager(self.dc.get_connection()) + + def test_hostname(self): + self.assertEqual(self.client_manager.hostname, "devicecloud.digi.com") + + def test_username(self): + self.assertEqual(self.client_manager.username, "user") + + def test_password(self): + self.assertEqual(self.client_manager.password, "pass") diff --git a/devicecloud/test/unit/test_sci.py b/devicecloud/test/unit/test_sci.py new file mode 100644 index 0000000..0428f64 --- /dev/null +++ b/devicecloud/test/unit/test_sci.py @@ -0,0 +1,274 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. + +import unittest + +import httpretty +import mock +import six +import re +import xml.etree.ElementTree as ET + +from devicecloud import DeviceCloud +from devicecloud.sci import DeviceTarget, GroupTarget, AsyncRequestProxy, ServerCommandInterfaceAPI +from devicecloud.test.unit.test_utilities import HttpTestBase + + +EXAMPLE_SCI_DEVICE_NOT_CONNECTED = """\ +Device Not Connected +""" + +EXAMPLE_SCI_BAD_DEVICE = """\ + + + + + Invalid target. Device not found. + + + Invalid SCI request. No valid targets found. + + +""" + +EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED = """\ +completeDevice Not Connected +""" + +EXAMPLE_ASYNC_SCI_INCOMPLETE = """\ +in_progress +""" + +EXAMPLE_SCI_REQUEST_PAYLOAD = """\ + + + + + +""" + +EXAMPLE_SCI_REQUEST_RESPONSE = """\ + + + + + + + running + + + + + + +""" + +EXAMPLE_ASYNC_SCI_RESPONSE = """\ + + + 133225503 + + +""" + +EXAMPLE_UPDATE_FIRMWARE_INVALID_ATTRIBUTE_REQUEST_PAYLOAD = """ +aHNxcAbAADUct1cAAACAHEBAAAEABEAwAIBAAQAAACOFFzU +""" + +EXAMPLE_UPDATE_FIRMWARE_INVALID_ATTRIBUTE_RESPONSE = """\ + +Default target not availabe, specify a target number of filename + + +""" + +class TestSCI(HttpTestBase): + def _prepare_sci_response(self, response, status=200): + self.prepare_response("POST", "/ws/sci", response, status) + + def test_sci_successful_error(self): + self._prepare_sci_response(EXAMPLE_SCI_DEVICE_NOT_CONNECTED) + self.dc.get_sci_api().send_sci( + operation="send_message", + target=DeviceTarget("00000000-00000000-00409DFF-FF58175B"), + payload="") + self.assertEqual(httpretty.last_request().body, + six.b('' + '' + '' + '' + '' + '' + '' + '')) + + def test_sci_successful_group_target(self): + self._prepare_sci_response(EXAMPLE_SCI_DEVICE_NOT_CONNECTED) + self.dc.get_sci_api().send_sci( + operation="send_message", + target=GroupTarget("TestGroup"), + payload="") + self.assertEqual(httpretty.last_request().body, + six.b('' + '' + '' + '' + '' + '' + '' + '')) + + def test_sci_no_parameters(self): + self._prepare_sci_response(EXAMPLE_SCI_REQUEST_RESPONSE) + self.dc.get_sci_api().send_sci( + operation="send_message", + target=DeviceTarget('00000000-00000000-00409dff-ffaabbcc'), + payload=EXAMPLE_SCI_REQUEST_PAYLOAD) + request = httpretty.last_request().body.decode('utf8') + # Strip white space from lines and concatenate request + request = ''.join([line.strip() for line in request.splitlines()]) + self.assertEqual(request, + six.u('' + '' + '' + '' + '' + '' + '' + '' + '' + '' + '' + '')) + + def test_sci_with_parameters(self): + self._prepare_sci_response(EXAMPLE_SCI_REQUEST_RESPONSE) + self.dc.get_sci_api().send_sci( + operation="send_message", + target=DeviceTarget('00000000-00000000-00409dff-ffaabbcc'), + payload=EXAMPLE_SCI_REQUEST_PAYLOAD, + reply="all", + synchronous=True, + sync_timeout=42, + cache=False, + allow_offline=True, + wait_for_reconnect=True, + ) + request = httpretty.last_request().body.decode('utf8') + # Verify attributes exist in + expected_attrib = { + "reply": "all", + "synchronous": "true", + "syncTimeout": "42", + "cache": "false", + "allowOffline": "true", + "waitForReconnect": "true", + } + request_e = ET.fromstring(request) + send_message_e = request_e.find('./send_message') + self.assertEqual(expected_attrib, send_message_e.attrib) + # Strip white space from lines and concatenate request + request = ''.join([line.strip() for line in request.splitlines()]) + # Replace from request with one without parameters so the final check can be done + match = re.search('', request) + request = request[:match.start()] + '' + request[match.end():] + self.assertEqual(request, + six.u('' + '' + '' + '' + '' + '' + '' + '' + '' + '' + '' + '')) + + def test_sci_update_firmware_attribute(self): + + self._prepare_sci_response(EXAMPLE_UPDATE_FIRMWARE_INVALID_ATTRIBUTE_RESPONSE) + self.dc.get_sci_api().send_sci( + operation="update_firmware", + attribute="filename=\"abcd.bin\"", + target=DeviceTarget('00000000-00000000-00409dff-ffaabbcc'), + payload=EXAMPLE_UPDATE_FIRMWARE_INVALID_ATTRIBUTE_REQUEST_PAYLOAD) + + request = httpretty.last_request().body.decode('utf8') + request = ''.join([line.strip() for line in request.splitlines()]) + self.assertEqual(request, + six.u('' + '' + '' + '' + '' + 'aHNxcAbAADUct1cAAACAHEBAAAEABEAwAIBAAQAAACOFFzU' + '' + '')) + + +class TestGetAsync(HttpTestBase): + def test_sci_get_async(self): + self.prepare_response("GET", "/ws/sci/123", EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED, 200) + resp = self.dc.get_sci_api().get_async_job(123) + self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.content, six.b(EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED)) + + +class TestAsyncProxy(HttpTestBase): + def setUp(self): + HttpTestBase.setUp(self) + self.fake_conn = mock.MagicMock() + + def test_ctor(self): + t = AsyncRequestProxy(123, self.fake_conn) + self.assertEqual(t.job_id, 123) + self.assertIs(t.response, None) + + def test_completed_false(self): + self.prepare_response("GET", "/ws/sci/123", EXAMPLE_ASYNC_SCI_INCOMPLETE, 200) + t = AsyncRequestProxy(123, self.dc.get_sci_api()._conn) + self.assertIs(t.completed, False) + self.assertIs(t.response, None) + + def test_completed_true(self): + self.prepare_response("GET", "/ws/sci/123", EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED, 200) + t = AsyncRequestProxy(123, self.dc.get_sci_api()._conn) + self.assertIs(t.completed, True) + self.assertEqual(t.response, six.b(EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED)) + + def test_completed_already(self): + self.prepare_response("GET", "/ws/sci/123", EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED, 200) + t = AsyncRequestProxy(123, self.dc.get_sci_api()._conn) + t.response = EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED + self.assertIs(t.completed, True) + + +class TestSendSciAsync(HttpTestBase): + @mock.patch.object(ServerCommandInterfaceAPI, "send_sci") + def test_bad_resp(self, fake_send_sci): + fake_resp = mock.MagicMock() + fake_resp.status_code = 400 + fake_resp.reason = "OK" + fake_resp.content = EXAMPLE_SCI_BAD_DEVICE + fake_send_sci.return_value = fake_resp + resp = self.dc.get_sci_api().send_sci_async("send_message", DeviceTarget('00000000-00000000-00409dff-ffaabbcc'), EXAMPLE_SCI_REQUEST_PAYLOAD) + self.assertIs(resp, None) + + @mock.patch.object(ServerCommandInterfaceAPI, "send_sci") + def test_resp_parse(self, fake_send_sci): + fake_resp = mock.MagicMock() + fake_resp.status_code = 200 + fake_resp.reason = "OK" + fake_resp.content = EXAMPLE_ASYNC_SCI_RESPONSE + fake_send_sci.return_value = fake_resp + resp = self.dc.get_sci_api().send_sci_async("send_message", DeviceTarget('00000000-00000000-00409dff-ffaabbcc'), EXAMPLE_SCI_REQUEST_PAYLOAD) + self.assertEqual(resp.job_id, 133225503) + + +if __name__ == "__main__": + unittest.main() diff --git a/devicecloud/test/test_streams.py b/devicecloud/test/unit/test_streams.py similarity index 85% rename from devicecloud/test/test_streams.py rename to devicecloud/test/unit/test_streams.py index f8bb840..c7bc119 100644 --- a/devicecloud/test/test_streams.py +++ b/devicecloud/test/unit/test_streams.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015-2018 Digi International Inc. import unittest import datetime @@ -11,12 +10,15 @@ from dateutil.tz import tzutc from devicecloud.streams import DataStream, STREAM_TYPE_FLOAT, DataPoint, NoSuchStreamException, ROLLUP_INTERVAL_HALF, \ - ROLLUP_METHOD_COUNT, STREAM_TYPE_INTEGER -from devicecloud.test.test_utilities import HttpTestBase + ROLLUP_METHOD_COUNT, STREAM_TYPE_INTEGER, DSTREAM_TYPE_MAP, STREAM_TYPE_JSON +from devicecloud.test.unit.test_utilities import HttpTestBase from devicecloud import DeviceCloudHttpException + # Example HTTP Responses import httpretty +import mock +import re import six CREATE_DATA_STREAM = { @@ -64,6 +66,46 @@ } """ +GET_DATA_STREAMS_0 = """ +{ + "resultSize": "1", + "requestedSize": "1000", + "pageCursor": "9d870afb-2-af668f74", + "items": [ + { + "cstId": "7603", + "streamId": "another/test", + "dataType": "INTEGER", + "forwardTo": "", + "description": "Some Integral Thing", + "units": "", + "dataTtl": "172800", + "rollupTtl": "432000" + } + ] +} +""" + +GET_DATA_STREAMS_1 = """ +{ + "resultSize": "1", + "requestedSize": "1000", + "pageCursor": "9d870afb-2-af668f74", + "items": [ + { + "cstId": "7603", + "streamId": "test", + "dataType": "FLOAT", + "forwardTo": "", + "description": "some description", + "units": "light years", + "dataTtl": "172800", + "rollupTtl": "432000" + } + ] +} +""" + GET_DATA_STREAMS_EMPTY = """ { "resultSize": "0", @@ -103,6 +145,36 @@ } """ +GET_TEST_DATA_STREAM_JSON = """\ +{ +"resultSize": "1", +"requestedSize": "1000", +"pageCursor": "88afa98e-1-7efbf125", +"items": [ +{ + "cstId": "7603", + "streamId": "test", + "dataType": "JSON", + "forwardTo": "", + "currentValue": { + "id": "07d77854-0557-11e4-ab44-fa163e7ebc6b", + "timestamp": "1404683207981", + "timestampISO": "2014-07-06T21:46:47.981Z", + "serverTimestamp": "1404683207981", + "serverTimestampISO": "2014-07-06T21:46:47.981Z", + "data": "{\\"key3\\": [1, 2, 3], \\"key1\\": \\"value1\\", \\"2\\": 2}", + "description": "Test", + "quality": "20", + "location": "1.0,2.0,3.0" + }, + "description": "some description", + "units": "light years", + "dataTtl": "172800", + "rollupTtl": "432000"} +] +} +""" + GET_TEST_DATA_STREAM_NO_CURRENT_VALUE = """\ { "resultSize": "1", @@ -303,6 +375,19 @@ def test_get_streams(self): self.assertIsInstance(streams[0], DataStream) self.assertIsInstance(streams[1], DataStream) + def test_get_streams_with_id(self): + self.prepare_response("GET", "/ws/DataStream/test", GET_DATA_STREAMS_1) + streams = list(self.dc.streams.get_streams('test')) + self.assertEqual(len(streams), 1) + self.assertIsInstance(streams[0], DataStream) + self.prepare_response("GET", "/ws/DataStream/another", GET_DATA_STREAMS_0) + streams = list(self.dc.streams.get_streams('another')) + self.assertEqual(len(streams), 1) + self.assertIsInstance(streams[0], DataStream) + self.prepare_response("GET", "/ws/DataStream/junk", GET_DATA_STREAMS_EMPTY) + streams = self.dc.streams.get_streams('junk') + self.assertEqual(list(streams), []) + def test_get_stream(self): # Get a stream by ID when there is no cache stream = self.dc.streams.get_stream("/test/stream") @@ -335,6 +420,7 @@ def test_bulk_write_datapoints_datapoint_has_no_stream_id(self): def test_bulk_write_multiple_pages(self): # Actual response has a ton of locations for the new data points requests = [] + def handle_request(request, uri, headers): requests.append(request) return (200, headers, '') @@ -351,7 +437,6 @@ def handle_request(request, uri, headers): self.dc.streams.bulk_write_datapoints(datapoints) self.assertEqual(len(requests), 2) - def parse_for_data(response): root = ET.fromstring(response) return [int(x.text) for x in root.iter('data')] @@ -414,7 +499,7 @@ def test_write_simple(self): data=123.4, )) - # verify that the body sent to the device cloud is sufficiently minimal + # verify that the body sent to Device Cloud is sufficiently minimal self.assertEqual( httpretty.last_request().body, six.b('' @@ -434,7 +519,7 @@ def test_write_full(self): units="scolvilles", )) - # verify that the body sent to the device cloud is sufficiently minimal + # verify that the body sent to Device Cloud is sufficiently minimal self.assertEqual( httpretty.last_request().body, six.b('' @@ -481,6 +566,7 @@ def test_bulk_write_multiple_pages(self): stream = self._get_stream("test") requests = [] + def handle_request(request, uri, headers): requests.append(request) return (200, headers, '') @@ -510,7 +596,6 @@ def parse_for_stream_id(response): self.assertEqual(parse_for_stream_id(requests[1].body), {'test'}) - class TestDataStreamDeleteDataPoints(HttpTestBase): def test_delete_datapoint(self): @@ -707,8 +792,8 @@ def test_accessors(self): dp = stream.get_current_value() self.assertEqual(dp.get_id(), "07d77854-0557-11e4-ab44-fa163e7ebc6b") - self.assertEqual(dp.get_timestamp(), datetime.datetime(2014, 7, 6, 21, 46, 47, 981, tzinfo=tzutc())) - self.assertEqual(dp.get_server_timestamp(), datetime.datetime(2014, 7, 6, 21, 46, 47, 981, tzinfo=tzutc())) + self.assertEqual(dp.get_timestamp(), datetime.datetime(2014, 7, 6, 21, 46, 47, 981000, tzinfo=tzutc())) + self.assertEqual(dp.get_server_timestamp(), datetime.datetime(2014, 7, 6, 21, 46, 47, 981000, tzinfo=tzutc())) self.assertEqual(dp.get_data(), 123.1) self.assertEqual(dp.get_description(), "Test") self.assertEqual(dp.get_quality(), 20) @@ -717,6 +802,62 @@ def test_accessors(self): self.assertEqual(dp.get_stream_id(), "test") self.assertEqual(dp.get_data_type(), "FLOAT") + def test_get_json_data(self): + stream = self._get_stream("test", with_cached_data=True) + self.prepare_response("GET", "/ws/DataStream/test", GET_TEST_DATA_STREAM_JSON) + + dp = stream.get_current_value() + expected_dict = {'key1': 'value1', '2': 2, 'key3': [1, 2, 3]} + self.assertDictEqual(expected_dict, dp.get_data()) + + def test_json_encode_to_xml(self): + my_dict = {'key1': 'value1', '2': 2, 'key3': [1, 2, 3]} + dp = DataPoint( + data_type=STREAM_TYPE_JSON, + data=my_dict, + ) + xml = dp.to_xml() + + self.assertIsNotNone(re.search('\{[ ",a-zA-Z0-9:[\]]+\}', xml)) + self.assertIsNotNone(re.search('"key1": "value1"', xml)) + self.assertIsNotNone(re.search('"2": 2', xml)) + self.assertIsNotNone(re.search('"key3": \[1, 2, 3\]', xml)) + + def test_from_json_conversion(self): + stream = self._get_stream("test", with_cached_data=False) + self.prepare_response("GET", "/ws/DataStream/test", GET_TEST_DATA_STREAM) + test_json_data = {six.u('description'): six.u('Test'), + six.u('quality'): six.u('20'), + six.u('timestamp'): six.u('1404683207981'), + six.u('data'): six.u('3.14159265358'), + six.u('serverTimestampISO'): six.u('2014-07-06T21:46:47.981Z'), + six.u('location'): six.u('1.0,2.0,3.0'), + six.u('timestampISO'): six.u('2014-07-06T21:46:47.981Z'), + six.u('serverTimestamp'): six.u('1404683207981'), + six.u('id'): six.u('07d77854-0557-11e4-ab44-fa163e7ebc6b')} + + dp = DataPoint.from_json(stream, test_json_data) + self.assertEqual(3.14159265358, dp.get_data()) + + def test_get_data_no_conversion(self): + # This is to prove that the issue of calling the DSTREAM_TYPE_MAP conversion + # methods is no longer done on the get_data call. Previously this could + # result in already converted data trying to be converted again. For most + # types this is not an issue (i.e. calling float on a float) however, some + # conversions could have typing issues when run on their own results. + old_float_conversion = DSTREAM_TYPE_MAP[STREAM_TYPE_FLOAT] + mfloat = mock.Mock(side_effect=float) + DSTREAM_TYPE_MAP[STREAM_TYPE_FLOAT] = (mfloat, str) + my_float = 3.14159265358 + dp = DataPoint( + data_type=STREAM_TYPE_FLOAT, + data=my_float, + quality=0 + ) + self.assertEqual(my_float, dp.get_data()) + self.assertFalse(mfloat.called) + DSTREAM_TYPE_MAP[STREAM_TYPE_FLOAT] = old_float_conversion + def test_bad_location_string(self): dp = DataPoint(123) self.assertRaises(ValueError, dp.set_location, "0,1") diff --git a/devicecloud/test/test_utilities.py b/devicecloud/test/unit/test_utilities.py similarity index 57% rename from devicecloud/test/test_utilities.py rename to devicecloud/test/unit/test_utilities.py index 11a3b97..57dc6a1 100644 --- a/devicecloud/test/test_utilities.py +++ b/devicecloud/test/unit/test_utilities.py @@ -2,20 +2,20 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015-2018 Digi International Inc. import unittest import json from devicecloud import DeviceCloud import httpretty +import six.moves.urllib.parse as urllib_parse class HttpTestBase(unittest.TestCase): def setUp(self): httpretty.enable() - # setup the Device cloud ping response + # setup Device Cloud ping response self.prepare_response("GET", "/ws/DeviceCore?size=1", "", status=200) self.dc = DeviceCloud('user', 'pass') @@ -26,16 +26,22 @@ def tearDown(self): def _get_last_request(self): return httpretty.last_request() - def prepare_response(self, method, path, data, status=200, match_querystring=False): + def _get_last_request_params(self): + # Get the query params from the last request as a dictionary + params = urllib_parse.parse_qs(urllib_parse.urlparse(self._get_last_request().path).query) + return {k: v[0] for k, v in params.items()} # convert from list values to single-value + + def prepare_response(self, method, path, data=None, status=200, match_querystring=False, **kwargs): # TODO: # Should probably assert on more request headers and # respond with correct content type, etc. - + if data is not None: + kwargs['body'] = data httpretty.register_uri(method, - "https://login.etherios.com{}".format(path), - data, + "https://devicecloud.digi.com{}".format(path), match_querystring=match_querystring, - status=status) + status=status, + **kwargs) def prepare_json_response(self, method, path, data, status=200): self.prepare_response(method, path, json.dumps(data), status=status) diff --git a/devicecloud/test/unit/test_version.py b/devicecloud/test/unit/test_version.py new file mode 100644 index 0000000..1d020f7 --- /dev/null +++ b/devicecloud/test/unit/test_version.py @@ -0,0 +1,15 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. + +import unittest + +from devicecloud.version import __version__ + + +class TestVersion(unittest.TestCase): + + def test_version_format(self): + self.assertTrue(len(__version__.split('.')) >= 3) diff --git a/devicecloud/test/unit/test_ws.py b/devicecloud/test/unit/test_ws.py new file mode 100644 index 0000000..4561ff4 --- /dev/null +++ b/devicecloud/test/unit/test_ws.py @@ -0,0 +1,47 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. + +import unittest + +from devicecloud import DeviceCloudException +from mock import MagicMock, patch + +from devicecloud.ws import WebServiceStub +from devicecloud import DeviceCloudConnection + + +class MockConnection(MagicMock): + + def get(self, *args, **kwargs): + return (args, kwargs) + + def post(self, *args, **kwargs): + return (args, kwargs) + + +class LegacyAPIMemberInternalsTests(unittest.TestCase): + + def setUp(self): + self.conn = MockConnection() + self.stub = WebServiceStub(self.conn, '/ws') + + def test_path_building(self): + test = self.stub.a.b.c + self.assertEqual(test._path, "/ws/a/b/c") + + def test_method_access(self): + res = self.stub.a.b.c.get() + self.assertEqual(res[0], ("/ws/a/b/c", )) + self.assertDictEqual(res[1], {}) + + def test_method_access_args_kwargs(self): + res = self.stub.a.test.path.post("foo", bar="baz") + self.assertEqual(res[0], ("/ws/a/test/path", "foo")) + self.assertDictEqual(res[1], {"bar": "baz"}) + + +if __name__ == '__main__': + unittest.main() diff --git a/devicecloud/util.py b/devicecloud/util.py index 7b7cc50..e573c77 100644 --- a/devicecloud/util.py +++ b/devicecloud/util.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015-2018 Digi International Inc. import datetime import arrow @@ -18,7 +17,7 @@ def conditional_write(strm, fmt, value, *args, **kwargs): def iso8601_to_dt(iso8601): - """Given an ISO8601 string as returned by the device cloud, convert to a datetime object""" + """Given an ISO8601 string as returned by Device Cloud, convert to a datetime object""" # We could just use arrow.get() but that is more permissive than we actually want. # Internal (but still public) to arrow is the actual parser where we can be # a bit more specific diff --git a/devicecloud/version.py b/devicecloud/version.py index ab86d3b..aee2b9e 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015-2018 Digi International Inc. -__version__ = "0.1.1" + +__version__ = "0.5.10" diff --git a/devicecloud/ws.py b/devicecloud/ws.py new file mode 100644 index 0000000..08d35a5 --- /dev/null +++ b/devicecloud/ws.py @@ -0,0 +1,50 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. +import functools +import inspect + + +class WebServiceStub(object): + """Provide a set of methods to directory access the web services API at a given path + + Web services stubs can be chained in order to build a path and, eventually, perform + an operation on the end result. For instance, the following will perform + a GET request on the path ``/some/base/with/some/added/stuff``:: + + WebServiceStub(conn, "/some/base").with.some.added.stuff.get() + + In the context of the this library, a more common example might be accessing + a V1 API such as /ws/v1/devices. That would be done like this:: + + response = dc.ws.v1.devices.get() + + Where response would end up being a `requests Response object + `. + + Any of the methods exposed by :class:`devicecloud.DeviceCloudConnection` may be + called and the path for the stub will be passed as the first argument to + the method with the same name in that class. + + """ + + def __init__(self, conn, path): + self._conn = conn + self._path = "/" + path if path[0] != '/' else path + + def __getattr__(self, attr): + """We implement this method to provide the "builder" syntax""" + conn_meth = getattr(self._conn, attr, None) + if conn_meth is not None and inspect.ismethod(conn_meth): + # If this is method on DeviceCloudConnection, then return a function bound to + # that method that will be called with this stub's path + @functools.wraps(conn_meth) + def bound_cloud_connection_method(*args, **kwargs): + return conn_meth(self._path, *args, **kwargs) + return bound_cloud_connection_method + + # Otherwise, assume that specified attribute is another path and return + # a new builder which is our path combined with the provided attribute + return WebServiceStub(self._conn, "{}/{}".format(self._path, attr)) diff --git a/docs/_static/.keepme b/docs/_static/.keepme new file mode 100644 index 0000000..e69de29 diff --git a/docs/conf.py b/docs/conf.py index a08b48c..3ddf4e3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,6 +22,8 @@ this_dir = os.path.dirname(__file__) sys.path.insert(0, os.path.abspath(os.path.join(this_dir, ".."))) +import devicecloud + # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. @@ -32,8 +34,13 @@ # ones. extensions = [ 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx' ] +intersphinx_mapping = { + 'python': ('http://docs.python.org/3', None), +} + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -48,16 +55,16 @@ # General information about the project. project = u'python-devicecloud' -copyright = u'2014, Etherios, Inc.' +copyright = u'2015-2018, Digi International Inc.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '0.1' +version = devicecloud.__version__ # The full version, including alpha/beta/rc tags. -release = '0.1' +release = devicecloud.__version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -186,22 +193,22 @@ # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # Additional stuff for the LaTeX preamble. + #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'python-devicecloud.tex', u'python-devicecloud Documentation', - u'Paul Osborne, Tom Manley, Stephen Stack', 'manual'), + ('index', 'python-devicecloud.tex', u'python-devicecloud Documentation', + u'Paul Osborne, Tom Manley, Stephen Stack, Brandon Moser', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -244,10 +251,10 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'python-devicecloud', u'python-devicecloud Documentation', - u'Paul Osborne, Tom Manley, Stephen Stack', 'python-devicecloud', - 'Device Cloud Web Services Client for Python', - 'Miscellaneous'), + ('index', 'python-devicecloud', u'python-devicecloud Documentation', + u'Paul Osborne, Tom Manley, Stephen Stack', 'python-devicecloud', + 'Device Cloud Web Services Client for Python', + 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. diff --git a/docs/cookbook.rst b/docs/cookbook.rst index b3e0bc9..89085c0 100644 --- a/docs/cookbook.rst +++ b/docs/cookbook.rst @@ -6,16 +6,16 @@ For more granular or specific examples of API usage check out the individual API .. note:: - There are also examples checked into source control under /devicecloud/examples/*_playground.py - which will provide additional example uses of the library. + There are also examples checked into source control under /devicecloud/examples/\*_playground.py + which will provide additional example uses of the library. Each example will assume an instance of :class:`devicecloud.DeviceCloud` has been created with something like so:: dc = DeviceCloud(, ) -Creating Streams and Data Points -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Streams - Creating Streams and Data Points +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ For this example, let's create a DataStream that represents a class room. As students enter the classroom there is a DataPoint representing them written to the stream. @@ -65,16 +65,16 @@ Finally, let's print the name of the student who most recently entered the class print json.loads(most_recent_student.get_data())['name'] # Prints 'Henry' -Delete a Stream -^^^^^^^^^^^^^^^^ +Streams - Deleting +^^^^^^^^^^^^^^^^^^^^ Let's delete the classroom stream from the above example so we can start fresh in the next example:: classroom.delete() -Roll-up Data -^^^^^^^^^^^^^^ +Streams - Roll-up Data +^^^^^^^^^^^^^^^^^^^^^^^^ Roll-up data is a way to group data points based on time intervals in which they were written to the cloud. From our previous example lets figure out which students @@ -143,3 +143,45 @@ number of students who entered the classroom that hour:: 22: 6, 23: 11} + +Device Core - Groups +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. note:: + + This assumes your device is provisioned. + +First, get a reference to the device which you would like to add a specific group:: + + device = devicecore.get_device('00:40:9D:50:B0:EA') + +Then you can add it to a group and fetch it to make sure it works:: + + device.add_to_group('mygroup') + device.get_group_path() # prints 'mygroup' (DC sometimes needs a second to catch up) + +Or remove it:: + + device.remove_from_group() + + +Device Core - Tags +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. note:: + + This assumes your device is provisioned. + +Similar to Groups, get a reference to the device which you would like to add a specific group:: + + device = devicecore.get_device('00:40:9D:50:B0:EA') + +Then you can add a tag and then get the new list:: + + device.add_tags('mytag') + device.get_tags() # prints ['mytags'] (DC sometimes needs a second to catch up) + +Or remove it:: + + device.remove_tag('mytag') + device.get_tags() # prints [] (DC sometimes needs a second to catch up) diff --git a/docs/core.rst b/docs/core.rst index fa77ad1..169bf21 100644 --- a/docs/core.rst +++ b/docs/core.rst @@ -1,11 +1,17 @@ Core API ======== -API Documentation ------------------ +DeviceCloud Core API +-------------------- The :class:`devicecloud.DeviceCloud` class contains the core interface which will be used by all clients using the devicecloud library. .. automodule:: devicecloud :members: + +Conditions API +-------------- + +.. automodule:: devicecloud.conditions + :members: diff --git a/docs/filedata.rst b/docs/filedata.rst index 042a667..87d104f 100644 --- a/docs/filedata.rst +++ b/docs/filedata.rst @@ -4,7 +4,7 @@ FileData API FileData Overview ----------------- -The FileData store on the device cloud provides a hierarchical mechanism for temporarily +The FileData store on Device Cloud provides a hierarchical mechanism for temporarily storing information in files sent from devices. With the APIs provided by the device cloud, it is possible to use the FileData store in a number of different ways to implement various use cases. @@ -20,7 +20,7 @@ basis. If using the FileData store as a queue, one will likely want to setup mo on FileData paths matching certain criterion. The set of files matching some condition can then be thought of as a channel. -This library seeks to make using the device cloud for both of these use cases simple +This library seeks to make using Device Cloud for both of these use cases simple and robust. Navigating the FileData Store @@ -129,7 +129,7 @@ API Documentation ----------------- The filedata module provides function for reading, writing, and -deleting "files" from the device cloud FileData store. +deleting "files" from Device Cloud FileData store. .. automodule:: devicecloud.filedata :members: diff --git a/docs/filesystem.rst b/docs/filesystem.rst new file mode 100644 index 0000000..94ab1ce --- /dev/null +++ b/docs/filesystem.rst @@ -0,0 +1,14 @@ +File System Service API +================================== + +File System Service Overview +---------------------------- + +Provide access to Device Cloud File System commands that use SCI to +get the data from your devices connected to the cloud. + +File System Service API Documentation +------------------------------------- + +.. automodule:: devicecloud.file_system_service + :members: diff --git a/docs/index.rst b/docs/index.rst index 9119adf..5f9a6d9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,28 +12,31 @@ Documention Map streams filedata sci + filesystem + monitor + ws cookbook Introduction ============ Python-devicecloud is a library providing simple, intuitive access to -the `Device Cloud by Etherios -`_ for clients written +the `Digi Device Cloud +`_ for clients written in Python. -The library wraps the Device Cloud REST API and hides the details of +The library wraps Device Cloud REST API and hides the details of forming HTTP requests in order to gain access to device information, -file data, streams, and other features of the device cloud. The API +file data, streams, and other features of Device Cloud. The API wrapped can be found `here `_. The primary target audience for this library is individuals -interfacing with the device cloud from the server side or developers +interfacing with Device Cloud from the server side or developers writing tools to aid device development. For efficient connectivity from devices, we suggest that you first look at using the `Device -Cloud Connector `_. +Cloud Connector `_. That being said, this library could also be used on devices if deemed suitable. @@ -44,18 +47,21 @@ quick example of what the API looks like:: dc = DeviceCloud('user', 'pass') + # You can also use API keys for the authentication + # dc = DeviceCloud(api_key_id='my-id', api_key_secret='my-secret') + # show the MAC address of all devices that are currently connected # - # This is done using the device cloud DeviceCore functionality + # This is done using Device Cloud DeviceCore functionality print "== Connected Devices ==" - for device in dc.devicecore.list_devices(): + for device in dc.devicecore.get_devices(): if device.is_connected(): print device.get_mac() # get the name and current value of all data streams having values # with a floating point type # - # This is done using the device cloud stream functionality + # This is done using Device Cloud stream functionality for stream in dc.streams.get_streams(): if stream.get_data_type().lower() in ('float', 'double'): print "%s -> %s" % (stream.get_stream_id(), stream.get_current_value()) diff --git a/docs/monitor.rst b/docs/monitor.rst new file mode 100644 index 0000000..31356d0 --- /dev/null +++ b/docs/monitor.rst @@ -0,0 +1,18 @@ +Monitor API +=========== + +Monitor Overview +---------------- + +Provide access to Device Cloud monitor API which can be used to +subscribe to topics to receive notifications when data is received +on Device Cloud. + +SCI API Documentation +--------------------- + +.. automodule:: devicecloud.monitor + :members: + +.. automodule:: devicecloud.monitor_tcp + :members: diff --git a/docs/sci.rst b/docs/sci.rst index e3281ff..7f7dbb2 100644 --- a/docs/sci.rst +++ b/docs/sci.rst @@ -4,8 +4,8 @@ SCI (Server Command Interface) API SCI Overview ------------ -Provide access to the device cloud Server Command Interface used for -sending messages to devices connected to the device cloud. +Provide access to Device Cloud Server Command Interface used for +sending messages to devices connected to Device Cloud. SCI API Documentation --------------------- diff --git a/docs/streams.rst b/docs/streams.rst index f78628c..5cbef2d 100644 --- a/docs/streams.rst +++ b/docs/streams.rst @@ -4,12 +4,12 @@ Streams API Streams Overview ---------------- -Data Streams on the device cloud provide a mechanism for storing time-series +Data Streams on Device Cloud provide a mechanism for storing time-series values over a long period of time. Each individual value in the time series is known as a Data Point. -There are a few basic operations supported by the device cloud on streams which -are supported by the device cloud and this library. Here we give examples of +There are a few basic operations supported by Device Cloud on streams which +are supported by Device Cloud and this library. Here we give examples of each. Listing Streams @@ -98,7 +98,7 @@ DataPoint objects The :class:`.DataPoint` class encapsulates all information required for both writing data points as well as retrieving information about data -points stored on the device cloud. +points stored on Device Cloud. API Documentation ----------------- diff --git a/docs/ws.rst b/docs/ws.rst new file mode 100644 index 0000000..fe0bb81 --- /dev/null +++ b/docs/ws.rst @@ -0,0 +1,42 @@ +Direct Web Services API +======================= + +Device Cloud exposes a large set of functionality to users and the +python-devicecloud library seeks to provide convenient and complete +APIs for a majority of these. However, there are APIs which the library +does not cover; some may have coverage in the future and others may never +have direct support in the library. + +The "ws" API provides a mechanism for directly making calls to +unsupported web services APIs. An example of the syntax exposed by the library +is probably best demonstrated with an example. + +An example of an API not currently supported by the library is the `Alarms API +`_. +This API is a "legacy" API (it is not prefixed with a "v1") and its basic interface is GET, POST, PUT, and +DELETE of the path "/ws/Alarm". The API returns response and expects payloads to be in XML according +to a format described in the documentation:: + + # List alarms + # + # Retrieves results of GET to /ws/Alarms with authentication and will raise + # the standard exceptions in the case of a failure response. + # + >>> dc = devicecloud.DeviceCloud('user', 'pass') + >>> response = dc.ws.Alarm.get() + >>> print response.content + ... A bunch of XML ... + >>> print dc.ws.Alarm.get_json() + ... A bunch of JSON ... + >>> print list(dc.ws.Alarm.iter_json_pages()) + ... All Alarms over all pages with result as list of dictionaries ... + + # + # Note that in the syntactic sugar may fall short. In those cases, you + # may need to fall back to using the underlying DeviceConnection + # + >>> alarm_id = 10 + >>> print dc.get_connection().get_json("/ws/Alarm/{}".format(alarm_id)) + ... Some JSON ... + +For more details, refer to the documentation on the methods for :py:class:`devicecloud.DeviceCloudConnection`. diff --git a/inttest.sh b/inttest.sh new file mode 100755 index 0000000..55ed206 --- /dev/null +++ b/inttest.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# This script will run all unit and integration +# tests. In order to run the integration tests, we +# need to get a username and password, which is what +# this script does. The rest of the work is done +# by the 'toxtest.sh' script +# + +export RUN_INTEGRATION_TESTS=yes +if [ -z $DC_USERNAME ]; then + echo -n 'username: ' + read username + export DC_USERNAME="$username" +fi + +if [ -z $DC_PASSWORD ]; then + echo -n 'password: ' + read -s password + export DC_PASSWORD="$password" +fi + +./toxtest.sh diff --git a/make.bat b/make.bat new file mode 100755 index 0000000..aad76c5 --- /dev/null +++ b/make.bat @@ -0,0 +1,16 @@ +@echo off + +rem Install/Upgrade Build Tools and Dependencies +pip install --upgrade setuptools wheel +pip install --upgrade twine + +rem Build distribution +python setup.py sdist bdist_wheel + +rem Upload to public Pypi Server +python -m twine upload dist/* + +rem Upload to Internal Pypi Server +python setup.py sdist upload -i http://pypi.digi.com/simple + +rem "Package uploaded to Pypi Servers" diff --git a/make.sh b/make.sh new file mode 100755 index 0000000..3f9a174 --- /dev/null +++ b/make.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# Install/Upgrade Build Tools and Dependencies +python -m pip install --upgrade setuptools wheel +python -m pip install --upgrade twine + +# Build distribution +python setup.py sdist bdist_wheel + +# Upload to public Pypi Server +python -m twine upload dist/* + +# Upload to Internal Pypi Server +python setup.py sdist upload -i http://pypi.digi.com/simple + +echo "Package uploaded to Pypi Servers" + diff --git a/requirements.txt b/requirements.txt index 98dd2ba..02bf492 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,9 @@ -six>=1.7.3 -requests>=2.2 -arrow>=0.4.4 +arrow>=1.2.2 +backports.functools-lru-cache>=1.5 +certifi>=2021.10.8 +chardet>=4.0.0 +idna>=3.3 +python-dateutil>=2.8.2 +requests>=2.27.1 +six>=1.16.0 +urllib3>=1.22 diff --git a/setup.py b/setup.py index d6b00ad..1f8bc9e 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,21 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. + import re import os from setuptools import setup, find_packages +VERSIONFILE = "devicecloud/version.py" def get_version(): # In order to get the version safely, we read the version.py file # as text. This is necessary as devicecloud/__init__.py uses # things that won't yet be present when the package is being # installed. - verstrline = open("devicecloud/version.py", "r").read() + verstrline = open(VERSIONFILE, "r").read() version_string_re = re.compile(r"^__version__ = ['\"]([^'\"]*)['\"]", re.MULTILINE) match = version_string_re.search(verstrline) if match: @@ -22,51 +29,36 @@ def get_long_description(): try: import subprocess import pandoc - - process = subprocess.Popen( - ['which pandoc'], - shell=True, - stdout=subprocess.PIPE, - universal_newlines=True) - - pandoc_path = process.communicate()[0] - pandoc_path = pandoc_path.strip('\n') - - pandoc.core.PANDOC_PATH = pandoc_path - doc = pandoc.Document() - doc.markdown = long_description - long_description = doc.rst - open("README.rst", "w").write(doc.rst) + doc.markdown = long_description.encode('utf-8') + open("README.rst", "wb").write(doc.rst) except: - if os.path.exists("README.rst"): - long_description = open("README.rst").read() - else: - print("Could not find pandoc or convert properly") - print(" make sure you have pandoc (system) and pyandoc (python module) installed") + print("Could not find pandoc or convert properly") + print(" make sure you have pandoc (system) and pyandoc (python module) installed") return long_description - setup( name="devicecloud", version=get_version(), - description="Python API to the Device Cloud by Etherios", + description="Python API to the Digi Device Cloud", + long_description_content_type='text/markdown', long_description=get_long_description(), - url="https://github.com/etherios/python-devicecloud", - author="Etherios, Inc.", - author_email="paul.osborne@etherios.com", # TODO: mailing list? + url="https://github.com/digidotcom/python-devicecloud", + author="Digi International Inc.", + author_email="brandon.moser@digi.com", packages=find_packages(), install_requires=open('requirements.txt').read().split(), classifiers=[ - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.2", - "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "Topic :: Software Development :: Libraries", "Operating System :: OS Independent", ], diff --git a/test-requirements.txt b/test-requirements.txt index a5dd38d..5133248 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,8 +1,4 @@ -r requirements.txt -tox +mock nose httpretty -pyandoc -sphinx -coverage -sphinx_rtd_theme diff --git a/tox.ini b/tox.ini index 87527c9..3b236b2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,18 +1,18 @@ [tox] -envlist = py27,py32,py33,py34,pypy +envlist = py27,py34,py35,py36,py37,pypy,pypy3 [testenv] -deps= - -rrequirements.txt - httpretty - nose -commands=nosetests +passenv = * +deps = +commands = + pip install -r test-requirements.txt + nosetests -m '^(int|unit)?[Tt]est' [testenv:coverage] deps= {[testenv]deps} - coverage - coveralls + coverage==3.7.1 + python-coveralls commands = coverage run --branch --omit={envdir}/* {envbindir}/nosetests coveralls diff --git a/toxtest.sh b/toxtest.sh index a7cac3b..8605c71 100755 --- a/toxtest.sh +++ b/toxtest.sh @@ -6,17 +6,22 @@ # testing using each of those interpreters. # -pyversions=(2.7.7 - 3.2.5 - 3.3.5 - 3.4.1 - pypy-2.3.1) +pyversions=(2.7.16 + 3.4.10 + 3.5.7 + 3.6.8 + 3.7.3 + pypy2.7-6.0.0 + pypy3.5-6.0.0) # first make sure that pyenv is installed if [ ! -s "$HOME/.pyenv/bin/pyenv" ]; then - curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash + curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash fi +# Update pyenv (required for new python versions to be available) +pyenv update + # add pyenv to our path and initialize (if this has not already been done) export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)"