From 7c111cf24deb6668cba8503e09bb8b568520309b Mon Sep 17 00:00:00 2001 From: Stephen Stack Date: Wed, 24 Sep 2014 11:04:30 -0500 Subject: [PATCH 001/140] API: Change list_devices to get_devices and add get_device call. --- devicecloud/devicecore.py | 42 +++++++++++++++++++++++------ devicecloud/test/test_devicecore.py | 37 +++++++++++++------------ 2 files changed, 54 insertions(+), 25 deletions(-) diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index 5ddcd2a..5437b3c 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -15,15 +15,41 @@ class DeviceCoreAPI(APIBase): def __init__(self, conn, sci): APIBase.__init__(self, conn) self._sci = sci + self._devices_cache = None - def list_devices(self): - """Retrieve a list of :class:`Device` objects for this device cloud account""" - devicecore_data = self._conn.get_json("/ws/DeviceCore") - json_dump = devicecore_data["items"] - devices = [] - for device_json in json_dump: - devices.append(Device(self._conn, self._sci, device_json)) - return devices + def get_devices(self, cached=True): + """Retrieve a dict of :class:`Device` objects for this device cloud account + + :param cached: Retrieve a cached dictionary of the devices + :returns: Dictionary of devices in form {: :class:`~Device`} + """ + if (self._devices_cache is None) or (cached is False): + self._devices_cache = dict() + devicecore_data = self._conn.get_json("/ws/DeviceCore") + json_dump = devicecore_data["items"] + for device_json in json_dump: + device = Device(self._conn, self._sci, device_json) + self._devices_cache[device.get_mac()] = device + return self._devices_cache + + def get_device(self, mac): + """Get a reference to a single connected device + + :param mac: Mac address of the device in the form xx:xx:xx:xx:xx:xx + :returns: :class:`~Device` object or None + """ + + if self._devices_cache is None: + # No cache available, grab a fresh one and get the device if available + return self.get_devices(cached=False).get(mac) + else: + # Try the existing cache + found = self._devices_cache.get(mac) + if found: + return found + else: + # Update the cache and return device if available + return self.get_devices(cached=False).get(mac) class Device(object): diff --git a/devicecloud/test/test_devicecore.py b/devicecloud/test/test_devicecore.py index 97c1415..335d72a 100644 --- a/devicecloud/test/test_devicecore.py +++ b/devicecloud/test/test_devicecore.py @@ -82,29 +82,16 @@ class TestDeviceCore(HttpTestBase): - def _get_device(self, mac): - devices = self.dc.devicecore.list_devices() - self.assertEqual(len(devices), 2) - - # get a ref to device with mac "00:40:9D:58:17:5B" - for device in devices: - if device.get_mac() == mac: - break - else: - self.fail("No device with expected MAC address") - - return device - def test_dc_get_devices(self): self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - devices = self.dc.devicecore.list_devices() + devices = self.dc.devicecore.get_devices() self.assertEqual(len(devices), 2) self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - dev1 = self._get_device("00:40:9D:58:17:5B") + dev1 = self.dc.devicecore.get_device("00:40:9D:58:17:5B") self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - dev2 = self._get_device("00:1d:09:2b:7d:8c") + dev2 = self.dc.devicecore.get_device("00:1d:09:2b:7d:8c") self.assertEqual(dev1.get_mac(), "00:40:9D:58:17:5B") self.assertEqual(dev1.get_mac_last4(), "175B") @@ -143,12 +130,28 @@ def test_refresh_from_cache(self): get_devices_update["items"][0]["dpDeviceType"] = "Turboencabulator" del get_devices_update["items"][1] # remove the other item... close enough self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - device = self._get_device("00:40:9D:58:17:5B") + device = self.dc.devicecore.get_device("00:40:9D:58:17:5B") self.prepare_json_response("GET", "/ws/DeviceCore/702077", get_devices_update) self.assertEqual(device.get_device_type(), "ConnectPort X5 R") self.assertEqual(device.get_device_type(False), "Turboencabulator") self.assertEqual(device.get_device_type(), "Turboencabulator") # make sure cache updated + def test_get_device_in_cache(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.dc.devicecore.get_devices() + dev = self.dc.devicecore.get_device("00:40:9D:58:17:5B") + self.assertEqual(dev.get_mac(), "00:40:9D:58:17:5B") + + def test_get_device_not_in_cache(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + dev = self.dc.devicecore.get_device("00:40:9D:58:17:5B") + self.assertEqual(dev.get_mac(), "00:40:9D:58:17:5B") + + def test_get_device_not_on_account(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + dev = self.dc.devicecore.get_device("xx:xx:xx:xx:xx:xx") + self.assertEqual(dev, None) + if __name__ == '__main__': unittest.main() From ee40cf0f1a976d91da97fcda3170d4f44f55ee50 Mon Sep 17 00:00:00 2001 From: Stephen Stack Date: Wed, 8 Oct 2014 00:04:44 -0500 Subject: [PATCH 002/140] Docs: Update references to `list_devices` --- README.md | 2 +- devicecloud/__init__.py | 2 +- docs/index.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a994f1c..83fdc17 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ dc = DeviceCloud('user', 'pass') # # This is done using the device cloud DeviceCore functionality print "== Connected Devices ==" -for device in dc.devicecore.list_devices(): +for device in dc.devicecore.get_devices(): if device.is_connected(): print device.get_mac() diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index fd8e6d6..02b6a4a 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -114,7 +114,7 @@ class DeviceCloud(object): dc = DeviceCloud('user', 'pass') if dc.has_valid_credentials(): - print dc.devicecore.list_devices() + print dc.devicecore.get_devices() From there, access to all of the device clouds features are possible. In some cases, methods for quickly performing selected actions may be provided directly via the ``DeviceCloud`` object diff --git a/docs/index.rst b/docs/index.rst index 9119adf..56314db 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -48,7 +48,7 @@ quick example of what the API looks like:: # # This is done using the device cloud DeviceCore functionality print "== Connected Devices ==" - for device in dc.devicecore.list_devices(): + for device in dc.devicecore.get_devices(): if device.is_connected(): print device.get_mac() From 2c02593e4cbb6eac7cff546b41e3ec614bb70837 Mon Sep 17 00:00:00 2001 From: Stephen Stack Date: Tue, 21 Oct 2014 00:06:52 -0500 Subject: [PATCH 003/140] Add some TODO's based on feedback. - Support varying types of lookup creteria other than 'mac' for .get_device() - Figure out how to support small requests to DC rather than getting "all devices" --- devicecloud/devicecore.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index 5437b3c..3909b88 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -39,6 +39,11 @@ def get_device(self, mac): :returns: :class:`~Device` object or None """ + # TODO: Getting all of the devices from DC can be an expensive operation, + # support small requests at some point. + + # TODO: Support multiple descriptors (args) to retrieve devices + if self._devices_cache is None: # No cache available, grab a fresh one and get the device if available return self.get_devices(cached=False).get(mac) From 19ae3a12a7e8198c9683290cca0e9892545458fa Mon Sep 17 00:00:00 2001 From: Stephen Stack Date: Tue, 21 Oct 2014 00:16:46 -0500 Subject: [PATCH 004/140] Implement paging on .get_devices() api. --- devicecloud/devicecore.py | 77 +++++++++---------- devicecloud/examples/devicecore_playground.py | 32 ++++++++ devicecloud/filedata.py | 2 +- devicecloud/test/test_devicecore.py | 29 ++----- 4 files changed, 76 insertions(+), 64 deletions(-) create mode 100644 devicecloud/examples/devicecore_playground.py diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index 3909b88..b6e0f2d 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -6,7 +6,16 @@ # Etherios, Inc. is a Division of Digi International. from devicecloud.apibase import APIBase -from devicecloud.util import iso8601_to_dt +from devicecloud.conditions import Attribute, Expression +from devicecloud.util import iso8601_to_dt, validate_type +import six + + +dev_mac = Attribute('devMac') +group_id = Attribute('grpId') +group_path = Attribute('grpPath') +dev_connectware_id = Attribute('devConnectwareId') +# TODO: Can we support location based device lookups? (e.g. lat/long?) class DeviceCoreAPI(APIBase): @@ -15,46 +24,35 @@ class DeviceCoreAPI(APIBase): def __init__(self, conn, sci): APIBase.__init__(self, conn) self._sci = sci - self._devices_cache = None - def get_devices(self, cached=True): - """Retrieve a dict of :class:`Device` objects for this device cloud account + def get_devices(self, condition=None, page_size=1000): + """Retrieve :class:`Device`(s) from this device cloud account - :param cached: Retrieve a cached dictionary of the devices - :returns: Dictionary of devices in form {: :class:`~Device`} - """ - if (self._devices_cache is None) or (cached is False): - self._devices_cache = dict() - devicecore_data = self._conn.get_json("/ws/DeviceCore") - json_dump = devicecore_data["items"] - for device_json in json_dump: - device = Device(self._conn, self._sci, device_json) - self._devices_cache[device.get_mac()] = device - return self._devices_cache - - def get_device(self, mac): - """Get a reference to a single connected device - - :param mac: Mac address of the device in the form xx:xx:xx:xx:xx:xx - :returns: :class:`~Device` object or None + :param condition: An :class:`.Expression` which defines the condition + which must be matched on the devicecore. + :returns: Generator of :class:`~Device`(s) """ - # TODO: Getting all of the devices from DC can be an expensive operation, - # support small requests at some point. - - # TODO: Support multiple descriptors (args) to retrieve devices - - if self._devices_cache is None: - # No cache available, grab a fresh one and get the device if available - return self.get_devices(cached=False).get(mac) - else: - # Try the existing cache - found = self._devices_cache.get(mac) - if found: - return found - else: - # Update the cache and return device if available - return self.get_devices(cached=False).get(mac) + condition = validate_type(condition, type(None), Expression, *six.string_types) + page_size = validate_type(page_size, *six.integer_types) + offset = 0 + remaining_size = 1 # just needs to be non-zero + + while remaining_size > 0: + req = ( + "/ws/DeviceCore?embed=true" + "&start={offset}" + "&size={page_size}".format( + page_size=page_size, + offset=offset) + ) + if condition is not None: + req = "".join([req, "&condition={0}".format(condition.compile())]) + response = self._conn.get_json(req) + offset += page_size + remaining_size = int(response.get("remainingSize", "0")) + for device_json in response.get("items", []): + yield Device(self._conn, self._sci, device_json) class Device(object): @@ -83,7 +81,8 @@ def get_device_json(self, use_cached=True): """ if not use_cached: - devicecore_data = self._conn.get_json("/ws/DeviceCore/{}".format(self.get_device_id())) + devicecore_data = self._conn.get_json( + "/ws/DeviceCore/{}".format(self.get_device_id())) self._device_json = devicecore_data["items"][0] # should only be 1 return self._device_json @@ -226,7 +225,7 @@ def get_zb_pan_id(self, use_cached=True): """Get the Zigbee PAN ID from the device if present""" return self.get_device_json(use_cached).get("dpPanId") - def get_zb_extended_address(self, use_cached=True): + def get_zb_extended_address(self, use_cached=True): """Get the Zigbee extended address of this device if present""" return self.get_device_json(use_cached).get("xpExtAddr") diff --git a/devicecloud/examples/devicecore_playground.py b/devicecloud/examples/devicecore_playground.py new file mode 100644 index 0000000..c7f33e2 --- /dev/null +++ b/devicecloud/examples/devicecore_playground.py @@ -0,0 +1,32 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2014 Etherios, Inc. All rights reserved. +# Etherios, Inc. is a Division of Digi International. +from getpass import getpass + +from devicecloud import DeviceCloud +from devicecloud.devicecore import dev_mac + + +def get_authenticated_dc(): + while True: + user = raw_input("username: ") + password = getpass("password: ") + dc = DeviceCloud(user, password, + base_url="https://test-idigi-com-2v5p9uat81qu.runscope.net") + if dc.has_valid_credentials(): + print ("Credentials accepted!") + return dc + else: + print ("Invalid username or password provided, try again") + + +if __name__ == '__main__': + dc = get_authenticated_dc() + devices = dc.devicecore.get_devices( + (dev_mac == '00:40:9D:50:B0:EA') + ) + for dev in devices: + print dev diff --git a/devicecloud/filedata.py b/devicecloud/filedata.py index 971e373..98ccd5b 100644 --- a/devicecloud/filedata.py +++ b/devicecloud/filedata.py @@ -113,7 +113,7 @@ def write_file(self, path, name, data, content_type=None, archive=False): "archive": archive_str } self._conn.put( - "/ws/FileData{path}{name}".format(path=path,name=name), + "/ws/FileData{path}{name}".format(path=path, name=name), sio.getvalue(), params=params) diff --git a/devicecloud/test/test_devicecore.py b/devicecloud/test/test_devicecore.py index 335d72a..1d6ccb1 100644 --- a/devicecloud/test/test_devicecore.py +++ b/devicecloud/test/test_devicecore.py @@ -85,13 +85,9 @@ class TestDeviceCore(HttpTestBase): def test_dc_get_devices(self): self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) devices = self.dc.devicecore.get_devices() - self.assertEqual(len(devices), 2) - - self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - dev1 = self.dc.devicecore.get_device("00:40:9D:58:17:5B") - - self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - dev2 = self.dc.devicecore.get_device("00:1d:09:2b:7d:8c") + dev1 = devices.next() + dev2 = devices.next() + self.assertRaises(StopIteration, devices.next) self.assertEqual(dev1.get_mac(), "00:40:9D:58:17:5B") self.assertEqual(dev1.get_mac_last4(), "175B") @@ -130,28 +126,13 @@ def test_refresh_from_cache(self): get_devices_update["items"][0]["dpDeviceType"] = "Turboencabulator" del get_devices_update["items"][1] # remove the other item... close enough self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - device = self.dc.devicecore.get_device("00:40:9D:58:17:5B") + devices = self.dc.devicecore.get_devices() + device = devices.next() self.prepare_json_response("GET", "/ws/DeviceCore/702077", get_devices_update) self.assertEqual(device.get_device_type(), "ConnectPort X5 R") self.assertEqual(device.get_device_type(False), "Turboencabulator") self.assertEqual(device.get_device_type(), "Turboencabulator") # make sure cache updated - def test_get_device_in_cache(self): - self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - self.dc.devicecore.get_devices() - dev = self.dc.devicecore.get_device("00:40:9D:58:17:5B") - self.assertEqual(dev.get_mac(), "00:40:9D:58:17:5B") - - def test_get_device_not_in_cache(self): - self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - dev = self.dc.devicecore.get_device("00:40:9D:58:17:5B") - self.assertEqual(dev.get_mac(), "00:40:9D:58:17:5B") - - def test_get_device_not_on_account(self): - self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - dev = self.dc.devicecore.get_device("xx:xx:xx:xx:xx:xx") - self.assertEqual(dev, None) - if __name__ == '__main__': unittest.main() From e2e643891c0323b9ea92d7692708503f56a22df3 Mon Sep 17 00:00:00 2001 From: Stephen Stack Date: Tue, 21 Oct 2014 00:33:37 -0500 Subject: [PATCH 005/140] Fix python 3/2 generator compatibility. (i.e. .next()) --- devicecloud/test/test_devicecore.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/devicecloud/test/test_devicecore.py b/devicecloud/test/test_devicecore.py index 1d6ccb1..581a3eb 100644 --- a/devicecloud/test/test_devicecore.py +++ b/devicecloud/test/test_devicecore.py @@ -10,6 +10,7 @@ from dateutil.tz import tzutc from devicecloud.test.test_utilities import HttpTestBase +import six EXAMPLE_GET_DEVICES = { @@ -85,9 +86,9 @@ class TestDeviceCore(HttpTestBase): def test_dc_get_devices(self): self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) devices = self.dc.devicecore.get_devices() - dev1 = devices.next() - dev2 = devices.next() - self.assertRaises(StopIteration, devices.next) + dev1 = six.next(devices) + dev2 = six.next(devices) + self.assertRaises(StopIteration, six.next, devices) self.assertEqual(dev1.get_mac(), "00:40:9D:58:17:5B") self.assertEqual(dev1.get_mac_last4(), "175B") @@ -127,7 +128,7 @@ def test_refresh_from_cache(self): del get_devices_update["items"][1] # remove the other item... close enough self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) devices = self.dc.devicecore.get_devices() - device = devices.next() + device = six.next(devices) self.prepare_json_response("GET", "/ws/DeviceCore/702077", get_devices_update) self.assertEqual(device.get_device_type(), "ConnectPort X5 R") self.assertEqual(device.get_device_type(False), "Turboencabulator") From 3fe7a54c0dcc3d9d84d42626a6ad90d8dd031992 Mon Sep 17 00:00:00 2001 From: Stephen Stack Date: Tue, 21 Oct 2014 11:51:06 -0500 Subject: [PATCH 006/140] Paging test for .get_devices() and update docs. --- devicecloud/__init__.py | 2 +- devicecloud/devicecore.py | 6 +++++ devicecloud/test/test_devicecore.py | 37 +++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index 02b6a4a..7014739 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -114,7 +114,7 @@ class DeviceCloud(object): dc = DeviceCloud('user', 'pass') if dc.has_valid_credentials(): - print dc.devicecore.get_devices() + print list(dc.devicecore.get_devices()) From there, access to all of the device clouds features are possible. In some cases, methods for quickly performing selected actions may be provided directly via the ``DeviceCloud`` object diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index b6e0f2d..acc74ad 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -28,8 +28,14 @@ def __init__(self, conn, sci): def get_devices(self, condition=None, page_size=1000): """Retrieve :class:`Device`(s) from this device cloud account + .. note:: + + This method returns a generator. + + :param condition: An :class:`.Expression` which defines the condition which must be matched on the devicecore. + :param int page_size: The number of results to fetch in a single page. :returns: Generator of :class:`~Device`(s) """ diff --git a/devicecloud/test/test_devicecore.py b/devicecloud/test/test_devicecore.py index 581a3eb..5c5c604 100644 --- a/devicecloud/test/test_devicecore.py +++ b/devicecloud/test/test_devicecore.py @@ -82,6 +82,33 @@ } +GET_DEVICES_PAGE1 = """\ +{ + "resultTotalRows": "2", + "requestedStartRow": "0", + "resultSize": "1", + "requestedSize": "1", + "remainingSize": "1", + "items": [ + {"id": {"devId": "702077","devVersion": "6"},"devRecordStartDate": "2013-02-28T19:54:00.000Z","devMac": "00:40:9D:58:17:5B","devCellularModemId": "354374042391400","devConnectwareId": "00000000-00000000-00409DFF-FF58175B","cstId": "1872","grpId": "2331","devEffectiveStartDate": "2013-02-28T19:53:00.000Z","devTerminated": "false","dvVendorId": "4261412864","dpDeviceType": "ConnectPort X5 R","dpFirmwareLevel": "34537482","dpFirmwareLevelDesc": "2.15.0.10","dpRestrictedStatus": "0","dpLastKnownIp": "10.35.1.107","dpGlobalIp": "204.182.3.237","dpConnectionStatus": "0","dpLastConnectTime": "2013-04-08T04:01:20.633Z","dpContact": "","dpDescription": "","dpLocation": "","dpMapLat": "34.964465","dpMapLong": "40.268198","dpServerId": "","dpZigbeeCapabilities": "0","dpCapabilities": "6707","grpPath": "","dpLastDisconnectTime": "2013-04-16T19:46:06.557Z"} + ] + } +""" + +GET_DEVICES_PAGE2 = """\ +{ + "resultTotalRows": "2", + "requestedStartRow": "1", + "resultSize": "1", + "requestedSize": "1", + "remainingSize": "0", + "items": [ + {"id": {"devId": "702078","devVersion": "6"},"devRecordStartDate": "2013-02-28T19:54:00.000Z","devMac": "00:40:9D:58:17:5B","devCellularModemId": "354374042391400","devConnectwareId": "00000000-00000000-00409DFF-FF58175B","cstId": "1872","grpId": "2331","devEffectiveStartDate": "2013-02-28T19:53:00.000Z","devTerminated": "false","dvVendorId": "4261412864","dpDeviceType": "ConnectPort X5 R","dpFirmwareLevel": "34537482","dpFirmwareLevelDesc": "2.15.0.10","dpRestrictedStatus": "0","dpLastKnownIp": "10.35.1.107","dpGlobalIp": "204.182.3.237","dpConnectionStatus": "0","dpLastConnectTime": "2013-04-08T04:01:20.633Z","dpContact": "","dpDescription": "","dpLocation": "","dpMapLat": "34.964465","dpMapLong": "40.268198","dpServerId": "","dpZigbeeCapabilities": "0","dpCapabilities": "6707","grpPath": "","dpLastDisconnectTime": "2013-04-16T19:46:06.557Z"} + ] + } +""" + + class TestDeviceCore(HttpTestBase): def test_dc_get_devices(self): self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) @@ -122,6 +149,16 @@ def test_dc_get_devices(self): self.assertEqual(dev1.get_provision_id(), None) self.assertEqual(dev1.get_current_connect_pw(), None) + def test_dc_get_devices_paged(self): + self.prepare_response("GET", "/ws/DeviceCore", GET_DEVICES_PAGE1) + gen = self.dc.devicecore.get_devices(page_size=1) + dev1 = six.next(gen) + self.prepare_response("GET", "/ws/DeviceCore", GET_DEVICES_PAGE2) + dev2 = six.next(gen) + self.assertRaises(StopIteration, six.next, gen) + self.assertEqual(dev1.get_device_id(), '702077') + self.assertEqual(dev2.get_device_id(), '702078') + def test_refresh_from_cache(self): get_devices_update = copy.deepcopy(EXAMPLE_GET_DEVICES) get_devices_update["items"][0]["dpDeviceType"] = "Turboencabulator" From ca53ca3fda7085c13683e855bd14281089ad41e8 Mon Sep 17 00:00:00 2001 From: Stephen Stack Date: Tue, 21 Oct 2014 12:12:55 -0500 Subject: [PATCH 007/140] Add test with condition specified for .get_devices() --- devicecloud/examples/devicecore_playground.py | 2 +- devicecloud/test/test_devicecore.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/devicecloud/examples/devicecore_playground.py b/devicecloud/examples/devicecore_playground.py index c7f33e2..337c2a4 100644 --- a/devicecloud/examples/devicecore_playground.py +++ b/devicecloud/examples/devicecore_playground.py @@ -29,4 +29,4 @@ def get_authenticated_dc(): (dev_mac == '00:40:9D:50:B0:EA') ) for dev in devices: - print dev + print(dev) diff --git a/devicecloud/test/test_devicecore.py b/devicecloud/test/test_devicecore.py index 5c5c604..dbcf58f 100644 --- a/devicecloud/test/test_devicecore.py +++ b/devicecloud/test/test_devicecore.py @@ -9,7 +9,9 @@ import unittest from dateutil.tz import tzutc +from devicecloud.devicecore import dev_mac from devicecloud.test.test_utilities import HttpTestBase +import httpretty import six @@ -159,6 +161,16 @@ def test_dc_get_devices_paged(self): self.assertEqual(dev1.get_device_id(), '702077') self.assertEqual(dev2.get_device_id(), '702078') + def test_dc_get_devices_with_condition(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + gen = self.dc.devicecore.get_devices(dev_mac == 'xx:xx:xx:xx:xx', page_size=1) + six.next(gen) + qs = httpretty.last_request().querystring + self.assertEqual(qs['condition'][0], "devMac='xx:xx:xx:xx:xx'") + self.assertEqual(qs['size'][0], "1") + self.assertEqual(qs['embed'][0], "true") + self.assertEqual(qs['start'][0], "0") + def test_refresh_from_cache(self): get_devices_update = copy.deepcopy(EXAMPLE_GET_DEVICES) get_devices_update["items"][0]["dpDeviceType"] = "Turboencabulator" From 7529d57527a2f416a09889e49d2de2e20d3cc4d2 Mon Sep 17 00:00:00 2001 From: Stephen Stack Date: Tue, 21 Oct 2014 12:33:47 -0500 Subject: [PATCH 008/140] PYTHON-24: Group operations (add and remove) --- devicecloud/devicecore.py | 30 +++++++++++++++++++++++++-- devicecloud/test/test_devicecore.py | 21 +++++++++++++++++++ docs/cookbook.rst | 32 +++++++++++++++++++++++------ 3 files changed, 75 insertions(+), 8 deletions(-) diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index acc74ad..d10aa2d 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -18,6 +18,15 @@ # TODO: Can we support location based device lookups? (e.g. lat/long?) +ADD_GROUP_TEMPLATE = \ +""" + + {connectware_id} + {group_path} + +""" + + class DeviceCoreAPI(APIBase): """Encapsulate DeviceCore interface""" @@ -66,8 +75,6 @@ class Device(object): # TODO: provide ability to set/update available data items # TODO: add/remove tags - # TODO: add device to group - # TODO: remove device from a group # TODO: provision a new device (probably add top-level method for this) def __init__(self, conn, sci, device_json): @@ -248,3 +255,22 @@ def get_provision_id(self, use_cached=True): def get_current_connect_pw(self, use_cached=True): """Get the current connection password for this device""" return self.get_device_json(use_cached).get("dpCurrentConnectPw") + + def add_to_group(self, group_path): + """Add a device to a group, if the group doesn't exist it is created + + :param group_path: Path or "name" of the group + """ + + if self.get_group_path() != group_path: + post_data = ADD_GROUP_TEMPLATE.format(connectware_id=self.get_connectware_id(), + group_path=group_path) + self._conn.put('/ws/DeviceCore', post_data) + + def remove_from_group(self): + """Place a device back into the root group""" + + if self.get_group_path() != '': + post_data = ADD_GROUP_TEMPLATE.format(connectware_id=self.get_connectware_id(), + group_path='') + self._conn.put('/ws/DeviceCore', post_data) diff --git a/devicecloud/test/test_devicecore.py b/devicecloud/test/test_devicecore.py index dbcf58f..0af1bfb 100644 --- a/devicecloud/test/test_devicecore.py +++ b/devicecloud/test/test_devicecore.py @@ -12,6 +12,7 @@ from devicecloud.devicecore import dev_mac from devicecloud.test.test_utilities import HttpTestBase import httpretty +from devicecloud.devicecore import ADD_GROUP_TEMPLATE import six @@ -183,6 +184,26 @@ def test_refresh_from_cache(self): self.assertEqual(device.get_device_type(False), "Turboencabulator") self.assertEqual(device.get_device_type(), "Turboencabulator") # make sure cache updated + def test_add_device_to_group(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + dev.add_to_group('testgrp') + expected = ADD_GROUP_TEMPLATE.format(connectware_id=dev.get_connectware_id(), + group_path='testgrp') + self.assertEqual(six.b(expected), httpretty.last_request().body) + + def test_remove_device_from_group(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + dev.get_group_path = lambda: 'something other than empty string' + dev.remove_from_group() + expected = ADD_GROUP_TEMPLATE.format(connectware_id=dev.get_connectware_id(), + group_path='') + self.assertEqual(six.b(expected), httpretty.last_request().body) if __name__ == '__main__': unittest.main() diff --git a/docs/cookbook.rst b/docs/cookbook.rst index b3e0bc9..fe7dcee 100644 --- a/docs/cookbook.rst +++ b/docs/cookbook.rst @@ -14,8 +14,8 @@ created with something like so:: dc = DeviceCloud(, ) -Creating Streams and Data Points -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Streams - Creating Streams and Data Points +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ For this example, let's create a DataStream that represents a class room. As students enter the classroom there is a DataPoint representing them written to the stream. @@ -65,16 +65,16 @@ Finally, let's print the name of the student who most recently entered the class print json.loads(most_recent_student.get_data())['name'] # Prints 'Henry' -Delete a Stream -^^^^^^^^^^^^^^^^ +Streams - Deleting +^^^^^^^^^^^^^^^^^^^^ Let's delete the classroom stream from the above example so we can start fresh in the next example:: classroom.delete() -Roll-up Data -^^^^^^^^^^^^^^ +Streams - Roll-up Data +^^^^^^^^^^^^^^^^^^^^^^^^ Roll-up data is a way to group data points based on time intervals in which they were written to the cloud. From our previous example lets figure out which students @@ -143,3 +143,23 @@ number of students who entered the classroom that hour:: 22: 6, 23: 11} + +Device Core - Groups +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. note:: + + This assumes your device is provisioned. + +First, get a reference to the device which you would like to add a specific group:: + + device = devicecore.get_device('00:40:9D:50:B0:EA') + +Then you can add it to a group and fetch it to make sure it works:: + + device.add_to_group('mygroup') + device.get_group_path() # prints 'mygroup' (the DC sometimes needs a second to catch up) + +Or remove it:: + + device.remove_from_group() From c6948277d56415af6b73397e9ecae1bc37cca16f Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Thu, 23 Oct 2014 00:26:39 -0500 Subject: [PATCH 009/140] docs: fix sphinx warnings and expand on docs for get_devices() This started as just addressing warnings, but I decided to add some examples and further clarification to get_devices() while I was making changes. This is likely to be one of the methods that almost every developer will use and I thought the addition of some additional examples was justifiable. --- devicecloud/devicecore.py | 25 +++++++++++++++++++------ docs/_static/.keepme | 0 docs/cookbook.rst | 4 ++-- 3 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 docs/_static/.keepme diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index acc74ad..82545ca 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -26,17 +26,30 @@ def __init__(self, conn, sci): self._sci = sci def get_devices(self, condition=None, page_size=1000): - """Retrieve :class:`Device`(s) from this device cloud account + """Iterates over each :class:`Device` for this device cloud account - .. note:: + Examples:: - This method returns a generator. + # get a list of all devices + all_devices = list(dc.devicecore.get_devices()) + # build a mapping of devices by their vendor id using a + # dict comprehension + devices = dc.devicecore.get_devices() # generator object + devs_by_vendor_id = {d.get_vendor_id(): d for d in devices} + + # iterate over all devices in 'minnesota' group and + # print the device mac and location + for device in dc.get_devices(group_path == 'minnesota'): + print "%s at %s" % (device.get_mac(), device.get_location()) :param condition: An :class:`.Expression` which defines the condition - which must be matched on the devicecore. - :param int page_size: The number of results to fetch in a single page. - :returns: Generator of :class:`~Device`(s) + which must be matched on the devicecore. If unspecified, + an iterator over all devices will be returned. + :param int page_size: The number of results to fetch in a + single page. In general, the default will suffice. + :returns: Iterator over each :class:`~Device` in this device cloud + account in the form of a generator object. """ condition = validate_type(condition, type(None), Expression, *six.string_types) diff --git a/docs/_static/.keepme b/docs/_static/.keepme new file mode 100644 index 0000000..e69de29 diff --git a/docs/cookbook.rst b/docs/cookbook.rst index b3e0bc9..8daa9cc 100644 --- a/docs/cookbook.rst +++ b/docs/cookbook.rst @@ -6,8 +6,8 @@ For more granular or specific examples of API usage check out the individual API .. note:: - There are also examples checked into source control under /devicecloud/examples/*_playground.py - which will provide additional example uses of the library. + There are also examples checked into source control under /devicecloud/examples/\*_playground.py + which will provide additional example uses of the library. Each example will assume an instance of :class:`devicecloud.DeviceCloud` has been created with something like so:: From b52b0255b45f119fa173656a5d61cc8b9fb9e909 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Thu, 23 Oct 2014 00:57:15 -0500 Subject: [PATCH 010/140] docs: add note about pyenv dependencies for running ./toxtest.sh --- HACKING.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/HACKING.md b/HACKING.md index 07cdde8..d334e60 100644 --- a/HACKING.md +++ b/HACKING.md @@ -44,7 +44,10 @@ against all supported versions of python, just do the following: $ ./toxtest.sh This might take awhile the first time as it will build from source a -version of the interpreter for each version supported. +version of the interpreter for each version supported. If you recieve +errors from pyenv, there may be addition dependencies required. +Please visit https://github.com/yyuu/pyenv/wiki/Common-build-problems +for additional pointers. Build the Documentation From 2e83ec30bc733151dcf6dc90e4021a95bfd00ef8 Mon Sep 17 00:00:00 2001 From: Stephen Stack Date: Thu, 23 Oct 2014 15:53:37 -0500 Subject: [PATCH 011/140] Invalidate device cache after adding and removing from groups. --- devicecloud/devicecore.py | 6 ++++++ devicecloud/test/test_devicecore.py | 6 ++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index d10aa2d..6494cbc 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -267,6 +267,9 @@ def add_to_group(self, group_path): group_path=group_path) self._conn.put('/ws/DeviceCore', post_data) + # Invalidate cache + self._device_json = None + def remove_from_group(self): """Place a device back into the root group""" @@ -274,3 +277,6 @@ def remove_from_group(self): post_data = ADD_GROUP_TEMPLATE.format(connectware_id=self.get_connectware_id(), group_path='') self._conn.put('/ws/DeviceCore', post_data) + + # Invalidate cache + self._device_json = None \ No newline at end of file diff --git a/devicecloud/test/test_devicecore.py b/devicecloud/test/test_devicecore.py index 0af1bfb..328c623 100644 --- a/devicecloud/test/test_devicecore.py +++ b/devicecloud/test/test_devicecore.py @@ -189,9 +189,10 @@ def test_add_device_to_group(self): self.prepare_response("PUT", "/ws/DeviceCore", '') gen = self.dc.devicecore.get_devices(page_size=1) dev = six.next(gen) - dev.add_to_group('testgrp') expected = ADD_GROUP_TEMPLATE.format(connectware_id=dev.get_connectware_id(), group_path='testgrp') + dev.add_to_group('testgrp') + self.assertIsNone(dev._device_json) self.assertEqual(six.b(expected), httpretty.last_request().body) def test_remove_device_from_group(self): @@ -200,9 +201,10 @@ def test_remove_device_from_group(self): gen = self.dc.devicecore.get_devices(page_size=1) dev = six.next(gen) dev.get_group_path = lambda: 'something other than empty string' - dev.remove_from_group() expected = ADD_GROUP_TEMPLATE.format(connectware_id=dev.get_connectware_id(), group_path='') + dev.remove_from_group() + self.assertIsNone(dev._device_json) self.assertEqual(six.b(expected), httpretty.last_request().body) if __name__ == '__main__': From 2d77af238ac0ed91fa19fc2a39ef21c9ab24df11 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 24 Oct 2014 00:54:35 -0500 Subject: [PATCH 012/140] refactor: extract method for iterating over paged results set Several APIs in the library had code that was very near to being identitical. This change extracts that logic into the DeviceCloudConnection object. New tests are added against this core code and existing tests for existing functionality pass without modification (as they should). --- devicecloud/__init__.py | 32 ++++++++++++ devicecloud/devicecore.py | 25 +++------- devicecloud/filedata.py | 20 ++------ devicecloud/test/test_core.py | 79 ++++++++++++++++++++++++++++++ devicecloud/test/test_utilities.py | 6 +++ 5 files changed, 127 insertions(+), 35 deletions(-) create mode 100644 devicecloud/test/test_core.py diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index 7014739..7f6b509 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -4,6 +4,7 @@ # # Copyright (c) 2014 Etherios, Inc. All rights reserved. # Etherios, Inc. is a Division of Digi International. +from devicecloud.util import validate_type from requests.auth import HTTPBasicAuth import logging @@ -12,6 +13,7 @@ import json from devicecloud.version import __version__ +import six __all__ = ( 'DeviceCloud', @@ -70,6 +72,36 @@ def _make_request(self, retries, method, url, **kwargs): err = "DC %s to %s failed - HTTP(%s)" % (method, url, response.status_code) raise DeviceCloudHttpException(response, err) + def iter_json_pages(self, path, page_size=1000, **params): + """Return an iterator over JSON items from a paginated resource + + Legacy resources (prior to V1) implemented a common paging interfaces for + several different resources. This method handles the details of iterating + over the paged result set, yielding only the JSON data for each item + within the aggregate resource. + + :param str path: The base path to the resource being requested (e.g. /ws/Group) + :param int page_size: The number of items that should be requested for each page. A larger + page_size may mean fewer HTTP requests but could also increase the time to get a first + result back from the device cloud. + :param params: These are additional query parameters that should be sent with each + request to the device cloud. + + """ + path = validate_type(path, *six.string_types) + page_size = validate_type(page_size, *six.integer_types) + + offset = 0 + remaining_size = 1 # just needs to be non-zero + while remaining_size > 0: + reqparams = {"start": offset, "size": page_size} + reqparams.update(params) + response = self.get_json(path, params=reqparams) + offset += page_size + remaining_size = int(response.get("remainingSize", "0")) + for item_json in response.get("items", []): + yield item_json + def ping(self): """Ping the Device Cloud using the authorization provided diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index b626ba9..827c911 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -63,24 +63,13 @@ def get_devices(self, condition=None, page_size=1000): condition = validate_type(condition, type(None), Expression, *six.string_types) page_size = validate_type(page_size, *six.integer_types) - offset = 0 - remaining_size = 1 # just needs to be non-zero - - while remaining_size > 0: - req = ( - "/ws/DeviceCore?embed=true" - "&start={offset}" - "&size={page_size}".format( - page_size=page_size, - offset=offset) - ) - if condition is not None: - req = "".join([req, "&condition={0}".format(condition.compile())]) - response = self._conn.get_json(req) - offset += page_size - remaining_size = int(response.get("remainingSize", "0")) - for device_json in response.get("items", []): - yield Device(self._conn, self._sci, device_json) + + params = {"embed": "true"} + if condition is not None: + params["condition"] = condition.compile() + + for device_json in self._conn.iter_json_pages("/ws/DeviceCore", page_size=page_size, **params): + yield Device(self._conn, self._sci, device_json) class Device(object): diff --git a/devicecloud/filedata.py b/devicecloud/filedata.py index 98ccd5b..9e4b737 100644 --- a/devicecloud/filedata.py +++ b/devicecloud/filedata.py @@ -47,26 +47,12 @@ def get_filedata(self, condition=None, page_size=1000): condition = validate_type(condition, type(None), Expression, *six.string_types) page_size = validate_type(page_size, *six.integer_types) - offset = 0 - remaining_size = 1 # just needs to be non-zero - if condition is None: condition = (fd_path == "~/") # home directory - while remaining_size > 0: - response = self._conn.get_json( - "/ws/FileData?embed=true" - "&start={offset}" - "&size={page_size}" - "&condition={condition}".format( - condition=condition.compile(), - page_size=page_size, - offset=offset)) - - offset += page_size - remaining_size = int(response.get("remainingSize", "0")) - for fd_json in response.get("items", []): - yield FileDataObject.from_json(self, fd_json) + params = {"embed": "true", "condition": condition.compile()} + for fd_json in self._conn.iter_json_pages("/ws/FileData", page_size=page_size, **params): + yield FileDataObject.from_json(self, fd_json) def write_file(self, path, name, data, content_type=None, archive=False): """Write a file to the file data store at the given path diff --git a/devicecloud/test/test_core.py b/devicecloud/test/test_core.py new file mode 100644 index 0000000..c912551 --- /dev/null +++ b/devicecloud/test/test_core.py @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2014 Etherios, Inc. All rights reserved. +# Etherios, Inc. is a Division of Digi International. +import json +import unittest +from devicecloud.test.test_utilities import HttpTestBase + + +TEST_BASIC_RESPONSE = """\ +{ + "resultTotalRows": "2", + "requestedStartRow": "0", + "resultSize": "2", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { "id": 1, "name": "bob" }, + { "id": 2, "name": "tim" } + ] +} +""" + +TEST_PAGED_RESPONSE_PAGE1 = """\ +{ + "resultTotalRows": "2", + "requestedStartRow": "0", + "resultSize": "1", + "requestedSize": "1", + "remainingSize": "1", + "items": [ + { "id": 1, "name": "bob" } + ] +} +""" + +TEST_PAGED_RESPONSE_PAGE2 = """\ +{ + "resultTotalRows": "2", + "requestedStartRow": "1", + "resultSize": "1", + "requestedSize": "1", + "remainingSize": "0", + "items": [ + { "id": 2, "name": "tim" } + ] +} +""" + + + +class TestDeviceCloudConnection(HttpTestBase): + + def test_iter_json_with_params(self): + it = self.dc._conn.iter_json_pages("/test/path", foo="bar", key="value") + self.prepare_response("GET", "/test/path", TEST_BASIC_RESPONSE) + self.assertEqual(len(list(it)), 2) + self.assertDictEqual(self._get_last_request_params(), { + "start": "0", + "foo": "bar", + "key": "value", + "size": "1000", + }) + + def test_iter_json_pages_paged_noparams(self): + it = self.dc._conn.iter_json_pages("/test/path", page_size=1) + self.prepare_response("GET", "/test/path", TEST_PAGED_RESPONSE_PAGE1) + self.assertEqual(it.next()["id"], 1) + self.prepare_response("GET", "/test/path", TEST_PAGED_RESPONSE_PAGE2) + self.assertEqual(it.next()["id"], 2) + self.assertDictEqual(self._get_last_request_params(), { + "size": "1", + "start": "1" + }) + +if __name__ == "__main__": + unittest.main() diff --git a/devicecloud/test/test_utilities.py b/devicecloud/test/test_utilities.py index 11a3b97..fcb024b 100644 --- a/devicecloud/test/test_utilities.py +++ b/devicecloud/test/test_utilities.py @@ -10,6 +10,7 @@ from devicecloud import DeviceCloud import httpretty +import six.moves.urllib.parse as urllib_parse class HttpTestBase(unittest.TestCase): @@ -26,6 +27,11 @@ def tearDown(self): def _get_last_request(self): return httpretty.last_request() + def _get_last_request_params(self): + # Get the query params from the last request as a dictionary + params = urllib_parse.parse_qs(urllib_parse.urlparse(self._get_last_request().path).query) + return {k: v[0] for k, v in params.items()} # convert from list values to single-value + def prepare_response(self, method, path, data, status=200, match_querystring=False): # TODO: # Should probably assert on more request headers and From 763ddb45aa14424c2afcef898532cf8ff96afc67 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 24 Oct 2014 01:01:39 -0500 Subject: [PATCH 013/140] refactor: make DeviceCloudConnection a public interface object The connection object has utility within the library but is also potentially useful for external users. The fact is, our library does not implement all features of the device cloud and likely will never. I see no reason not to make this interface part of the public API (although marked as being low-level). Also, fix some small Py3 test failures with new tests. --- devicecloud/__init__.py | 115 ++++++++++++++++++++++++++++++++-- devicecloud/test/test_core.py | 12 ++-- 2 files changed, 115 insertions(+), 12 deletions(-) diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index 7f6b509..1b3206d 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -43,11 +43,15 @@ def __init__(self, response, *args, **kwargs): self.response = response -class _DeviceCloudConnection(object): - """Encapsulate information about a connection to the device cloud +class DeviceCloudConnection(object): + """Provide low-level access to the Device Cloud web services - This class is used internally and does not represent a part of the public API - to the device cloud. + This is a convenience object that provides methods that make sending requests to the + device cloud easier. This object is used extensively within the library but can + also be used externally (for instance, to support an API exposed by the device + cloud that is not currently supported in the library). + + This object is accessible via :meth:`~DeviceCloud.get_connection`. """ @@ -112,10 +116,47 @@ def ping(self): return self.get("/ws/DeviceCore?size=1") def get(self, path, retries=0, **kwargs): + """Perform an HTTP GET request of the specified path in the device cloud + + Make an HTTP GET request against the device cloud with this accounts + credentials and base url. This method uses the + `requests `_ library + `request method `_ + and all keyword arguments will be passed on to that method. + + :param str path: The device cloud path to GET + :param int retries: The number of times the request should be retried if an + unsuccessful response is received. Most likely, you should leave this at 0. + :raises DeviceCloudHttpException: if a non-success response to the request is received + from the device cloud + :returns: A requests ``Response`` object + + """ url = self._make_url(path) return self._make_request(retries, "GET", url, **kwargs) def get_json(self, path, retries=0, **kwargs): + """Perform an HTTP GET request with JSON headers of the specified path against the device cloud + + Make an HTTP GET request against the device cloud with this accounts + credentials and base url. This method uses the + `requests `_ library + `request method `_ + and all keyword arguments will be passed on to that method. + + This method will automatically add the ``Accept: application/json`` and parse the + JSON response from the device cloud. + + :param str path: The device cloud path to GET + :param int retries: The number of times the request should be retried if an + unsuccessful response is received. Most likely, you should leave this at 0. + :raises DeviceCloudHttpException: if a non-success response to the request is received + from the device cloud + :returns: A python data structure containing the results of calling ``json.loads`` on the + body of the response from the device cloud. + + """ + url = self._make_url(path) headers = kwargs.setdefault('headers', {}) headers.update({'Accept': 'application/json'}) @@ -123,14 +164,67 @@ def get_json(self, path, retries=0, **kwargs): return json.loads(response.text) def post(self, path, data, retries=0, **kwargs): + """Perform an HTTP POST request of the specified path in the device cloud + + Make an HTTP POST request against the device cloud with this accounts + credentials and base url. This method uses the + `requests `_ library + `request method `_ + and all keyword arguments will be passed on to that method. + + :param str path: The device cloud path to POST + :param int retries: The number of times the request should be retried if an + unsuccessful response is received. Most likely, you should leave this at 0. + :param data: The data to be posted in the body of the POST request (see docs for + ``requests.post`` + :raises DeviceCloudHttpException: if a non-success response to the request is received + from the device cloud + :returns: A requests ``Response`` object + + """ url = self._make_url(path) return self._make_request(retries, "POST", url, data=data, **kwargs) def put(self, path, data, retries=0, **kwargs): + """Perform an HTTP PUT request of the specified path in the device cloud + + Make an HTTP PUT request against the device cloud with this accounts + credentials and base url. This method uses the + `requests `_ library + `request method `_ + and all keyword arguments will be passed on to that method. + + :param str path: The device cloud path to PUT + :param int retries: The number of times the request should be retried if an + unsuccessful response is received. Most likely, you should leave this at 0. + :param data: The data to be posted in the body of the POST request (see docs for + ``requests.post`` + :raises DeviceCloudHttpException: if a non-success response to the request is received + from the device cloud + :returns: A requests ``Response`` object + + """ + url = self._make_url(path) return self._make_request(retries, "PUT", url, data=data, **kwargs) - def delete(self, path, retries=0): + def delete(self, path, retries=0, **kwargs): + """Perform an HTTP DELETE request of the specified path in the device cloud + + Make an HTTP DELETE request against the device cloud with this accounts + credentials and base url. This method uses the + `requests `_ library + `request method `_ + and all keyword arguments will be passed on to that method. + + :param str path: The device cloud path to DELETE + :param int retries: The number of times the request should be retried if an + unsuccessful response is received. Most likely, you should leave this at 0. + :raises DeviceCloudHttpException: if a non-success response to the request is received + from the device cloud + :returns: A requests ``Response`` object + + """ url = self._make_url(path) return self._make_request(retries, "DELETE", url) @@ -155,7 +249,7 @@ class DeviceCloud(object): """ def __init__(self, username, password, base_url="https://login.etherios.com"): - self._conn = _DeviceCloudConnection(HTTPBasicAuth(username, password), base_url) + self._conn = DeviceCloudConnection(HTTPBasicAuth(username, password), base_url) self._streams_api = None # streams property api ref self._filedata_api = None # filedata property api ref self._devicecore_api = None # devicecore property api ref @@ -206,6 +300,15 @@ def sci(self): self._sci_api = self.get_sci_api() return self._sci_api + def get_connection(self): + """Get the low-level :class:`~DeviceCloudConnection` for this device cloud instance + + This object provides a low-level interface for making authenticated requests + to the device cloud. + + """ + return self._conn + def get_streams_api(self): """Returns a :class:`.StreamsAPI` bound to this device cloud instance diff --git a/devicecloud/test/test_core.py b/devicecloud/test/test_core.py index c912551..48b10a3 100644 --- a/devicecloud/test/test_core.py +++ b/devicecloud/test/test_core.py @@ -4,9 +4,10 @@ # # Copyright (c) 2014 Etherios, Inc. All rights reserved. # Etherios, Inc. is a Division of Digi International. -import json import unittest + from devicecloud.test.test_utilities import HttpTestBase +import six TEST_BASIC_RESPONSE = """\ @@ -50,11 +51,10 @@ """ - class TestDeviceCloudConnection(HttpTestBase): def test_iter_json_with_params(self): - it = self.dc._conn.iter_json_pages("/test/path", foo="bar", key="value") + it = self.dc.get_connection().iter_json_pages("/test/path", foo="bar", key="value") self.prepare_response("GET", "/test/path", TEST_BASIC_RESPONSE) self.assertEqual(len(list(it)), 2) self.assertDictEqual(self._get_last_request_params(), { @@ -65,11 +65,11 @@ def test_iter_json_with_params(self): }) def test_iter_json_pages_paged_noparams(self): - it = self.dc._conn.iter_json_pages("/test/path", page_size=1) + it = self.dc.get_connection().iter_json_pages("/test/path", page_size=1) self.prepare_response("GET", "/test/path", TEST_PAGED_RESPONSE_PAGE1) - self.assertEqual(it.next()["id"], 1) + self.assertEqual(six.next(it)["id"], 1) self.prepare_response("GET", "/test/path", TEST_PAGED_RESPONSE_PAGE2) - self.assertEqual(it.next()["id"], 2) + self.assertEqual(six.next(it)["id"], 2) self.assertDictEqual(self._get_last_request_params(), { "size": "1", "start": "1" From 8c0f964a9b236035e125d76e43d15b7b94f45cdf Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Sun, 26 Oct 2014 02:24:16 -0500 Subject: [PATCH 014/140] groups: add basic support for getting information about groups I've written and rewritten this functionality a few times before it got to its current state, but I think I am OK with it as it is now. A few notes: 1. When calling `get_groups`, the groups returned will not have their children associations populated. This is to allow for getting just a single group via a condition. In cases where you want to do something with the hiearchical assocaitions, one should really be using `get_group_tree_root` anyway. 2. Group objects to not provide direct access to devices. This could be added in the future, but for now I decided that it was easy enough to pass in the group path as a condition parameters to `get_devices`. --- devicecloud/devicecore.py | 162 ++++++++++++++++++ devicecloud/examples/devicecore_playground.py | 22 ++- devicecloud/test/test_devicecore.py | 74 +++++++- 3 files changed, 254 insertions(+), 4 deletions(-) diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index 827c911..742abc5 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -4,6 +4,7 @@ # # Copyright (c) 2014 Etherios, Inc. All rights reserved. # Etherios, Inc. is a Division of Digi International. +import sys from devicecloud.apibase import APIBase from devicecloud.conditions import Attribute, Expression @@ -71,6 +72,167 @@ def get_devices(self, condition=None, page_size=1000): for device_json in self._conn.iter_json_pages("/ws/DeviceCore", page_size=page_size, **params): yield Device(self._conn, self._sci, device_json) + def get_group_tree_root(self, page_size=1000): + r"""Return the root group for this accounts' group tree + + This will return the root group for this tree but with all links + between nodes (i.e. children starting from root) populated. + + Examples:: + + # print the group hierarchy to stdout + dc.devicecore.get_group_tree_root().print_subtree() + + # gather statistics about devices in each group including + # the count from its subgroups (recursively) + # + # This also shows how you can go from a group reference to devices + # for that particular group. + stats = {} # group -> devices count including children + def count_nodes(group): + count_for_this_node = \ + len(list(dc.devicecore.get_devices(group_path == group.get_path()))) + subnode_count = 0 + for child in group.get_children(): + subnode_count += count_nodes(child) + total = count_for_this_node + subnode_count + stats[group] = total + return total + count_nodes(dc.devicecore.get_group_tree_root()) + + :param int page_size: The number of results to fetch in a + single page. In general, the default will suffice. + :returns: The root group for this device cloud accounts group + hierarchy. + + """ + + # first pass, build mapping + group_map = {} # map id -> group + page_size = validate_type(page_size, *six.integer_types) + for group in self.get_groups(page_size=page_size): + group_map[group.get_id()] = group + + # second pass, find root and populate list of children for each node + root = None + for group_id, group in group_map.items(): + if group.is_root(): + root = group + else: + parent = group_map[group.get_parent_id()] + parent.add_child(group) + return root + + def get_groups(self, condition=None, page_size=1000): + """Return an iterator over all groups in this device cloud account + + Optionally, a condition can be specified to limit the number of + groups returned. + + Examples:: + + # Get all groups and print information about them + for group in dc.devicecore.get_groups(): + print group + + # Iterate over all devices which are in a group with a specific + # ID. + group = dc.devicore.get_groups(group_id == 123)[0] + for device in dc.devicecore.get_devices(group_path == group.get_path()): + print device.get_mac() + + :param condition: A condition to use when filtering the results set. If + unspecified, all groups will be returned. + :param int page_size: The number of results to fetch in a + single page. In general, the default will suffice. + :returns: Generator over the groups in this device cloud account. No + guarantees about the order of results is provided and child links + between nodes will not be populated. + + """ + query_kwargs = {} + if condition is not None: + query_kwargs["condition"] = condition.compile() + for group_data in self._conn.iter_json_pages("/ws/Group", page_size=page_size, **query_kwargs): + yield Group.from_json(group_data) + + +class Group(object): + """Provides access to information about a group in the device cloud + + .. note:: + + This is primarily a container object and does not provide any functions itself at + this time. Information from here can be used along with other APIs to, for example, + get all devices with a given group path. This may change in the future. + + """ + + def __init__(self, group_id, name, description, path, parent_id): + self._id = group_id + self._name = name + self._description = description + self._path = path + self._parent_id = parent_id + self._children = [] + + @classmethod + def from_json(cls, json_data): + """Build and return a new Group object from json data (used internally)""" + # Example Data: + # { "grpId": "11817", "grpName": "7603_Etherios", "grpDescription": "7603_Etherios root group", + # "grpPath": "\/7603_Etherios\/", "grpParentId": "1"} + return cls( + group_id=json_data["grpId"], + name=json_data["grpName"], + description=json_data.get("grpDescription", ""), + path=json_data["grpPath"], + parent_id=json_data["grpParentId"], + ) + + def __repr__(self): + return "Group(group_id={!r}, name={!r}, description{!r}, path={!r}, parent_id={!r})".format( + self._id, self._name, self._description, self._path, self._parent_id + ) + + def print_subtree(self, fobj=sys.stdout, level=0): + """Print this group node and the subtree rooted at it""" + fobj.write("{}{!r}\n".format(" " * (level * 2), self)) + for child in self.get_children(): + child.print_subtree(fobj, level + 1) + + def is_root(self): + """Return True if the group is the root for this account""" + return self.get_parent_id() == "1" + + def add_child(self, group): + """Add a child group reference to this one""" + self._children.append(group) + + def get_children(self): + """Return each child :class:`Group` of this one in a list""" + return self._children[:] + + def get_id(self): + """Get the ID of this group as as string""" + return self._id + + def get_name(self): + """Get the name of this group as a string""" + return self._name + + def get_description(self): + """Get the description of this group as a string""" + return self._description + + def get_path(self): + """Get the full path of this group as a string""" + return self._path + + def get_parent_id(self): + """Get the ID of this groups parent as a string""" + return self._parent_id + class Device(object): """Interface to a device in the device cloud""" diff --git a/devicecloud/examples/devicecore_playground.py b/devicecloud/examples/devicecore_playground.py index 337c2a4..7056115 100644 --- a/devicecloud/examples/devicecore_playground.py +++ b/devicecloud/examples/devicecore_playground.py @@ -7,7 +7,7 @@ from getpass import getpass from devicecloud import DeviceCloud -from devicecloud.devicecore import dev_mac +from devicecloud.devicecore import dev_mac, group_path def get_authenticated_dc(): @@ -15,7 +15,7 @@ def get_authenticated_dc(): user = raw_input("username: ") password = getpass("password: ") dc = DeviceCloud(user, password, - base_url="https://test-idigi-com-2v5p9uat81qu.runscope.net") + base_url="https://login.etherios.com") if dc.has_valid_credentials(): print ("Credentials accepted!") return dc @@ -23,6 +23,22 @@ def get_authenticated_dc(): print ("Invalid username or password provided, try again") +def show_group_tree(dc): + stats = {} # group -> devices count including children + def count_nodes(group): + count_for_this_node = \ + len(list(dc.devicecore.get_devices(group_path == group.get_path()))) + subnode_count = 0 + for child in group.get_children(): + subnode_count += count_nodes(child) + total = count_for_this_node + subnode_count + stats[group] = total + return total + count_nodes(dc.devicecore.get_group_tree_root()) + print(stats) + dc.devicecore.get_group_tree_root().print_subtree() + + if __name__ == '__main__': dc = get_authenticated_dc() devices = dc.devicecore.get_devices( @@ -30,3 +46,5 @@ def get_authenticated_dc(): ) for dev in devices: print(dev) + + show_group_tree(dc) diff --git a/devicecloud/test/test_devicecore.py b/devicecloud/test/test_devicecore.py index 328c623..93cd77d 100644 --- a/devicecloud/test/test_devicecore.py +++ b/devicecloud/test/test_devicecore.py @@ -9,7 +9,7 @@ import unittest from dateutil.tz import tzutc -from devicecloud.devicecore import dev_mac +from devicecloud.devicecore import dev_mac, group_id from devicecloud.test.test_utilities import HttpTestBase import httpretty from devicecloud.devicecore import ADD_GROUP_TEMPLATE @@ -111,8 +111,78 @@ } """ +EXAMPLE_GET_GROUPS = """\ +{ + "resultTotalRows": "2", + "requestedStartRow": "0", + "resultSize": "2", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { "grpId": "11817", "grpName": "7603_Etherios", "grpDescription": "7603_Etherios root group", "grpPath": "\/7603_Etherios\/", "grpParentId": "1"}, + { "grpId": "13542", "grpName": "Demo", "grpPath": "\/7603_Etherios\/Demo\/", "grpParentId": "11817"} + ] +} +""" + +EXAMPLE_GET_GROUPS_EXTENDED = """\ +{ + "resultTotalRows": "4", + "requestedStartRow": "0", + "resultSize": "4", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { "grpId": "11817", "grpName": "7603_Etherios", "grpDescription": "7603_Etherios root group", "grpPath": "\/7603_Etherios\/", "grpParentId": "1"}, + { "grpId": "13542", "grpName": "Demo", "grpPath": "\/7603_Etherios\/Demo\/", "grpParentId": "11817"}, + { "grpId": "13544", "grpName": "SubDir2", "grpPath": "\/7603_Etherios\/Demo\/SubDir2\/", "grpParentId": "13542"}, + { "grpId": "13545", "grpName": "Another Second Level", "grpDescription": "Another Second Level", "grpPath": "\/7603_Etherios\/Another Second Level\/", "grpParentId": "11817"} + ] +} +""" + + +class TestDeviceCoreGroups(HttpTestBase): + + def test_get_groups(self): + self.prepare_response("GET", "/ws/Group", EXAMPLE_GET_GROUPS) + it = self.dc.devicecore.get_groups() + + grp = six.next(it) + self.assertEqual(grp.is_root(), True) + self.assertEqual(grp.get_id(), "11817") + self.assertEqual(grp.get_name(), "7603_Etherios") + self.assertEqual(grp.get_description(), "7603_Etherios root group") + self.assertEqual(grp.get_path(), "/7603_Etherios/") + self.assertEqual(grp.get_parent_id(), "1") + + grp = six.next(it) + self.assertEqual(grp.is_root(), False) + self.assertEqual(grp.get_id(), "13542") + self.assertEqual(grp.get_name(), "Demo") + self.assertEqual(grp.get_description(), "") + self.assertEqual(grp.get_path(), "/7603_Etherios/Demo/") + self.assertEqual(grp.get_parent_id(), "11817") + + def test_repr_and_tree_print(self): + self.prepare_response("GET", "/ws/Group", EXAMPLE_GET_GROUPS_EXTENDED) + fobj = six.StringIO() + root = self.dc.devicecore.get_group_tree_root() + root.print_subtree(fobj) # the order of the traversal can vary, so just assert on the length + if six.PY2: + self.assertEqual(len(fobj.getvalue()), 513) + elif six.PY3: + self.assertEqual(len(fobj.getvalue()), 495) # no u'' on repr for strings + + def test_get_groups_condition(self): + self.prepare_response("GET", "/ws/Group", EXAMPLE_GET_GROUPS) + list(self.dc.devicecore.get_groups(group_id == "123")) + params = self._get_last_request_params() + self.assertEqual(params["condition"], "grpId='123'") + + +class TestDeviceCoreDevices(HttpTestBase): -class TestDeviceCore(HttpTestBase): def test_dc_get_devices(self): self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) devices = self.dc.devicecore.get_devices() From d2b1e59662ed9f1628ecaab593843bad73534aa4 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 26 Nov 2014 15:40:30 -0600 Subject: [PATCH 015/140] travis: fix build failure due to old version of coverage Error on travisci when running 'coverage' testenv... Traceback (most recent call last): File ".tox/coverage/bin/coveralls", line 5, in from pkg_resources import load_entry_point File "/home/travis/build/Etherios/python-devicecloud/.tox/coverage/lib/python2.7/site-packages/pkg_resources.py", line 2829, in working_set = WorkingSet._build_master() File "/home/travis/build/Etherios/python-devicecloud/.tox/coverage/lib/python2.7/site-packages/pkg_resources.py", line 451, in _build_master return cls._build_from_requirements(__requires__) File "/home/travis/build/Etherios/python-devicecloud/.tox/coverage/lib/python2.7/site-packages/pkg_resources.py", line 464, in _build_from_requirements dists = ws.resolve(reqs, Environment()) File "/home/travis/build/Etherios/python-devicecloud/.tox/coverage/lib/python2.7/site-packages/pkg_resources.py", line 639, in resolve raise DistributionNotFound(req) pkg_resources.DistributionNotFound: coverage>=3.6,<3.999 ERROR: InvocationError: '/home/travis/build/Etherios/python-devicecloud/.tox/coverage/bin/coveralls' --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 87527c9..4923fe0 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ commands=nosetests [testenv:coverage] deps= {[testenv]deps} - coverage + coverage>=3.6,<3.999 coveralls commands = coverage run --branch --omit={envdir}/* {envbindir}/nosetests From 28b947fefa8832e8e2acec5edc2fdbcd617f7fa2 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 24 Dec 2014 18:01:07 -0600 Subject: [PATCH 016/140] core: improve documentation and provide better error messages for HTTP Previously, when a call failed (for any unhandled reason), the user of the library would just get an exception indicating that a DeviceCloudHttpException occurred without much additional information. This documents the 'response' attribute (now a property) on the exception object and adds a __str__ method that will print the failing response body on failure. This is related to PYTHONDC-96. Signed-off-by: Paul Osborne --- devicecloud/__init__.py | 38 +++++++++++++++++++++++++++++++++-- devicecloud/test/test_core.py | 18 +++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index 1b3206d..04588d9 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -36,11 +36,45 @@ class DeviceCloudException(Exception): class DeviceCloudHttpException(DeviceCloudException): - """Exception raised when we failed a request to the DC over HTTP""" + """Exception raised when we failed a request to the DC over HTTP + + This exception will be raised whenever a non-success HTTP status + code is received from the device cloud and there is no other logic + in place for gracefully handling the error case. + + Often, if there is an error with a request to the device cloud, the device + cloud will respond with an error status and include additional information + about the nature of the error in the response body. This information + can be accessed via the :attr:`~response` property. + + """ def __init__(self, response, *args, **kwargs): DeviceCloudException.__init__(self, *args, **kwargs) - self.response = response + self._response = response + + def __str__(self): + return "HTTP Status {code}: {content}".format( + code=self.response.status_code, + content=self.response.content + ) + + @property + def response(self): + """Get the requests response object for the failing HTTP request + + This object will be an instance of :class:`requests.Response` which + in turn provides information including the content (body) of the response + and the HTTP status code:: + + try: + dc.sci.send_request(...) + except DeviceCloudHttpException as e: + print "HTTP Error: %s" % e.response.status_code + print e.response.content + + """ + return self._response class DeviceCloudConnection(object): diff --git a/devicecloud/test/test_core.py b/devicecloud/test/test_core.py index 48b10a3..bbce904 100644 --- a/devicecloud/test/test_core.py +++ b/devicecloud/test/test_core.py @@ -5,6 +5,7 @@ # Copyright (c) 2014 Etherios, Inc. All rights reserved. # Etherios, Inc. is a Division of Digi International. import unittest +from devicecloud import DeviceCloudHttpException from devicecloud.test.test_utilities import HttpTestBase import six @@ -50,6 +51,12 @@ } """ +TEST_ERROR_RESPONSE = six.b("""\ +\ +Invalid target. Device not found.\ +Invalid SCI request. No valid targets found.\ +""") + class TestDeviceCloudConnection(HttpTestBase): @@ -75,5 +82,16 @@ def test_iter_json_pages_paged_noparams(self): "start": "1" }) + def test_http_exception(self): + self.prepare_response("POST", "/test/path", TEST_ERROR_RESPONSE, status=400) + try: + self.dc.get_connection().post("/test/path", "bad data") + except DeviceCloudHttpException as e: + str(e) # ensure this does not stack trace at least + self.assertEqual(e.response.status_code, 400) + self.assertEqual(e.response.content, TEST_ERROR_RESPONSE) + else: + self.fail("DeviceCloudHttpException not raised") + if __name__ == "__main__": unittest.main() From 316eaf05e3908726eb74cb115a04720f59f5ef1a Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Sat, 27 Dec 2014 22:45:28 -0600 Subject: [PATCH 017/140] core: do not encode url path prior to hitting requests library This fixes an issue introduced somewhere in the past few versions of requests (which we had not frozen previously). Rather than freezing to an old version of requests, we bump the version to the latest and make the required changes in order to make the libary work with requests. This resolves PYTHONDC-97 against the issue. Signed-off-by: Paul Osborne --- devicecloud/conditions.py | 5 ++--- devicecloud/test/test_conditions.py | 8 ++++---- requirements.txt | 6 +++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/devicecloud/conditions.py b/devicecloud/conditions.py index 81046e0..a038ae8 100644 --- a/devicecloud/conditions.py +++ b/devicecloud/conditions.py @@ -16,8 +16,6 @@ from devicecloud.util import isoformat, to_none_or_dt -import six - def _quoted(value): """Return a single-quoted and escaped (percent-encoded) version of value @@ -32,7 +30,8 @@ def _quoted(value): else: value = str(value) - return "'{}'".format(six.moves.urllib.parse.quote(value)) + return "'{}'".format(value) + class Expression(object): diff --git a/devicecloud/test/test_conditions.py b/devicecloud/test/test_conditions.py index d36df29..f7c0ba5 100644 --- a/devicecloud/test/test_conditions.py +++ b/devicecloud/test/test_conditions.py @@ -16,11 +16,11 @@ def test_lt(self): def test_eq(self): a = Attribute("a") - self.assertEqual((a == "a string").compile(), "a='a%20string'") + self.assertEqual((a == "a string").compile(), "a='a string'") def test_like(self): a = Attribute("a") - self.assertEqual(a.like(r"%.txt").compile(), "a like '%25.txt'") + self.assertEqual(a.like(r"%.txt").compile(), "a like '%.txt'") def test_and(self): a = Attribute("a") @@ -32,12 +32,12 @@ def test_or(self): a = Attribute("a") b = Attribute("b") expr = (a.like("%.csv")) | (b < 1024) - self.assertEqual(expr.compile(), "a like '%25.csv' or b<'1024'") + self.assertEqual(expr.compile(), "a like '%.csv' or b<'1024'") def test_datacmp(self): a = Attribute("a") self.assertEqual((a < datetime.datetime(2014, 7, 7)).compile(), - "a<'2014-07-07T00%3A00%3A00Z'") + "a<'2014-07-07T00:00:00Z'") def test_multi_combination(self): a = Attribute("a") diff --git a/requirements.txt b/requirements.txt index 98dd2ba..1f7a761 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -six>=1.7.3 -requests>=2.2 -arrow>=0.4.4 +six==1.8.0 +requests==2.5.1 +arrow==0.4.4 From 1c0e15c8fd51ceea14862e82eb2a76b60214a1da Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Sat, 27 Dec 2014 23:03:28 -0600 Subject: [PATCH 018/140] filedata: add delete support This resolves PYHONDC-84 for adding support for deleting files and directories from the filedata store. Signed-off-by: Paul Osborne --- devicecloud/examples/filedata_playground.py | 23 +++++++++------------ devicecloud/filedata.py | 21 +++++++++++++++++++ devicecloud/test/test_filedata.py | 21 +++++++++++++++++++ 3 files changed, 52 insertions(+), 13 deletions(-) diff --git a/devicecloud/examples/filedata_playground.py b/devicecloud/examples/filedata_playground.py index 5f333e5..1555761 100644 --- a/devicecloud/examples/filedata_playground.py +++ b/devicecloud/examples/filedata_playground.py @@ -7,20 +7,20 @@ from getpass import getpass from devicecloud import DeviceCloud -from devicecloud.filedata import fd_name, fd_size, fd_type +from devicecloud.filedata import fd_name, fd_size, fd_type, fd_path import six - +from six.moves import input def get_authenticated_dc(): while True: - user = raw_input("username: ") + user = input("username: ") password = getpass("password: ") - dc = DeviceCloud(user, password, base_url="https://test-idigi-com-2v5p9uat81qu.runscope.net") + dc = DeviceCloud(user, password, base_url="https://login-etherios-com-2v5p9uat81qu.runscope.net") if dc.has_valid_credentials(): - print ("Credentials accepted!") + print("Credentials accepted!") return dc else: - print ("Invalid username or password provided, try again") + print("Invalid username or password provided, try again") if __name__ == '__main__': @@ -29,13 +29,10 @@ def get_authenticated_dc(): dc.filedata.write_file("/~/test_dir/", "test_file.txt", six.b("Helllo, world!"), "text/plain") dc.filedata.write_file("/~/test_dir/", "test_file2.txt", six.b("Hello, again!")) - query = None # (fd_path == '/db/public/') - for dirpath, directories, files in dc.filedata.walk(): + for dirpath, directories, files in dc.filedata.walk("/"): for fd_file in files: - print fd_file + print(fd_file) + - for fd in dc.filedata.get_filedata( - (fd_type == "file") & - (fd_size < 250) & - (fd_name.like('%.txt')), page_size=1): + for fd in dc.filedata.get_filedata(fd_path=="~/"): print (fd) diff --git a/devicecloud/filedata.py b/devicecloud/filedata.py index 9e4b737..87f9f61 100644 --- a/devicecloud/filedata.py +++ b/devicecloud/filedata.py @@ -103,6 +103,23 @@ def write_file(self, path, name, data, content_type=None, archive=False): sio.getvalue(), params=params) + def delete_file(self, path): + """Delete a file or directory from the filedata store + + This method removes a file or directory (recursively) from + the filedata store. + + :param path: The path of the file or directory to remove + from the file data store. + + """ + path = validate_type(path, *six.string_types) + if not path.startswith("/"): + path = "/" + path + + self._conn.delete("/ws/FileData{path}".format(path=path)) + + def walk(self, root="~/"): """Emulation of os.walk behavior against the device cloud filedata store @@ -155,6 +172,10 @@ def __init__(self, fdapi, json_data): self._fdapi = fdapi self._json_data = json_data + def delete(self): + """Delete this file or directory""" + return self._fdapi.delete_file(self.get_full_path()) + def get_data(self): """Get the data associated with this filedata object diff --git a/devicecloud/test/test_filedata.py b/devicecloud/test/test_filedata.py index 90319ef..4f72a59 100644 --- a/devicecloud/test/test_filedata.py +++ b/devicecloud/test/test_filedata.py @@ -121,6 +121,13 @@ def test_write_file_simple(self): self.assertEqual(base64.decodestring(six.b(fd_data)), data) self.assertEqual(root.find("fdArchive").text, "true") + def test_delete_path(self): + self.prepare_response("DELETE", "/ws/FileData/test", "") + self.dc.filedata.delete_file("/test") + req = self._get_last_request() + self.assertEqual(req.method, "DELETE") + self.assertEqual(req.path, "/ws/FileData/test") + def test_walk(self): self.prepare_response("GET", "/ws/FileData", GET_HOME_RESULT) gen = self.dc.filedata.walk() @@ -161,6 +168,20 @@ def test_walk(self): class TestFileDataObject(HttpTestBase): + + def test_file_delete(self): + self.prepare_response("GET", "/ws/FileData", GET_FILEDATA_SIMPLE) + objects = list(self.dc.filedata.get_filedata()) + self.assertEqual(len(objects), 2) + obj = objects[0] + self.assertEqual(obj.get_full_path(), "/db/blah/test.txt") + + self.prepare_response("DELETE", "/ws/FileData/db/blah/test.txt", "") + obj.delete() + req = self._get_last_request() + self.assertEqual(req.method, "DELETE") + self.assertEqual(req.path, "/ws/FileData/db/blah/test.txt") + def test_file_metadata_access(self): self.prepare_response("GET", "/ws/FileData", GET_FILEDATA_SIMPLE) objects = list(self.dc.filedata.get_filedata()) From 93e023caf7de94bc06f9bc8a30593fc10f78466d Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Mon, 12 Jan 2015 20:52:05 -0600 Subject: [PATCH 019/140] test: streams: introduce infrastructure for integration tests This commit introduces infrastructure for performing integration tests against the device cloud and a few basic tests (based off of examples) for streams. There are still some kinks to work out but the basic concept appears to work well. Most of the problems being seen have to do with throttling and not having a great way to "reset" the state of the device cloud (particularly when things fail due to throttling). Signed-off-by: Paul Osborne --- devicecloud/__init__.py | 4 +- devicecloud/test/integration/__init__.py | 6 + .../test/integration/inttest_streams.py | 113 ++++++++++++++++++ .../test/integration/inttest_utilities.py | 41 +++++++ devicecloud/test/unit/__init__.py | 6 + .../test/{ => unit}/test_conditions.py | 0 devicecloud/test/{ => unit}/test_core.py | 4 +- .../test/{ => unit}/test_devicecore.py | 2 +- devicecloud/test/{ => unit}/test_filedata.py | 2 +- devicecloud/test/{ => unit}/test_sci.py | 2 +- devicecloud/test/{ => unit}/test_streams.py | 3 +- devicecloud/test/{ => unit}/test_utilities.py | 0 inttest.sh | 23 ++++ tox.ini | 2 +- 14 files changed, 200 insertions(+), 8 deletions(-) create mode 100644 devicecloud/test/integration/__init__.py create mode 100644 devicecloud/test/integration/inttest_streams.py create mode 100644 devicecloud/test/integration/inttest_utilities.py create mode 100644 devicecloud/test/unit/__init__.py rename devicecloud/test/{ => unit}/test_conditions.py (100%) rename devicecloud/test/{ => unit}/test_core.py (97%) rename devicecloud/test/{ => unit}/test_devicecore.py (99%) rename devicecloud/test/{ => unit}/test_filedata.py (99%) rename devicecloud/test/{ => unit}/test_sci.py (96%) rename devicecloud/test/{ => unit}/test_streams.py (99%) rename devicecloud/test/{ => unit}/test_utilities.py (100%) create mode 100755 inttest.sh diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index 04588d9..a7fb9b0 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -282,7 +282,9 @@ class DeviceCloud(object): """ - def __init__(self, username, password, base_url="https://login.etherios.com"): + def __init__(self, username, password, base_url=None): + if base_url is None: + base_url = "https://login.etherios.com" self._conn = DeviceCloudConnection(HTTPBasicAuth(username, password), base_url) self._streams_api = None # streams property api ref self._filedata_api = None # filedata property api ref diff --git a/devicecloud/test/integration/__init__.py b/devicecloud/test/integration/__init__.py new file mode 100644 index 0000000..856342e --- /dev/null +++ b/devicecloud/test/integration/__init__.py @@ -0,0 +1,6 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015 Etherios, Inc. All rights reserved. +# Etherios, Inc. is a Division of Digi International. diff --git a/devicecloud/test/integration/inttest_streams.py b/devicecloud/test/integration/inttest_streams.py new file mode 100644 index 0000000..e6312d5 --- /dev/null +++ b/devicecloud/test/integration/inttest_streams.py @@ -0,0 +1,113 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015 Etherios, Inc. All rights reserved. +# Etherios, Inc. is a Division of Digi International. + +"""Integration tests for streams functionality + +These tests test that the streams functionality actually works against the device +cloud itself. + +""" +from math import pi +import datetime + +from devicecloud.streams import DataPoint, STREAM_TYPE_INTEGER +from devicecloud.test.integration.inttest_utilities import DeviceCloudIntegrationTestCase, dc_inttest_main +import six + + +class StreamsIntegrationTestCase(DeviceCloudIntegrationTestCase): + + def test_basic_nonbulk_stream_operations(self): + # + # This test verifiest that we can perform a number of simple operations on + # a data stream (non-bulk). The ops are create, write, read, and delete + # + SID = "pythondc-inttest/test_basic_nonbulk_stream_operations" + + # get a test stream reference + test_stream = self._dc.streams.get_stream_if_exists(SID) + + # we want a clean stream to work with. If the stream exists, nuke it + if test_stream is not None: + test_stream.delete() + + test_stream = self._dc.streams.create_stream( + stream_id=SID, + data_type='float', + description='a stream used for testing', + units='some-unit', + ) + + for i in range(5): + test_stream.write(DataPoint( + data=i * pi, + description="This is {} * pi".format(i) + )) + + for i, dp in enumerate(test_stream.read(newest_first=False)): + self.assertAlmostEqual(dp.get_data(), i * pi) + + # now cleanup by deleting the stream + test_stream.delete() + + def test_bulk_write_datapoints_multiple_streams(self): + # + # This test verifies that we can write in bulk a bunch of datapoints to several + # datastreams and read them back. + # + SID_FMT="pythondc-inttest/test_bulk_write_datapoints_multiple_streams-{}" + datapoints = [] + for i in range(300): + datapoints.append(DataPoint( + stream_id=SID_FMT.format(i % 3), + data_type=STREAM_TYPE_INTEGER, + units="meters", + data=i, + )) + self._dc.streams.bulk_write_datapoints(datapoints) + + for i in range(3): + stream = self._dc.streams.get_stream(SID_FMT.format(i)) + for j, dp in enumerate(stream.read(newest_first=False)): + self.assertEqual(dp.get_data(), j * 3 + i) + stream.delete() + + def test_bulk_write_datapoints_single_stream(self): + # + # This test verifies that we can write in bulk a bunch of datapoints to a single + # stream and read them back. + # + datapoints = [] + for i in range(300): + datapoints.append(DataPoint( + data_type=STREAM_TYPE_INTEGER, + units="meters", + data=i, + )) + + stream = self._dc.streams.get_stream("pythondc-inttest/test_bulk_write_datapoints_single_stream") + stream.bulk_write_datapoints(datapoints) + stream_contents_asc = list(stream.read(newest_first=False)) + self.assertEqual(len(stream_contents_asc), 300) + for i, dp in enumerate(stream_contents_asc): + self.assertEqual(dp.get_units(), "meters") + self.assertEqual(dp.get_data_type(), STREAM_TYPE_INTEGER) + self.assertEqual(dp.get_data(), i) + self.assertEqual(dp.get_stream_id(), "pythondc-inttest/test_bulk_write_datapoints_single_stream") + self.assertEqual(dp.get_location(), None) + self.assertEqual(dp.get_description(), "") + self.assertIsInstance(dp.get_server_timestamp(), datetime.datetime) + self.assertIsInstance(dp.get_id(), *six.string_types) + self.assertEqual(dp.get_quality(), 0) + self.assertIsInstance(dp.get_timestamp(), datetime.datetime) + + # Cleanup by deleting the stream + stream.delete() + + +if __name__ == '__main__': + dc_inttest_main() diff --git a/devicecloud/test/integration/inttest_utilities.py b/devicecloud/test/integration/inttest_utilities.py new file mode 100644 index 0000000..f92cae5 --- /dev/null +++ b/devicecloud/test/integration/inttest_utilities.py @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015 Etherios, Inc. All rights reserved. +# Etherios, Inc. is a Division of Digi International. + +from getpass import getpass +import unittest + +from devicecloud import DeviceCloud +import os +from six.moves import input + + +class DeviceCloudIntegrationTestCase(unittest.TestCase): + + def setUp(self): + if not os.environ.get("RUN_INTEGRATION_TESTS", False): + self.skipTest("Not performing integration tests") + else: + self._username = os.environ.get("DC_USERNAME", None) + self._password = os.environ.get("DC_PASSWORD", None) + self._base_url = os.environ.get("DC_URL", None) # will use default if unspecified + if not self._username or not self._password: + self.fail("DC_USERNAME and DC_PASSWORD must be set for integration tests to run") + self._dc = DeviceCloud(self._username, self._password, base_url=self._base_url) + + +def dc_inttest_main(): + """Helper method for kicking off integration tests in a module + + This is used in the same way that one might use 'unittest.main()' in a normal + function. + """ + os.environ["RUN_INTEGRATION_TESTS"] = "yes" + if not os.environ.get("DC_USERNAME"): + os.environ["DC_USERNAME"] = input("username: ") + if not os.environ.get("DC_PASSWORD"): + os.environ["DC_PASSWORD"] = getpass("password: ") + unittest.main() diff --git a/devicecloud/test/unit/__init__.py b/devicecloud/test/unit/__init__.py new file mode 100644 index 0000000..856342e --- /dev/null +++ b/devicecloud/test/unit/__init__.py @@ -0,0 +1,6 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015 Etherios, Inc. All rights reserved. +# Etherios, Inc. is a Division of Digi International. diff --git a/devicecloud/test/test_conditions.py b/devicecloud/test/unit/test_conditions.py similarity index 100% rename from devicecloud/test/test_conditions.py rename to devicecloud/test/unit/test_conditions.py diff --git a/devicecloud/test/test_core.py b/devicecloud/test/unit/test_core.py similarity index 97% rename from devicecloud/test/test_core.py rename to devicecloud/test/unit/test_core.py index bbce904..156d2e6 100644 --- a/devicecloud/test/test_core.py +++ b/devicecloud/test/unit/test_core.py @@ -5,9 +5,9 @@ # Copyright (c) 2014 Etherios, Inc. All rights reserved. # Etherios, Inc. is a Division of Digi International. import unittest -from devicecloud import DeviceCloudHttpException -from devicecloud.test.test_utilities import HttpTestBase +from devicecloud import DeviceCloudHttpException +from devicecloud.test.unit.test_utilities import HttpTestBase import six diff --git a/devicecloud/test/test_devicecore.py b/devicecloud/test/unit/test_devicecore.py similarity index 99% rename from devicecloud/test/test_devicecore.py rename to devicecloud/test/unit/test_devicecore.py index 93cd77d..9edb860 100644 --- a/devicecloud/test/test_devicecore.py +++ b/devicecloud/test/unit/test_devicecore.py @@ -10,7 +10,7 @@ from dateutil.tz import tzutc from devicecloud.devicecore import dev_mac, group_id -from devicecloud.test.test_utilities import HttpTestBase +from devicecloud.test.unit.test_utilities import HttpTestBase import httpretty from devicecloud.devicecore import ADD_GROUP_TEMPLATE import six diff --git a/devicecloud/test/test_filedata.py b/devicecloud/test/unit/test_filedata.py similarity index 99% rename from devicecloud/test/test_filedata.py rename to devicecloud/test/unit/test_filedata.py index 4f72a59..699f14d 100644 --- a/devicecloud/test/test_filedata.py +++ b/devicecloud/test/unit/test_filedata.py @@ -4,7 +4,7 @@ import datetime from dateutil.tz import tzutc -from devicecloud.test.test_utilities import HttpTestBase +from devicecloud.test.unit.test_utilities import HttpTestBase import six diff --git a/devicecloud/test/test_sci.py b/devicecloud/test/unit/test_sci.py similarity index 96% rename from devicecloud/test/test_sci.py rename to devicecloud/test/unit/test_sci.py index 8099152..7ccd163 100644 --- a/devicecloud/test/test_sci.py +++ b/devicecloud/test/unit/test_sci.py @@ -8,7 +8,7 @@ import unittest from devicecloud.sci import DeviceTarget -from devicecloud.test.test_utilities import HttpTestBase +from devicecloud.test.unit.test_utilities import HttpTestBase import httpretty import six diff --git a/devicecloud/test/test_streams.py b/devicecloud/test/unit/test_streams.py similarity index 99% rename from devicecloud/test/test_streams.py rename to devicecloud/test/unit/test_streams.py index f8bb840..85dd854 100644 --- a/devicecloud/test/test_streams.py +++ b/devicecloud/test/unit/test_streams.py @@ -12,9 +12,10 @@ from dateutil.tz import tzutc from devicecloud.streams import DataStream, STREAM_TYPE_FLOAT, DataPoint, NoSuchStreamException, ROLLUP_INTERVAL_HALF, \ ROLLUP_METHOD_COUNT, STREAM_TYPE_INTEGER -from devicecloud.test.test_utilities import HttpTestBase +from devicecloud.test.unit.test_utilities import HttpTestBase from devicecloud import DeviceCloudHttpException + # Example HTTP Responses import httpretty import six diff --git a/devicecloud/test/test_utilities.py b/devicecloud/test/unit/test_utilities.py similarity index 100% rename from devicecloud/test/test_utilities.py rename to devicecloud/test/unit/test_utilities.py diff --git a/inttest.sh b/inttest.sh new file mode 100755 index 0000000..55ed206 --- /dev/null +++ b/inttest.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# This script will run all unit and integration +# tests. In order to run the integration tests, we +# need to get a username and password, which is what +# this script does. The rest of the work is done +# by the 'toxtest.sh' script +# + +export RUN_INTEGRATION_TESTS=yes +if [ -z $DC_USERNAME ]; then + echo -n 'username: ' + read username + export DC_USERNAME="$username" +fi + +if [ -z $DC_PASSWORD ]; then + echo -n 'password: ' + read -s password + export DC_PASSWORD="$password" +fi + +./toxtest.sh diff --git a/tox.ini b/tox.ini index 4923fe0..72b0f83 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ deps= -rrequirements.txt httpretty nose -commands=nosetests +commands=nosetests -m '^(int|unit)?[Tt]est' [testenv:coverage] deps= From 366bd881546cea56484c236185d51c9f5b3a8dac Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Sun, 11 Jan 2015 15:45:06 -0600 Subject: [PATCH 020/140] test: streams: include explicit timestamp with bulk data writes This change adds timestamps to the bulk data writes. Without this, the device cloud would seem to sometimes provide back elements out of the expected order when read back. Technically, that behavior cannot be considered wrong as all items were effectively put into the stream at the same moment and they did not have any additional information which would result in the expected ordinal return. Signed-off-by: Paul Osborne --- devicecloud/test/integration/inttest_streams.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/devicecloud/test/integration/inttest_streams.py b/devicecloud/test/integration/inttest_streams.py index e6312d5..4c47b50 100644 --- a/devicecloud/test/integration/inttest_streams.py +++ b/devicecloud/test/integration/inttest_streams.py @@ -61,11 +61,13 @@ def test_bulk_write_datapoints_multiple_streams(self): # SID_FMT="pythondc-inttest/test_bulk_write_datapoints_multiple_streams-{}" datapoints = [] + dt = datetime.datetime.now() for i in range(300): datapoints.append(DataPoint( stream_id=SID_FMT.format(i % 3), data_type=STREAM_TYPE_INTEGER, units="meters", + timestamp=dt - datetime.timedelta(seconds=300 - i), data=i, )) self._dc.streams.bulk_write_datapoints(datapoints) @@ -82,10 +84,12 @@ def test_bulk_write_datapoints_single_stream(self): # stream and read them back. # datapoints = [] + dt = datetime.datetime.now() for i in range(300): datapoints.append(DataPoint( data_type=STREAM_TYPE_INTEGER, units="meters", + timestamp=dt - datetime.timedelta(seconds=300 - i), data=i, )) From f1d4c649dd44866cb3e53eb915a762fd061f74da Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 13 Jan 2015 14:14:23 -0600 Subject: [PATCH 021/140] core: add support for retry on HTTP throttling response This changes resolves PYTHONDC-98 and introduces a solution for a problem often encountered by users of the device cloud. The problem is that the device cloud has limits on the number of requests that can be issued in each 10 second window, depending on one's account type. Writing one's code to be knowledgeable of this limit can be a challenge and presents a difficulty for users of the device cloud that they should not have to be worried about. This change should make it possible for users of the python-devicecloud library to remain mostly ignorant of the fact that throttling is taking place. There will still, of course, be an added delay on any web service call that happens to occur. I am not aware of a better alternative than is implemented here for avoiding the problem. Signed-off-by: Paul Osborne Conflicts: devicecloud/__init__.py --- dev-requirements.txt | 6 ++ devicecloud/__init__.py | 104 +++++++++++++++++++++++------ devicecloud/test/unit/test_core.py | 13 ++++ test-requirements.txt | 6 +- tox.ini | 4 +- 5 files changed, 103 insertions(+), 30 deletions(-) create mode 100644 dev-requirements.txt diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..d8f8f40 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,6 @@ +-r test-requirements.txt +coverage +tox +pyandoc +sphinx +sphinx_rtd_theme diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index a7fb9b0..6a86b57 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -4,14 +4,13 @@ # # Copyright (c) 2014 Etherios, Inc. All rights reserved. # Etherios, Inc. is a Division of Digi International. -from devicecloud.util import validate_type - -from requests.auth import HTTPBasicAuth import logging -import requests import time import json +from devicecloud.util import validate_type +from requests.auth import HTTPBasicAuth +import requests from devicecloud.version import __version__ import six @@ -28,6 +27,28 @@ 204, # No Content (success for DELETE operation) ] +HTTP_THROTTLED_CODES = [ + 429 +] + +# How long in seconds should we delay if a request is throttled? +# +# With a start of 1 second delay and a max of 10 and a default of 5 retries with a backoff coefficient +# of 1.5, then the backoff sequence will be the following: +# +# 1, 1.5, 2.25, 3.375, ~5 +# +# With total delays of the following adding up at each period: +# +# 1, 2.5, 4.75, 8.125, 13.1875 +# +# Assuming a 10s window, this will ensure that we hit a new window before the 5 retries +# are exhausted +DEFAULT_THROTTLE_RETRIES = 5 +DEFAULT_THROTTLE_DELAY_INIT = 1.0 +DEFAULT_THROTTLE_DELAY_MAX = 10.0 +DEFAULT_THROTTLE_DELAY_BACKOFF_COEFFICIENT = 1.5 + logger = logging.getLogger("devicecloud") @@ -89,23 +110,51 @@ class DeviceCloudConnection(object): """ - def __init__(self, auth, base_url): + def __init__(self, auth, base_url, + throttle_retries=DEFAULT_THROTTLE_RETRIES, + throttle_delay_init=DEFAULT_THROTTLE_DELAY_INIT, + throttle_delay_max=DEFAULT_THROTTLE_DELAY_MAX, + throttle_delay_backoff_coefficient=DEFAULT_THROTTLE_DELAY_BACKOFF_COEFFICIENT): self._auth = auth self._base_url = base_url + self._throttle_retries = throttle_retries + self._throttle_delay_init = throttle_delay_init + self._throttle_delay_max = throttle_delay_max + self._throttle_delay_backoff_coefficient = throttle_delay_backoff_coefficient def _make_url(self, path): if not path.startswith("/"): path = "/" + path return "%s%s" % (self._base_url, path) - def _make_request(self, retries, method, url, **kwargs): - remaining_attempts = retries + 1 + def _make_request(self, method, url, **kwargs): + # + # Make a request, retrying up to 'retries' times backing off according to defined constants + # + throttle_retries = kwargs.pop('throttle_retries', kwargs.pop('retries', self._throttle_retries)) + throttle_delay_init = kwargs.pop('throttle_delay_init', self._throttle_delay_init) + throttle_delay_max = kwargs.pop('throttle_delay_max', self._throttle_delay_max) + throttle_delay_backoff_coefficient = \ + kwargs.pop('throttle_delay_backoff_coefficient', self._throttle_delay_backoff_coefficient) + + remaining_attempts = throttle_retries + 1 + retry_delay = throttle_delay_init while remaining_attempts > 0: response = requests.request(method, url, auth=self._auth, **kwargs) if response.status_code in SUCCESSFUL_STATUS_CODES: return response - remaining_attempts -= 1 - time.sleep(1) + elif response.status_code in HTTP_THROTTLED_CODES: + remaining_attempts -= 1 + if remaining_attempts > 0: + logger.info("Request throttled on attempt {attempt}/{max_attempts}, retrying in {delay} seconds".format( + attempt=(throttle_retries + 1 - remaining_attempts), + max_attempts=throttle_retries, + delay=retry_delay + )) + time.sleep(retry_delay) + retry_delay = min(retry_delay * throttle_delay_backoff_coefficient, throttle_delay_max) + else: + break err = "DC %s to %s failed - HTTP(%s)" % (method, url, response.status_code) raise DeviceCloudHttpException(response, err) @@ -149,7 +198,7 @@ def ping(self): """ return self.get("/ws/DeviceCore?size=1") - def get(self, path, retries=0, **kwargs): + def get(self, path, **kwargs): """Perform an HTTP GET request of the specified path in the device cloud Make an HTTP GET request against the device cloud with this accounts @@ -167,9 +216,9 @@ def get(self, path, retries=0, **kwargs): """ url = self._make_url(path) - return self._make_request(retries, "GET", url, **kwargs) + return self._make_request("GET", url, **kwargs) - def get_json(self, path, retries=0, **kwargs): + def get_json(self, path, **kwargs): """Perform an HTTP GET request with JSON headers of the specified path against the device cloud Make an HTTP GET request against the device cloud with this accounts @@ -194,10 +243,10 @@ def get_json(self, path, retries=0, **kwargs): url = self._make_url(path) headers = kwargs.setdefault('headers', {}) headers.update({'Accept': 'application/json'}) - response = self._make_request(retries, "GET", url, **kwargs) + response = self._make_request("GET", url, **kwargs) return json.loads(response.text) - def post(self, path, data, retries=0, **kwargs): + def post(self, path, data, **kwargs): """Perform an HTTP POST request of the specified path in the device cloud Make an HTTP POST request against the device cloud with this accounts @@ -217,9 +266,9 @@ def post(self, path, data, retries=0, **kwargs): """ url = self._make_url(path) - return self._make_request(retries, "POST", url, data=data, **kwargs) + return self._make_request("POST", url, data=data, **kwargs) - def put(self, path, data, retries=0, **kwargs): + def put(self, path, data, **kwargs): """Perform an HTTP PUT request of the specified path in the device cloud Make an HTTP PUT request against the device cloud with this accounts @@ -240,9 +289,9 @@ def put(self, path, data, retries=0, **kwargs): """ url = self._make_url(path) - return self._make_request(retries, "PUT", url, data=data, **kwargs) + return self._make_request("PUT", url, data=data, **kwargs) - def delete(self, path, retries=0, **kwargs): + def delete(self, path, retries=DEFAULT_THROTTLE_RETRIES, **kwargs): """Perform an HTTP DELETE request of the specified path in the device cloud Make an HTTP DELETE request against the device cloud with this accounts @@ -260,7 +309,7 @@ def delete(self, path, retries=0, **kwargs): """ url = self._make_url(path) - return self._make_request(retries, "DELETE", url) + return self._make_request("DELETE", url, **kwargs) class DeviceCloud(object): @@ -282,10 +331,21 @@ class DeviceCloud(object): """ - def __init__(self, username, password, base_url=None): + def __init__(self, username, password, base_url=None, + throttle_retries=DEFAULT_THROTTLE_RETRIES, + throttle_delay_init=DEFAULT_THROTTLE_DELAY_INIT, + throttle_delay_max=DEFAULT_THROTTLE_DELAY_MAX, + throttle_delay_backoff_coefficient=DEFAULT_THROTTLE_DELAY_BACKOFF_COEFFICIENT): if base_url is None: - base_url = "https://login.etherios.com" - self._conn = DeviceCloudConnection(HTTPBasicAuth(username, password), base_url) + base_url = "https://login.etherios.com" + self._conn = DeviceCloudConnection( + auth=HTTPBasicAuth(username, password), + base_url=base_url, + throttle_retries=throttle_retries, + throttle_delay_init=throttle_delay_init, + throttle_delay_max=throttle_delay_max, + throttle_delay_backoff_coefficient=throttle_delay_backoff_coefficient + ) self._streams_api = None # streams property api ref self._filedata_api = None # filedata property api ref self._devicecore_api = None # devicecore property api ref diff --git a/devicecloud/test/unit/test_core.py b/devicecloud/test/unit/test_core.py index 156d2e6..4578fc9 100644 --- a/devicecloud/test/unit/test_core.py +++ b/devicecloud/test/unit/test_core.py @@ -8,6 +8,7 @@ from devicecloud import DeviceCloudHttpException from devicecloud.test.unit.test_utilities import HttpTestBase +from mock import patch, call import six @@ -60,6 +61,18 @@ class TestDeviceCloudConnection(HttpTestBase): + @patch("time.sleep", return_value=None) + def test_throttle_retries(self, patched_time_sleep): + self.prepare_response("GET", "/test/path", "", status=429) + self.assertRaises(DeviceCloudHttpException, self.dc.get_connection().get, "/test/path", retries=5) + patched_time_sleep.assert_has_calls([ + call(1.5 ** 0), + call(1.5 ** 1), + call(1.5 ** 2), + call(1.5 ** 3), + call(1.5 ** 4), + ]) + def test_iter_json_with_params(self): it = self.dc.get_connection().iter_json_pages("/test/path", foo="bar", key="value") self.prepare_response("GET", "/test/path", TEST_BASIC_RESPONSE) diff --git a/test-requirements.txt b/test-requirements.txt index a5dd38d..5133248 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,8 +1,4 @@ -r requirements.txt -tox +mock nose httpretty -pyandoc -sphinx -coverage -sphinx_rtd_theme diff --git a/tox.ini b/tox.ini index 72b0f83..065fdbb 100644 --- a/tox.ini +++ b/tox.ini @@ -3,9 +3,7 @@ envlist = py27,py32,py33,py34,pypy [testenv] deps= - -rrequirements.txt - httpretty - nose + -rtest-requirements.txt commands=nosetests -m '^(int|unit)?[Tt]est' [testenv:coverage] From 080c4b7894f83bdf451e1627745a2066bfbd1aa3 Mon Sep 17 00:00:00 2001 From: Dan Harrison Date: Tue, 13 Jan 2015 15:39:06 -0600 Subject: [PATCH 022/140] devicecore: add support for provisioning devices This commit adds support and tests for adding devices to ones Device Cloud account by either MAC address or ID. For both, one can choose to either add a single device or multiple. This resolves PYTHONDC-21 (registering a single device) and PYTHONDC-23 (registering multiple devices). Signed-off-by: Paul Osborne --- devicecloud/__init__.py | 1 + devicecloud/devicecore.py | 52 ++++++++ devicecloud/test/unit/test_provision.py | 158 ++++++++++++++++++++++++ devicecloud/test/unit/test_utilities.py | 9 +- 4 files changed, 216 insertions(+), 4 deletions(-) create mode 100644 devicecloud/test/unit/test_provision.py diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index 6a86b57..881c00c 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -25,6 +25,7 @@ 201, # Created 202, # Accepted 204, # No Content (success for DELETE operation) + 207, # Multi-Status (some success for provisioning, parse it before raising exception) ] HTTP_THROTTLED_CODES = [ diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index 742abc5..61f3c4a 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -11,6 +11,8 @@ from devicecloud.util import iso8601_to_dt, validate_type import six +import xml.etree.ElementTree as ET + dev_mac = Attribute('devMac') group_id = Attribute('grpId') @@ -156,6 +158,56 @@ def get_groups(self, condition=None, page_size=1000): for group_data in self._conn.iter_json_pages("/ws/Group", page_size=page_size, **query_kwargs): yield Group.from_json(group_data) + def provision_by_mac(self, mac_list): + """ + Provision a series of devices. + + The device IDs will be generated from the supplied iterable of MAC + address (usually the device's primary network interface). + + Returns a list (or single item, if supplied) of response dictionaries + corresponding to each item for which provisioning was requested. + """ + nonlist = isinstance(mac_list, *six.string_types) + if nonlist: + mac_list = [mac_list] + messages = ["" + + mac + "" for mac in mac_list] + r = self._provision(messages) + if nonlist: + r = r[0] + return r + + def provision_by_id(self, device_id_list): + """ + Provisions a series of devices. + + The iterable passed must iterate over a series of device IDs. + + Returns a list (or single item, if supplied) of response dictionaries + corresponding to each item for which provisioning was requested. + """ + nonlist = isinstance(device_id_list, *six.string_types) + if nonlist: + device_id_list = [device_id_list] + messages = ["" + + did + "" for did in device_id_list] + r = self._provision(messages) + if nonlist: + r = r[0] + return r + + def _provision(self, messages): + resp = self._conn.post('/ws/DeviceCore', '' + ''.join(messages) + '') + resp_xml = ET.fromstring(resp.content) + results = [] + for child in list(resp_xml): + if child.tag == 'location': + results.append({'location': child.text, 'error': False}) + else: + results.append({'location': None, 'error': child.text}) + return results + class Group(object): """Provides access to information about a group in the device cloud diff --git a/devicecloud/test/unit/test_provision.py b/devicecloud/test/unit/test_provision.py new file mode 100644 index 0000000..22a26ce --- /dev/null +++ b/devicecloud/test/unit/test_provision.py @@ -0,0 +1,158 @@ +""" +Tests the provisioning feature of the devicecore. + +Isolated to its own file, because it's so different from everything else. +""" +import sys +import unittest + +import mock + +from devicecloud import DeviceCloudHttpException +from devicecloud.devicecore import DeviceCoreAPI +from devicecloud.test.unit.test_utilities import HttpTestBase + +SUCCESS_STATUS = 201 +ERRORS_STATUS = 207 + + +def build_response(response_list): + """ + Generate response from an expected result. + + Just a utility function for convenience. + """ + if type(response_list) == dict: + response_list = [response_list] + xml_tags = [r['error'] and + '' + r['error'] + '' or + '' + r['location'] + '' + for r in response_list] + return '\n' + ''.join(xml_tags) + '' + + +class ProvisioningParserTests(HttpTestBase): + def test_xml_error(self): + # Make sure that if their interface changes, and suddenly the XML stuff + # stops working, that an exception will indeed be thrown. + self.prepare_response('POST', '/ws/DeviceCore', 'Bad Request', status=400) + try: + self.dc.devicecore._provision(['whatever']) + except DeviceCloudHttpException: + pass + else: + assert False, "should have thrown exception" + self.prepare_response('POST', '/ws/DeviceCore', 'Internal Server Error', status=500) + try: + self.dc.devicecore._provision(['whatever']) + except DeviceCloudHttpException: + pass + else: + assert False, "should have thrown exception" + + def test_one_good(self): + correct_response = [{'location': 'DeviceCore/123474/0', 'error': False}] + self.prepare_response('POST', '/ws/DeviceCore', build_response(correct_response), status=SUCCESS_STATUS) + result = self.dc.devicecore._provision(['whatever']) + self.assertIsInstance(result, list) + self.assertEqual(len(result), 1) + self.assertDictEqual(result[0], correct_response[0]) + + def test_one_error(self): + correct_response = [{'error': 'The device is already provisioned.', 'location': None}] + self.prepare_response('POST', '/ws/DeviceCore', build_response(correct_response), status=ERRORS_STATUS) + result = self.dc.devicecore._provision(['whatever']) + self.assertIsInstance(result, list) + self.assertEqual(len(result), 1) + self.assertDictEqual(result[0], correct_response[0]) + + def test_two_good(self): + correct_response = [{'location': 'DeviceCore/123474/0', 'error': False}, + {'location': 'DeviceCore/123475/0', 'error': False}] + self.prepare_response('POST', '/ws/DeviceCore', build_response(correct_response), status=SUCCESS_STATUS) + result = self.dc.devicecore._provision(['whatever'] * 2) + self.assertIsInstance(result, list) + self.assertEqual(len(result), 2) + for i in range(len(result)): + self.assertDictEqual(result[i], correct_response[i]) + + def test_two_error(self): + correct_response = [{'location': None, 'error': 'Problem with Device ID: Problem with segment number 4'}, + {'location': None, 'error': 'The device is already provisioned.'}] + self.prepare_response('POST', '/ws/DeviceCore', build_response(correct_response), status=ERRORS_STATUS) + result = self.dc.devicecore._provision(['whatever'] * 2) + self.assertIsInstance(result, list) + self.assertEqual(len(result), 2) + for i in range(len(result)): + self.assertDictEqual(result[i], correct_response[i]) + + def test_two_split(self): + correct_response = [{'location': 'DeviceCore/123474/0', 'error': False}, + {'location': None, 'error': 'The device is already provisioned.'}] + self.prepare_response('POST', '/ws/DeviceCore', build_response(correct_response), status=ERRORS_STATUS) + result = self.dc.devicecore._provision(['whatever'] * 2) + self.assertIsInstance(result, list) + self.assertEqual(len(result), 2) + for i in range(len(result)): + self.assertDictEqual(result[i], correct_response[i]) + + +provision = mock.MagicMock() +provision.post.return_value.status_code = 201 +provision.post.return_value.content = '' + + +class ProvisionByTests(object): + def setUp(self): + HttpTestBase.setUp(self) + global provision + provision.reset_mock() + + def test_single(self): + correct_response = {'location': 'DeviceCore/123474/0', 'error': False} + self.prepare_response('POST', '/ws/DeviceCore', build_response(correct_response), status=SUCCESS_STATUS) + result = self.provision_call('00:40:9d:aa:bb:cc') + self.assertDictEqual(result, correct_response) + + def test_single_list(self): + correct_response = [{'location': 'DeviceCore/123474/0', 'error': False}] + self.prepare_response('POST', '/ws/DeviceCore', build_response(correct_response), status=SUCCESS_STATUS) + result = self.provision_call(['00:40:9d:aa:bb:cc']) + self.assertIsInstance(result, list) + self.assertDictEqual(result[0], correct_response[0]) + + @mock.patch.object(DeviceCoreAPI, '_provision', provision) + def test_single_xml(self): + self.provision_call('00:40:9d:aa:bb:cc') + provision.assert_called_once_with(['<{0}>00:40:9d:aa:bb:cc'.format(self.call_tag)]) + + @mock.patch.object(DeviceCoreAPI, '_provision', provision) + def test_single_list_xml(self): + self.provision_call(['00:40:9d:aa:bb:cc']) + provision.assert_called_once_with(['<{0}>00:40:9d:aa:bb:cc'.format(self.call_tag)]) + + @mock.patch.object(DeviceCoreAPI, '_provision', provision) + def test_multi_list_xml(self): + self.provision_call(['00:40:9d:aa:bb:cc', '00:40:9d:aa:bb:cd']) + provision.assert_called_once_with(['<{0}>00:40:9d:aa:bb:cc'.format(self.call_tag), + '<{0}>00:40:9d:aa:bb:cd'.format(self.call_tag)]) + + +class ProvisionByIdTests(ProvisionByTests, HttpTestBase): + def setUp(self): + HttpTestBase.setUp(self) + ProvisionByTests.setUp(self) + self.provision_call = self.dc.devicecore.provision_by_id + self.call_tag = "devConnectwareId" + + +class ProvisionByMacTests(ProvisionByTests, HttpTestBase): + def setUp(self): + HttpTestBase.setUp(self) + ProvisionByTests.setUp(self) + self.provision_call = self.dc.devicecore.provision_by_mac + self.call_tag = "devMac" + + +if __name__ == '__main__': + unittest.main() diff --git a/devicecloud/test/unit/test_utilities.py b/devicecloud/test/unit/test_utilities.py index fcb024b..3b4786c 100644 --- a/devicecloud/test/unit/test_utilities.py +++ b/devicecloud/test/unit/test_utilities.py @@ -32,16 +32,17 @@ def _get_last_request_params(self): params = urllib_parse.parse_qs(urllib_parse.urlparse(self._get_last_request().path).query) return {k: v[0] for k, v in params.items()} # convert from list values to single-value - def prepare_response(self, method, path, data, status=200, match_querystring=False): + def prepare_response(self, method, path, data=None, status=200, match_querystring=False, **kwargs): # TODO: # Should probably assert on more request headers and # respond with correct content type, etc. - + if data is not None: + kwargs['body'] = data httpretty.register_uri(method, "https://login.etherios.com{}".format(path), - data, match_querystring=match_querystring, - status=status) + status=status, + **kwargs) def prepare_json_response(self, method, path, data, status=200): self.prepare_response(method, path, json.dumps(data), status=status) From 8288ecbff24ea72696100852b68f4a2d3e33580b Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Mon, 19 Jan 2015 11:10:54 -0600 Subject: [PATCH 023/140] ws: provide API for making authenticated web service requests This commit adds a new API endpoint, ws, which provides some syntactic sugar for getting nice looking paths to various web service endpoints on the device cloud. In addition, some textual documentation has been added that covers this new interface and makes additional reference to the DeviceCloudConnection APIs that are now part of the public API for the library (previously private). This work is based on the initial patch by Dan Harrison . Signed-off-by: Paul Osborne --- devicecloud/__init__.py | 24 ++++++++++++++- devicecloud/test/unit/test_ws.py | 40 +++++++++++++++++++++++++ devicecloud/ws.py | 51 ++++++++++++++++++++++++++++++++ docs/index.rst | 1 + docs/ws.rst | 42 ++++++++++++++++++++++++++ 5 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 devicecloud/test/unit/test_ws.py create mode 100644 devicecloud/ws.py create mode 100644 docs/ws.rst diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index 881c00c..b263921 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -14,10 +14,12 @@ from devicecloud.version import __version__ import six + __all__ = ( 'DeviceCloud', 'DeviceCloudException', 'DeviceCloudHttpException', + 'DeviceCloudConnection', ) SUCCESSFUL_STATUS_CODES = [ @@ -338,7 +340,7 @@ def __init__(self, username, password, base_url=None, throttle_delay_max=DEFAULT_THROTTLE_DELAY_MAX, throttle_delay_backoff_coefficient=DEFAULT_THROTTLE_DELAY_BACKOFF_COEFFICIENT): if base_url is None: - base_url = "https://login.etherios.com" + base_url = "https://login.etherios.com" self._conn = DeviceCloudConnection( auth=HTTPBasicAuth(username, password), base_url=base_url, @@ -351,6 +353,7 @@ def __init__(self, username, password, base_url=None, self._filedata_api = None # filedata property api ref self._devicecore_api = None # devicecore property api ref self._sci_api = None # sci property api ref + self._legacy_api = None # legacy property api ref def has_valid_credentials(self): """Verify that the device cloud url, username, and password are valid @@ -397,6 +400,11 @@ def sci(self): self._sci_api = self.get_sci_api() return self._sci_api + @property + def ws(self): + """Property providing access to the :class:`.WebServiceStub` with a base of ``/ws``""" + return self.get_web_service_stub() + def get_connection(self): """Get the low-level :class:`~DeviceCloudConnection` for this device cloud instance @@ -461,3 +469,17 @@ def get_sci_api(self): from devicecloud.sci import ServerCommandInterfaceAPI return ServerCommandInterfaceAPI(self._conn) + + def get_web_service_stub(self): + """Returns a :class:`.WebServiceStub` bound to this device cloud instance + + This provides access to the same API as :attr:`.DeviceCloud.legacy` but will create + a new object (with a new cache) each time called. + + :return: WebServiceStub object bound to this device cloud account with a base of ``/ws`` + :rtype: :class:`.WebServiceStub` + + """ + from devicecloud.ws import WebServiceStub + + return WebServiceStub(self._conn, "/ws") diff --git a/devicecloud/test/unit/test_ws.py b/devicecloud/test/unit/test_ws.py new file mode 100644 index 0000000..2ae06b8 --- /dev/null +++ b/devicecloud/test/unit/test_ws.py @@ -0,0 +1,40 @@ +import unittest + +from devicecloud import DeviceCloudException +from mock import MagicMock, patch + +from devicecloud.ws import WebServiceStub +from devicecloud import DeviceCloudConnection + + +class MockConnection(MagicMock): + + def get(self, *args, **kwargs): + return (args, kwargs) + + def post(self, *args, **kwargs): + return (args, kwargs) + + +class LegacyAPIMemberInternalsTests(unittest.TestCase): + + def setUp(self): + self.conn = MockConnection() + self.stub = WebServiceStub(self.conn, '/ws') + + def test_path_building(self): + test = self.stub.a.b.c + self.assertEqual(test._path, "/ws/a/b/c") + + def test_method_access(self): + res = self.stub.a.b.c.get() + self.assertEqual(res[0], ("/ws/a/b/c", )) + self.assertDictEqual(res[1], {}) + + def test_method_access_args_kwargs(self): + res = self.stub.a.test.path.post("foo", bar="baz") + self.assertEqual(res[0], ("/ws/a/test/path", "foo")) + self.assertDictEqual(res[1], {"bar": "baz"}) + +if __name__ == '__main__': + unittest.main() diff --git a/devicecloud/ws.py b/devicecloud/ws.py new file mode 100644 index 0000000..ad5f68e --- /dev/null +++ b/devicecloud/ws.py @@ -0,0 +1,51 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015 Etherios, Inc. All rights reserved. +# Etherios, Inc. is a Division of Digi International. +import functools +import inspect + + +class WebServiceStub(object): + """Provide a set of methods to directory access the web services API at a given path + + Web services stubs can be chained in order to build a path and, eventually, perform + an operation on the end result. For instance, the following will perform + a GET request on the path ``/some/base/with/some/added/stuff``:: + + WebServiceStub(conn, "/some/base").with.some.added.stuff.get() + + In the context of the this library, a more common example might be accessing + a V1 API such as /ws/v1/devices. That would be done like this:: + + response = dc.ws.v1.devices.get() + + Where response would end up being a `requests Response object + `. + + Any of the methods exposed by :class:`devicecloud.DeviceCloudConnection` may be + called and the path for the stub will be passed as the first argument to + the method with the same name in that class. + + """ + + def __init__(self, conn, path): + self._conn = conn + self._path = "/" + path if path[0] != '/' else path + + def __getattr__(self, attr): + """We implement this method to provide the "builder" syntax""" + conn_meth = getattr(self._conn, attr, None) + if conn_meth is not None and inspect.ismethod(conn_meth): + # If this is method on DeviceCloudConnection, then return a function bound to + # that method that will be called with this stub's path + @functools.wraps(conn_meth) + def bound_cloud_connection_method(*args, **kwargs): + return conn_meth(self._path, *args, **kwargs) + return bound_cloud_connection_method + + # Otherwise, assume that specified attribute is another path and return + # a new builder which is our path combined with the provided attribute + return WebServiceStub(self._conn, "{}/{}".format(self._path, attr)) diff --git a/docs/index.rst b/docs/index.rst index 56314db..445eaac 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,6 +12,7 @@ Documention Map streams filedata sci + ws cookbook Introduction diff --git a/docs/ws.rst b/docs/ws.rst new file mode 100644 index 0000000..ee2a4a5 --- /dev/null +++ b/docs/ws.rst @@ -0,0 +1,42 @@ +Direct Web Services API +======================= + +The Device Cloud exposes a large set of functionality to users and the +python-devicecloud library seeks to provide convenient and complete +APIs for a majority of these. However, there are APIs which the library +does not cover; some may have coverage in the future and others may never +have direct support in the library. + +The "ws" API provides a mechanism for directly making calls to +unsupported web services APIs. An example of the syntax exposed by the library +is probably best demonstrated with an example. + +An example of an API not currently supported by the library is the `Alarms API +`_. +This API is a "legacy" API (it is not prefixed with a "v1") and its basic interface is GET, POST, PUT, and +DELETE of the path "/ws/Alarm". The API returns response and expects payloads to be in XML according +to a format described in the documentation:: + + # List alarms + # + # Retrieves results of GET to /ws/Alarms with authentication and will raise + # the standard exceptions in the case of a failure response. + # + >>> dc = devicecloud.DeviceCloud('user', 'pass') + >>> response = dc.ws.Alarm.get() + >>> print response.content + ... A bunch of XML ... + >>> print dc.ws.Alarm.get_json() + ... A bunch of JSON ... + >>> print list(dc.ws.Alarm.iter_json_pages()) + ... All Alarms over all pages with result as list of dictionaries ... + + # + # Note that in the syntactic sugar may fall short. In those cases, you + # may need to fall back to using the underlying DeviceConnection + # + >>> alarm_id = 10 + >>> print dc.get_connection().get_json("/ws/Alarm/{}".format(alarm_id)) + ... Some JSON ... + +For more details, refer to the documentation on the methods for :py:class:`devicecloud.DeviceCloudConnection`. From c7ba03ce45a22b79847e0e4413e3fc7fefb9eaa5 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 28 Jan 2015 15:25:29 -0600 Subject: [PATCH 024/140] devicecore: provisioning: revised API/implementation for provisioning This change adds documentation for provisioning and iterates on the work done by Dan Harrison to create an API for provisioning that is more consistent with the other APIs and which more fully exposes the various available fields that can be provided when provisioning a device in the cloud. This changes relates to PYTHONDC-21 for single-device registration and PYTHONDC-23 for registration of multiple devices. Signed-off-by: Paul Osborne --- devicecloud/devicecore.py | 174 +++++++++++++++++------ devicecloud/test/unit/test_devicecore.py | 156 ++++++++++++++++++++ devicecloud/test/unit/test_provision.py | 158 -------------------- 3 files changed, 285 insertions(+), 203 deletions(-) delete mode 100644 devicecloud/test/unit/test_provision.py diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index 61f3c4a..3a42a4b 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -5,14 +5,13 @@ # Copyright (c) 2014 Etherios, Inc. All rights reserved. # Etherios, Inc. is a Division of Digi International. import sys +import xml.etree.ElementTree as ET from devicecloud.apibase import APIBase from devicecloud.conditions import Attribute, Expression from devicecloud.util import iso8601_to_dt, validate_type import six -import xml.etree.ElementTree as ET - dev_mac = Attribute('devMac') group_id = Attribute('grpId') @@ -158,54 +157,140 @@ def get_groups(self, condition=None, page_size=1000): for group_data in self._conn.iter_json_pages("/ws/Group", page_size=page_size, **query_kwargs): yield Group.from_json(group_data) - def provision_by_mac(self, mac_list): - """ - Provision a series of devices. - - The device IDs will be generated from the supplied iterable of MAC - address (usually the device's primary network interface). + def provision_device(self, **kwargs): + """Provision a single device with the specified information + + This API call provisions a new device on this cloud account. In order for this + to work, a `mac_address`, `device_id`, or `imei` must be provided. All + other parameters are optional and can be specified in addition to the primary + identifier for the device. + + This request will always return a dictionary unless the request fails altogether. The + dictionary returned will have the following form:: + + { + "error": , + "error_msg": , + "location": , + } + + :param str mac_address: The MAC address of the device being added as a string in + the form "00:00:00:00:00:00". This is one of the options for the required + primary id. + :param str device_id: The ID of the device to add. This is the 'devConnectwareId' + referenced in the API docs and should look like "00000000-00000000-000000FF-FF000000". + This is one of the options for the required primary id. + :param str imei: The IMEI of the device to be added (if no MAC/ID available). This is one + of the options for the required primary id. + :param str group_path: (optional) Path of group that this device should be added to. + :param str metadata: (optional) Arbitrary metadata to associate with this device. + :param float map_lat: (optional) Latitude of this device in degrees + :param float map_long: (optional) Longitude of this device in degrees + :param str contact: (optional) Contact associted with this device (or whatever you want). + :param str description: (optional) Textual description of this device. + + :raises DeviceCloudHttpException: If there is an unexpected error reported by the device cloud. + :raises ValueError: If any input fields are known to have a bad form. + :return: A dictionary matching the format specified above. - Returns a list (or single item, if supplied) of response dictionaries - corresponding to each item for which provisioning was requested. """ - nonlist = isinstance(mac_list, *six.string_types) - if nonlist: - mac_list = [mac_list] - messages = ["" + - mac + "" for mac in mac_list] - r = self._provision(messages) - if nonlist: - r = r[0] - return r - - def provision_by_id(self, device_id_list): + # This snippet is from the device cloud API Explorer and shows the pieces of + # information that may be specified when adding a device. + # + # + # + # + # + # + # + # + # + # + # + # + # + # + # + + results = self.provision_devices([kwargs, ]) + return results[0] + + def provision_devices(self, devices): + """Provision multiple devices with a single API call + + This method takes an iterable of dictionaries where the values in the dictionary are + expected to match the arguments of a call to :meth:`provision_device`. The + contents of each dictionary will be validated. + + :param list devices: An iterable of dictionaries each containing information about + a device to be provision. The form of the dictionary should match the keyword + arguments taken by :meth:`provision_device`. + :raises DeviceCloudHttpException: If there is an unexpected error reported by the device cloud. + :raises ValueError: If any input fields are known to have a bad form. + :return: A list of dictionaries in the form described for :meth:`provision_device` in the + order matching the requested device list. Note that it is possible for there to + be mixed success and error when provisioning multiple devices. + """ - Provisions a series of devices. + # Validate all the input for each device provided + sio = six.StringIO() + + def write_tag(tag, val): + sio.write("<{tag}>{val}".format(tag=tag, val=val)) + + def maybe_write_element(tag, val): + if val is not None: + write_tag(tag, val) + return True + return False + + sio.write("") + for d in devices: + sio.write("") + + mac_address = d.get("mac_address") + device_id = d.get("device_id") + imei = d.get("imei") + if mac_address is not None: + write_tag("devMac", mac_address) + elif device_id is not None: + write_tag("devConnectwareId", device_id) + elif imei is not None: + write_tag("devCellularModemId", imei) + else: + raise ValueError("mac_address, device_id, or imei must be provided for device %r" % d) - The iterable passed must iterate over a series of device IDs. + # Write optional elements if present. + maybe_write_element("grpPath", d.get("group_path")) + maybe_write_element("dpUserMetaData", d.get("metadata")) + maybe_write_element("dpTags", d.get("tags")) + maybe_write_element("dpMapLong", d.get("map_long")) + maybe_write_element("dpMapLat", d.get("map_lat")) + maybe_write_element("dpContact", d.get("contact")) + maybe_write_element("dpDescription", d.get("description")) - Returns a list (or single item, if supplied) of response dictionaries - corresponding to each item for which provisioning was requested. - """ - nonlist = isinstance(device_id_list, *six.string_types) - if nonlist: - device_id_list = [device_id_list] - messages = ["" + - did + "" for did in device_id_list] - r = self._provision(messages) - if nonlist: - r = r[0] - return r - - def _provision(self, messages): - resp = self._conn.post('/ws/DeviceCore', '' + ''.join(messages) + '') - resp_xml = ET.fromstring(resp.content) + sio.write("") + sio.write("") + + # Send the request, set the Accept XML as a nicety results = [] - for child in list(resp_xml): - if child.tag == 'location': - results.append({'location': child.text, 'error': False}) - else: - results.append({'location': None, 'error': child.text}) + response = self._conn.post("/ws/DeviceCore", sio.getvalue(), headers={'Accept': 'application/xml'}) + root = ET.fromstring(response.content) # tag is root of response + for child in root: + if child.tag.lower() == "location": + results.append({ + "error": False, + "error_msg": None, + "location": child.text + }) + else: # we expect "error" but handle generically + results.append({ + "error": True, + "location": None, + "error_msg": child.text + }) + return results @@ -291,7 +376,6 @@ class Device(object): # TODO: provide ability to set/update available data items # TODO: add/remove tags - # TODO: provision a new device (probably add top-level method for this) def __init__(self, conn, sci, device_json): self._conn = conn diff --git a/devicecloud/test/unit/test_devicecore.py b/devicecloud/test/unit/test_devicecore.py index 9edb860..cdc24c0 100644 --- a/devicecloud/test/unit/test_devicecore.py +++ b/devicecloud/test/unit/test_devicecore.py @@ -9,6 +9,7 @@ import unittest from dateutil.tz import tzutc +from devicecloud import DeviceCloudHttpException from devicecloud.devicecore import dev_mac, group_id from devicecloud.test.unit.test_utilities import HttpTestBase import httpretty @@ -141,6 +142,35 @@ } """ +PROVISION_SUCCESS_RESPONSE1 = """\ + + + DeviceCore/946246/0 + +""" + +PROVISION_MULTIPLE_SUCCESS_RESPONSE1 = """\ + + + DeviceCore/1397876/0 + DeviceCore/946246/0 + +""" + +PROVISION_ERROR1 = """\ + + + The device 00000000-00000000-BC5FF4FF-FFF7908A is already provisioned. + +""" + +PROVISION_MIXED_RESULT_RESPONSE = """\ + + + DeviceCore/1397876/0 + The device 00000000-00000000-D48564FF-FF9D4FEE is already provisioned. + +""" class TestDeviceCoreGroups(HttpTestBase): @@ -181,6 +211,132 @@ def test_get_groups_condition(self): self.assertEqual(params["condition"], "grpId='123'") +class TestDeviceCoreProvisioning(HttpTestBase): + + def test_provision_one_simple_device_id(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_SUCCESS_RESPONSE1, status=207) + res = self.dc.devicecore.provision_device(device_id='00000000-00000000-0000DEFF-FFADBEEFF') + req = self._get_last_request() + self.assertEqual(req.body, six.b( + "" + "" + "00000000-00000000-0000DEFF-FFADBEEFF" + "" + "")) + self.assertDictEqual(res, {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) + + def test_provision_one_simple_mac(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_SUCCESS_RESPONSE1, status=207) + res = self.dc.devicecore.provision_device(mac_address="DE:AD:BE:EF:00:00") + req = self._get_last_request() + self.assertEqual(req.body, six.b( + "" + "" + "DE:AD:BE:EF:00:00" + "" + "")) + self.assertDictEqual(res, {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) + + def test_provision_imei(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_SUCCESS_RESPONSE1, status=207) + res = self.dc.devicecore.provision_device(imei="990000862471854") + req = self._get_last_request() + self.assertEqual(req.body, six.b( + "" + "" + "990000862471854" + "" + "")) + self.assertDictEqual(res, {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) + + def test_provision_all_the_fixins(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_SUCCESS_RESPONSE1, status=207) + res = self.dc.devicecore.provision_device( + mac_address="DE:AD:BE:EF:00:00", + group_path="/group/path", + metadata="Sweet, sweet metadata", + map_lat=44.9807496, + map_long=-93.1397815, + contact="Saint Paul Parks Department", + description="Buried Treasure", + ) + req = self._get_last_request() + self.assertEqual(req.body, six.b( + '' + '' + 'DE:AD:BE:EF:00:00' + '/group/path' + 'Sweet, sweet metadata' + '-93.1397815' + '44.9807496' + 'Saint Paul Parks Department' + 'Buried Treasure' + '' + '')) + self.assertDictEqual(res, {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) + + def test_provision_multiple_simple(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_MULTIPLE_SUCCESS_RESPONSE1, status=207) + res = self.dc.devicecore.provision_devices([ + {'device_id': "00000000-00000000-0000DEFF-FFADBEEFF"}, + {'mac_address': 'DE:AD:BE:EF:00:00'} + ]) + req = self._get_last_request() + self.assertEqual(req.body, six.b( + '' + '' + '00000000-00000000-0000DEFF-FFADBEEFF' + '' + '' + '' + 'DE:AD:BE:EF:00:00' + '' + '')) + self.assertTrue(len(res), 2) + self.assertDictEqual(res[0], {"error": False, "error_msg": None, "location": "DeviceCore/1397876/0"}) + self.assertDictEqual(res[1], {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) + + def test_without_required_param(self): + self.assertRaises(ValueError, self.dc.devicecore.provision_device, description="I should not work") + + def test_bad_request_400(self): + self.prepare_response('POST', '/ws/DeviceCore', 'Bad Request', status=400) + self.assertRaises(DeviceCloudHttpException, + self.dc.devicecore.provision_device, mac_address="DE:AD:BE:EF:00:00") + + def test_bad_request_500(self): + self.prepare_response('POST', '/ws/DeviceCore', 'Bad Request', status=500) + self.assertRaises(DeviceCloudHttpException, + self.dc.devicecore.provision_device, mac_address="DE:AD:BE:EF:00:00") + + def test_error_response(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_ERROR1, status=207) + res = self.dc.devicecore.provision_device(imei="990000862471854") + self.assertDictEqual(res, { + "error": True, + "error_msg": 'The device 00000000-00000000-BC5FF4FF-FFF7908A is already provisioned.', + "location": None} + ) + + def test_mixed_error_success_response(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_MIXED_RESULT_RESPONSE, status=207) + res = self.dc.devicecore.provision_devices([ + {'device_id': "00000000-00000000-0000DEFF-FFADBEEFF"}, + {'mac_address': 'DE:AD:BE:EF:00:00'} + ]) + self.assertTrue(len(res), 2) + self.assertDictEqual(res[0], { + 'error': False, + 'error_msg': None, + 'location': 'DeviceCore/1397876/0', + }) + self.assertDictEqual(res[1], { + 'error': True, + 'error_msg': 'The device 00000000-00000000-D48564FF-FF9D4FEE is already provisioned.', + 'location': None, + }) + + class TestDeviceCoreDevices(HttpTestBase): def test_dc_get_devices(self): diff --git a/devicecloud/test/unit/test_provision.py b/devicecloud/test/unit/test_provision.py deleted file mode 100644 index 22a26ce..0000000 --- a/devicecloud/test/unit/test_provision.py +++ /dev/null @@ -1,158 +0,0 @@ -""" -Tests the provisioning feature of the devicecore. - -Isolated to its own file, because it's so different from everything else. -""" -import sys -import unittest - -import mock - -from devicecloud import DeviceCloudHttpException -from devicecloud.devicecore import DeviceCoreAPI -from devicecloud.test.unit.test_utilities import HttpTestBase - -SUCCESS_STATUS = 201 -ERRORS_STATUS = 207 - - -def build_response(response_list): - """ - Generate response from an expected result. - - Just a utility function for convenience. - """ - if type(response_list) == dict: - response_list = [response_list] - xml_tags = [r['error'] and - '' + r['error'] + '' or - '' + r['location'] + '' - for r in response_list] - return '\n' + ''.join(xml_tags) + '' - - -class ProvisioningParserTests(HttpTestBase): - def test_xml_error(self): - # Make sure that if their interface changes, and suddenly the XML stuff - # stops working, that an exception will indeed be thrown. - self.prepare_response('POST', '/ws/DeviceCore', 'Bad Request', status=400) - try: - self.dc.devicecore._provision(['whatever']) - except DeviceCloudHttpException: - pass - else: - assert False, "should have thrown exception" - self.prepare_response('POST', '/ws/DeviceCore', 'Internal Server Error', status=500) - try: - self.dc.devicecore._provision(['whatever']) - except DeviceCloudHttpException: - pass - else: - assert False, "should have thrown exception" - - def test_one_good(self): - correct_response = [{'location': 'DeviceCore/123474/0', 'error': False}] - self.prepare_response('POST', '/ws/DeviceCore', build_response(correct_response), status=SUCCESS_STATUS) - result = self.dc.devicecore._provision(['whatever']) - self.assertIsInstance(result, list) - self.assertEqual(len(result), 1) - self.assertDictEqual(result[0], correct_response[0]) - - def test_one_error(self): - correct_response = [{'error': 'The device is already provisioned.', 'location': None}] - self.prepare_response('POST', '/ws/DeviceCore', build_response(correct_response), status=ERRORS_STATUS) - result = self.dc.devicecore._provision(['whatever']) - self.assertIsInstance(result, list) - self.assertEqual(len(result), 1) - self.assertDictEqual(result[0], correct_response[0]) - - def test_two_good(self): - correct_response = [{'location': 'DeviceCore/123474/0', 'error': False}, - {'location': 'DeviceCore/123475/0', 'error': False}] - self.prepare_response('POST', '/ws/DeviceCore', build_response(correct_response), status=SUCCESS_STATUS) - result = self.dc.devicecore._provision(['whatever'] * 2) - self.assertIsInstance(result, list) - self.assertEqual(len(result), 2) - for i in range(len(result)): - self.assertDictEqual(result[i], correct_response[i]) - - def test_two_error(self): - correct_response = [{'location': None, 'error': 'Problem with Device ID: Problem with segment number 4'}, - {'location': None, 'error': 'The device is already provisioned.'}] - self.prepare_response('POST', '/ws/DeviceCore', build_response(correct_response), status=ERRORS_STATUS) - result = self.dc.devicecore._provision(['whatever'] * 2) - self.assertIsInstance(result, list) - self.assertEqual(len(result), 2) - for i in range(len(result)): - self.assertDictEqual(result[i], correct_response[i]) - - def test_two_split(self): - correct_response = [{'location': 'DeviceCore/123474/0', 'error': False}, - {'location': None, 'error': 'The device is already provisioned.'}] - self.prepare_response('POST', '/ws/DeviceCore', build_response(correct_response), status=ERRORS_STATUS) - result = self.dc.devicecore._provision(['whatever'] * 2) - self.assertIsInstance(result, list) - self.assertEqual(len(result), 2) - for i in range(len(result)): - self.assertDictEqual(result[i], correct_response[i]) - - -provision = mock.MagicMock() -provision.post.return_value.status_code = 201 -provision.post.return_value.content = '' - - -class ProvisionByTests(object): - def setUp(self): - HttpTestBase.setUp(self) - global provision - provision.reset_mock() - - def test_single(self): - correct_response = {'location': 'DeviceCore/123474/0', 'error': False} - self.prepare_response('POST', '/ws/DeviceCore', build_response(correct_response), status=SUCCESS_STATUS) - result = self.provision_call('00:40:9d:aa:bb:cc') - self.assertDictEqual(result, correct_response) - - def test_single_list(self): - correct_response = [{'location': 'DeviceCore/123474/0', 'error': False}] - self.prepare_response('POST', '/ws/DeviceCore', build_response(correct_response), status=SUCCESS_STATUS) - result = self.provision_call(['00:40:9d:aa:bb:cc']) - self.assertIsInstance(result, list) - self.assertDictEqual(result[0], correct_response[0]) - - @mock.patch.object(DeviceCoreAPI, '_provision', provision) - def test_single_xml(self): - self.provision_call('00:40:9d:aa:bb:cc') - provision.assert_called_once_with(['<{0}>00:40:9d:aa:bb:cc'.format(self.call_tag)]) - - @mock.patch.object(DeviceCoreAPI, '_provision', provision) - def test_single_list_xml(self): - self.provision_call(['00:40:9d:aa:bb:cc']) - provision.assert_called_once_with(['<{0}>00:40:9d:aa:bb:cc'.format(self.call_tag)]) - - @mock.patch.object(DeviceCoreAPI, '_provision', provision) - def test_multi_list_xml(self): - self.provision_call(['00:40:9d:aa:bb:cc', '00:40:9d:aa:bb:cd']) - provision.assert_called_once_with(['<{0}>00:40:9d:aa:bb:cc'.format(self.call_tag), - '<{0}>00:40:9d:aa:bb:cd'.format(self.call_tag)]) - - -class ProvisionByIdTests(ProvisionByTests, HttpTestBase): - def setUp(self): - HttpTestBase.setUp(self) - ProvisionByTests.setUp(self) - self.provision_call = self.dc.devicecore.provision_by_id - self.call_tag = "devConnectwareId" - - -class ProvisionByMacTests(ProvisionByTests, HttpTestBase): - def setUp(self): - HttpTestBase.setUp(self) - ProvisionByTests.setUp(self) - self.provision_call = self.dc.devicecore.provision_by_mac - self.call_tag = "devMac" - - -if __name__ == '__main__': - unittest.main() From 5aa57eea72666c620e6d719d5dc83daecb86eb52 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Thu, 22 Jan 2015 17:53:50 -0600 Subject: [PATCH 025/140] docs: update developer's guide with 0.2 changes --- HACKING.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/HACKING.md b/HACKING.md index d334e60..a3193bf 100644 --- a/HACKING.md +++ b/HACKING.md @@ -7,7 +7,7 @@ Environment Setup All the requirements in order to perform development on the product should be installable in a virtualenv. - $ pip install -r test-requirements.txt + $ pip install -r dev-requirements.txt In order to build a release you will also need to install pandoc. On Ubuntu, you should be able to do: @@ -49,6 +49,19 @@ errors from pyenv, there may be addition dependencies required. Please visit https://github.com/yyuu/pyenv/wiki/Common-build-problems for additional pointers. +### Running Integration and Unittests + +There are some additional integration tests that run against and actual +device cloud account. These are a bit more fragile and when something +fails, you may need to go to your device cloud account to clean things +up. + +To run those tests, you can just do the following. This script runs +the toxtest.sh script with environment variables set with your +account information. The tests that were skipped before will now +be run with each supported version of the interpreter: + + $ ./inttest.sh Build the Documentation ----------------------- @@ -84,5 +97,5 @@ Each source file should be prefixed with the following header: # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # - # Copyright (c) 2014 Etherios, Inc. All rights reserved. + # Copyright (c) 2015 Etherios, Inc. All rights reserved. # Etherios, Inc. is a Division of Digi International. From 3a3992679d2f8f27852b9a8dcf3576e7ccd2c26b Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 28 Jan 2015 15:27:58 -0600 Subject: [PATCH 026/140] docs: readme and changelog updates for the v0.2 release --- CHANGELOG.md | 34 ++++++++++++++++++++++++++++++++++ HACKING.md | 2 +- README.md | 22 +++++++++++++++------- 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cf823c..d6c1879 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,39 @@ ## Python Devicecloud Library Changelog +### 0.2 / 2015-01-23 +[Full Changelog](https://github.com/etherios/python-devicecloud/compare/0.1.1...0.2) + +Enhancements: + +* Integration tests were added for some APIs to further test library correctness +* Documentation for several parts of the system, including streams + was enhanced. +* Several APIs now support pagination on large result sets that previously lacked support. +* We now expose the underlying object used to send device cloud API requests via + `dc.get_connection()`. This allows users to more easily talk to currently unsupported + APIs. Support for the basic API verbs plus helpers for handling pagination and other + concerns are included. +* Support for group operations in devicecore is now present +* Filedata files and directories can now be deleted +* A new 'ws' API has been added for making direct web service requests + +API Changes: + +* devicecore: `list_devices` was changed to `get_devices`. Previously, the + code did not match the documentation on this front. Calls to `dc.devicecore.list_devices` + will need to be changed to `dc.devicecore.get_devices` + +Bug Fixes: + +* streams: When creating a stream, the stream_id was used for the description rather + than the provided description. +* core: Library now freezes dependencies. Several dependencies updated to latest + versions (e.g. requests). Without this, some combinations of the library + and dependencies caused various errors. + +Thanks to Dan Harrison, Steve Stack, Tom Manley, and Paul Osborne for contributions +going into this release. + ### 0.1.1 / 2014-09-14 [Full Changelog](https://github.com/etherios/python-devicecloud/compare/0.1...0.1.1) diff --git a/HACKING.md b/HACKING.md index a3193bf..43e5c5b 100644 --- a/HACKING.md +++ b/HACKING.md @@ -51,7 +51,7 @@ for additional pointers. ### Running Integration and Unittests -There are some additional integration tests that run against and actual +There are some additional integration tests that run against an actual device cloud account. These are a bit more fragile and when something fails, you may need to go to your device cloud account to clean things up. diff --git a/README.md b/README.md index 83fdc17..1515a7d 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,14 @@ Overview -------- Python-devicecloud is a library providing simple, intuitive access to -the [Device Cloud by Etherios](http://www.etherios.com/products/devicecloud/) for clients written in -Python. +the [Device Cloud by Etherios](http://www.etherios.com/products/devicecloud/) +for clients written in Python. -The library wraps the Device Cloud REST API and hides the details of forming HTTP requests in order to gain access to device information, +The library wraps the Device Cloud REST API and hides the details of +forming HTTP requests in order to gain access to device information, file data, streams, and other features of the device cloud. The API -wrapped can be found [here](http://ftp1.digi.com/support/documentation/90002008_redirect.htm). - +wrapped can be found +[here](http://ftp1.digi.com/support/documentation/90002008_redirect.htm). The primary target audience for this library is individuals interfacing with the device cloud from the server side or developers @@ -70,9 +71,15 @@ This library can be installed using Python 3) are supported by the library. ```sh -pip install python-devicecloud +pip install devicecloud ``` +If you already have an older version of the library installed, you can +upgrade to the latest version by doing + +```sh +pip install --upgrade devicecloud +``` Supported Features ------------------ @@ -102,6 +109,8 @@ is not the case. The current features are supported by the library: * Get full metadata and contents of files and directories. * Low level support for performing basic SCI commands with limited parsing of results and support for only a subset of available services/commands. +* APIs to make direct web service calls to the device cloud with some details + handled by the library (see DeviceCloudConnection and 'ws' documentation) The following features are *not* supported at this time. Feedback on which features should be highest priority is always welcome. @@ -115,7 +124,6 @@ which features should be highest priority is always welcome. * DeviceMetaData * DeviceVendor * FileDataHistory -* Group Operations (CRUD) * NetworkInterface support * XBee specific support (XBeeCore) * Device Provisioning From 007179a645f77c77b5817d3b1202d0cdf9defc9f Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 28 Jan 2015 16:12:44 -0600 Subject: [PATCH 027/140] release: bump library version to 0.2 --- devicecloud/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devicecloud/version.py b/devicecloud/version.py index ab86d3b..851cca9 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -5,4 +5,4 @@ # Copyright (c) 2014 Etherios, Inc. All rights reserved. # Etherios, Inc. is a Division of Digi International. -__version__ = "0.1.1" +__version__ = "0.2" From c7a8dbfb78bae41423cf3f97ab41c00714e6287a Mon Sep 17 00:00:00 2001 From: Stephen Stack Date: Wed, 28 Jan 2015 17:31:03 -0600 Subject: [PATCH 028/140] readme: Move "Device Provision" from not-supported to supported --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1515a7d..0c1d0de 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,7 @@ is not the case. The current features are supported by the library: of results and support for only a subset of available services/commands. * APIs to make direct web service calls to the device cloud with some details handled by the library (see DeviceCloudConnection and 'ws' documentation) +* Device Provisioning via Mac Address, IMEI or Device ID The following features are *not* supported at this time. Feedback on which features should be highest priority is always welcome. @@ -126,7 +127,6 @@ which features should be highest priority is always welcome. * FileDataHistory * NetworkInterface support * XBee specific support (XBeeCore) -* Device Provisioning * Smart Energy APIs * SMS Support * Satellite/Iridium Support From 6b5eb4e4fbb3ed3ac77054b0081cee67fad07a9c Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 28 Jan 2015 17:38:12 -0600 Subject: [PATCH 029/140] changelog: add note about addition of provisioning --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6c1879..f79e80b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ Enhancements: * Support for group operations in devicecore is now present * Filedata files and directories can now be deleted * A new 'ws' API has been added for making direct web service requests +* DeviceCore now has support for provisioning one or multiple devices API Changes: From 471fb704c3b7393241f2ee8da9a8c1b1bf126dd4 Mon Sep 17 00:00:00 2001 From: Dan Harrison Date: Mon, 9 Mar 2015 13:01:42 -0500 Subject: [PATCH 030/140] device_cloud/streams.py: allowed get_streams() to accept a limiting subset parameter --- devicecloud/streams.py | 16 +++++--- devicecloud/test/unit/test_streams.py | 53 +++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/devicecloud/streams.py b/devicecloud/streams.py index 2c4fc77..d66ab58 100644 --- a/devicecloud/streams.py +++ b/devicecloud/streams.py @@ -84,11 +84,15 @@ class StreamsAPI(APIBase): def __init__(self, *args, **kwargs): APIBase.__init__(self, *args, **kwargs) - def _get_streams(self): + def _get_streams(self, uri_suffix=None): """Clear and update internal cache of stream objects""" # TODO: handle paging, perhaps change this to be a generator + if uri_suffix is not None and uri_suffix[0] != '/': + uri_suffix = '/' + uri_suffix + elif uri_suffix is None: + uri_suffix = "" streams = {} - response = self._conn.get_json("/ws/DataStream") + response = self._conn.get_json("/ws/DataStream" + uri_suffix) for stream_data in response["items"]: stream_id = stream_data["streamId"] stream = DataStream(self._conn, stream_id, stream_data) @@ -139,14 +143,16 @@ def create_stream(self, stream_id, data_type, description=None, data_ttl=None, stream = DataStream(self._conn, stream_id) return stream - def get_streams(self): - """Return the iterator over all streams present on the device cloud + def get_streams(self, stream_prefix=None): + """Return the iterator over streams preset on device cloud. + + :param stream_prefix: An optional prefix to limit the iterator to; all streams are returned if it is not specified. :return: iterator over all :class:`.DataStream` instances on the device cloud """ # TODO: deal with paging. We now return a generator, so the interface should look the same - return iter(self._get_streams().values()) + return iter(self._get_streams(stream_prefix).values()) def get_stream(self, stream_id): """Return a reference to a stream with the given ``stream_id`` diff --git a/devicecloud/test/unit/test_streams.py b/devicecloud/test/unit/test_streams.py index 85dd854..fbfe58c 100644 --- a/devicecloud/test/unit/test_streams.py +++ b/devicecloud/test/unit/test_streams.py @@ -65,6 +65,46 @@ } """ +GET_DATA_STREAMS_0 = """ +{ + "resultSize": "1", + "requestedSize": "1000", + "pageCursor": "9d870afb-2-af668f74", + "items": [ + { + "cstId": "7603", + "streamId": "another/test", + "dataType": "INTEGER", + "forwardTo": "", + "description": "Some Integral Thing", + "units": "", + "dataTtl": "172800", + "rollupTtl": "432000" + } + ] +} +""" + +GET_DATA_STREAMS_1 = """ +{ + "resultSize": "1", + "requestedSize": "1000", + "pageCursor": "9d870afb-2-af668f74", + "items": [ + { + "cstId": "7603", + "streamId": "test", + "dataType": "FLOAT", + "forwardTo": "", + "description": "some description", + "units": "light years", + "dataTtl": "172800", + "rollupTtl": "432000" + } + ] +} +""" + GET_DATA_STREAMS_EMPTY = """ { "resultSize": "0", @@ -304,6 +344,19 @@ def test_get_streams(self): self.assertIsInstance(streams[0], DataStream) self.assertIsInstance(streams[1], DataStream) + def test_get_streams_with_id(self): + self.prepare_response("GET", "/ws/DataStream/test", GET_DATA_STREAMS_1) + streams = list(self.dc.streams.get_streams('test')) + self.assertEqual(len(streams), 1) + self.assertIsInstance(streams[0], DataStream) + self.prepare_response("GET", "/ws/DataStream/another", GET_DATA_STREAMS_0) + streams = list(self.dc.streams.get_streams('another')) + self.assertEqual(len(streams), 1) + self.assertIsInstance(streams[0], DataStream) + self.prepare_response("GET", "/ws/DataStream/junk", GET_DATA_STREAMS_EMPTY) + streams = self.dc.streams.get_streams('junk') + self.assertEqual(list(streams), []) + def test_get_stream(self): # Get a stream by ID when there is no cache stream = self.dc.streams.get_stream("/test/stream") From f89d8c8b57c2120bdc61cb8443a1e3620ff44b9d Mon Sep 17 00:00:00 2001 From: Dan Harrison Date: Mon, 9 Mar 2015 15:38:06 -0500 Subject: [PATCH 031/140] devicecloud/streams.py: fixed pull request comments --- devicecloud/streams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devicecloud/streams.py b/devicecloud/streams.py index d66ab58..5196ec4 100644 --- a/devicecloud/streams.py +++ b/devicecloud/streams.py @@ -87,12 +87,12 @@ def __init__(self, *args, **kwargs): def _get_streams(self, uri_suffix=None): """Clear and update internal cache of stream objects""" # TODO: handle paging, perhaps change this to be a generator - if uri_suffix is not None and uri_suffix[0] != '/': + if uri_suffix is not None and not uri_suffix.startswith('/'): uri_suffix = '/' + uri_suffix elif uri_suffix is None: uri_suffix = "" streams = {} - response = self._conn.get_json("/ws/DataStream" + uri_suffix) + response = self._conn.get_json("/ws/DataStream{}".format(uri_suffix)) for stream_data in response["items"]: stream_id = stream_data["streamId"] stream = DataStream(self._conn, stream_id, stream_data) From 1f2318e16cbca89c81f82147d81a8d3c94a5098c Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 10 Mar 2015 15:25:31 -0500 Subject: [PATCH 032/140] build: fix failing (hanging) unit tests with python 3.4.1 There was a bug in version 3.4.0 and 3.4.1 of the interpreter that caused the build to hang in an infinite loop. The fix is to use python 3.4.3 that appears to have a fix. I have not determined the exact change in the interpreter that resolves the issue. Pyenv needs to be updaded in order to know how to build 3.4.3, so code has been added that will ensure that pyenv gets updated each time toxtests.sh is run. The assumption is that the master branch for pyenv will be reasonably stable. --- toxtest.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/toxtest.sh b/toxtest.sh index a7cac3b..7c38aad 100755 --- a/toxtest.sh +++ b/toxtest.sh @@ -9,7 +9,7 @@ pyversions=(2.7.7 3.2.5 3.3.5 - 3.4.1 + 3.4.3 pypy-2.3.1) # first make sure that pyenv is installed @@ -17,6 +17,9 @@ if [ ! -s "$HOME/.pyenv/bin/pyenv" ]; then curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash fi +# Update pyenv (required for new python versions to be available) +(cd $HOME/.pyenv && git pull) + # add pyenv to our path and initialize (if this has not already been done) export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)" From 82b08edccbcd8b8259578913d41d08249a5e7ed5 Mon Sep 17 00:00:00 2001 From: Dan Harrison Date: Mon, 30 Mar 2015 13:41:23 -0500 Subject: [PATCH 033/140] devicecloud/sci.py added get_job_async --- devicecloud/sci.py | 7 +++++++ devicecloud/test/unit/test_sci.py | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/devicecloud/sci.py b/devicecloud/sci.py index 80f0d84..2b07a0f 100644 --- a/devicecloud/sci.py +++ b/devicecloud/sci.py @@ -69,6 +69,13 @@ def to_xml(self): class ServerCommandInterfaceAPI(APIBase): """Encapsulate Server Command Interface API""" + def get_async_job(self, job_id): + """ Query an asynchronous SCI job that was not created with + send_sci_async. """ + uri = "/ws/sci/{0}".format(job_id) + # TODO: do parsing here? + return self._conn.get(uri) + def send_sci(self, operation, target, payload, reply=None, synchronous=None, sync_timeout=None, cache=None, allow_offline=None, wait_for_reconnect=None): """Send SCI request to 1 or more targets diff --git a/devicecloud/test/unit/test_sci.py b/devicecloud/test/unit/test_sci.py index 7ccd163..94b6de5 100644 --- a/devicecloud/test/unit/test_sci.py +++ b/devicecloud/test/unit/test_sci.py @@ -17,6 +17,10 @@ Device Not Connected """ +EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED = """\ +completeDevice Not Connected +""" + class TestSCI(HttpTestBase): def _prepare_sci_response(self, response, status=200): @@ -39,6 +43,13 @@ def test_sci_successful_error(self): '')) +class TestGetAsync(HttpTestBase): + def test_sci_get_async(self): + self.prepare_response("GET", "/ws/sci/123", EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED, 200) + resp = self.dc.get_sci_api().get_async_job(123) + self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.content, EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED) + if __name__ == "__main__": unittest.main() From 3975bd5feecf6d54ec7bf569a5174cf2efe77be9 Mon Sep 17 00:00:00 2001 From: Dan Harrison Date: Mon, 30 Mar 2015 15:52:09 -0500 Subject: [PATCH 034/140] sci.py: added send_sci_async and AsyncRequestProxy --- devicecloud/sci.py | 42 ++++++++++++++ devicecloud/test/unit/test_sci.py | 91 ++++++++++++++++++++++++++++++- 2 files changed, 131 insertions(+), 2 deletions(-) diff --git a/devicecloud/sci.py b/devicecloud/sci.py index 2b07a0f..a244f11 100644 --- a/devicecloud/sci.py +++ b/devicecloud/sci.py @@ -7,6 +7,7 @@ """Server Command Interface functionality""" from devicecloud.apibase import APIBase +from xml.etree import ElementTree as ET import six @@ -66,6 +67,35 @@ def to_xml(self): return ''.format(self._group) +class AsyncRequestProxy(object): + """ An object representing an asynychronous SCI request. Can be used for + polling the status of the corresponding request. + + It has three properties: + - job_id => the ID in device cloud of the job + - response => the response to the request if completed + - complete => True/False value indicating whether request has completed + """ + def __init__(self, job_id, conn): + self.job_id = job_id + self._conn = conn + self.response = None + + @property + def completed(self): + """ Return True if the request has completed, False otherwise. """ + if self.response is not None: + return True + resp = self._conn.get('/ws/sci/{0}'.format(self.job_id)) + dom = ET.fromstring(resp.content) + status = dom.find('.//status') + if status is not None and status.text == 'complete': + self.response = resp.content + return True + else: + return False + + class ServerCommandInterfaceAPI(APIBase): """Encapsulate Server Command Interface API""" @@ -76,6 +106,18 @@ def get_async_job(self, job_id): # TODO: do parsing here? return self._conn.get(uri) + def send_sci_async(self, operation, target, payload, **sci_options): + """ Sends an asynchronous SCI request, and wraps the job in an object + to manage it. """ + sci_options['synchronous'] = False + resp = self.send_sci(operation, target, payload, **sci_options) + dom = ET.fromstring(resp.content) + job_element = dom.find('.//jobId') + if job_element is None: + return + job_id = int(job_element.text) + return AsyncRequestProxy(job_id, self._conn) + def send_sci(self, operation, target, payload, reply=None, synchronous=None, sync_timeout=None, cache=None, allow_offline=None, wait_for_reconnect=None): """Send SCI request to 1 or more targets diff --git a/devicecloud/test/unit/test_sci.py b/devicecloud/test/unit/test_sci.py index 94b6de5..e077173 100644 --- a/devicecloud/test/unit/test_sci.py +++ b/devicecloud/test/unit/test_sci.py @@ -7,20 +7,56 @@ import unittest -from devicecloud.sci import DeviceTarget -from devicecloud.test.unit.test_utilities import HttpTestBase import httpretty +import mock import six +from devicecloud import DeviceCloud +from devicecloud.sci import DeviceTarget, AsyncRequestProxy, ServerCommandInterfaceAPI +from devicecloud.test.unit.test_utilities import HttpTestBase + EXAMPLE_SCI_DEVICE_NOT_CONNECTED = """\ Device Not Connected """ +EXAMPLE_SCI_BAD_DEVICE = """\ + + + + + Invalid target. Device not found. + + + Invalid SCI request. No valid targets found. + + +""" + EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED = """\ completeDevice Not Connected """ +EXAMPLE_ASYNC_SCI_INCOMPLETE = """\ +in_progress +""" + +EXAMPLE_SCI_REQUEST_PAYLOAD = """\ + + + + + +""" + +EXAMPLE_ASYNC_SCI_RESPONSE = """\ + + + 133225503 + + +""" + class TestSCI(HttpTestBase): def _prepare_sci_response(self, response, status=200): @@ -51,5 +87,56 @@ def test_sci_get_async(self): self.assertEqual(resp.content, EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED) +class TestAsyncProxy(HttpTestBase): + def setUp(self): + HttpTestBase.setUp(self) + self.fake_conn = mock.MagicMock() + + def test_ctor(self): + t = AsyncRequestProxy(123, self.fake_conn) + self.assertEqual(t.job_id, 123) + self.assertIs(t.response, None) + + def test_completed_false(self): + self.prepare_response("GET", "/ws/sci/123", EXAMPLE_ASYNC_SCI_INCOMPLETE, 200) + t = AsyncRequestProxy(123, self.dc.get_sci_api()._conn) + self.assertIs(t.completed, False) + self.assertIs(t.response, None) + + def test_completed_true(self): + self.prepare_response("GET", "/ws/sci/123", EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED, 200) + t = AsyncRequestProxy(123, self.dc.get_sci_api()._conn) + self.assertIs(t.completed, True) + self.assertEqual(t.response, EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED) + + def test_completed_already(self): + self.prepare_response("GET", "/ws/sci/123", EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED, 200) + t = AsyncRequestProxy(123, self.dc.get_sci_api()._conn) + t.response = EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED + self.assertIs(t.completed, True) + + +class TestSendSciAsync(HttpTestBase): + @mock.patch.object(ServerCommandInterfaceAPI, "send_sci") + def test_bad_resp(self, fake_send_sci): + fake_resp = mock.MagicMock() + fake_resp.status_code = 400 + fake_resp.reason = "OK" + fake_resp.content = EXAMPLE_SCI_BAD_DEVICE + fake_send_sci.return_value = fake_resp + resp = self.dc.get_sci_api().send_sci_async("send_message", DeviceTarget('00000000-00000000-00409dff-ffaabbcc'), EXAMPLE_SCI_REQUEST_PAYLOAD) + self.assertIs(resp, None) + + @mock.patch.object(ServerCommandInterfaceAPI, "send_sci") + def test_resp_parse(self, fake_send_sci): + fake_resp = mock.MagicMock() + fake_resp.status_code = 200 + fake_resp.reason = "OK" + fake_resp.content = EXAMPLE_ASYNC_SCI_RESPONSE + fake_send_sci.return_value = fake_resp + resp = self.dc.get_sci_api().send_sci_async("send_message", DeviceTarget('00000000-00000000-00409dff-ffaabbcc'), EXAMPLE_SCI_REQUEST_PAYLOAD) + self.assertEqual(resp.job_id, 133225503) + + if __name__ == "__main__": unittest.main() From 97b80f3da9b091ee6daf73d04410cc2bce831e61 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 31 Mar 2015 13:06:03 -0500 Subject: [PATCH 035/140] sci: fixed test issues with python 3 Httpretty returns bytes, so we need to convert the value we are comparing against to also be bytes with Python 3. Signed-off-by: Paul Osborne --- devicecloud/test/unit/test_sci.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devicecloud/test/unit/test_sci.py b/devicecloud/test/unit/test_sci.py index e077173..87b8c3e 100644 --- a/devicecloud/test/unit/test_sci.py +++ b/devicecloud/test/unit/test_sci.py @@ -84,7 +84,7 @@ def test_sci_get_async(self): self.prepare_response("GET", "/ws/sci/123", EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED, 200) resp = self.dc.get_sci_api().get_async_job(123) self.assertEqual(resp.status_code, 200) - self.assertEqual(resp.content, EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED) + self.assertEqual(resp.content, six.b(EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED)) class TestAsyncProxy(HttpTestBase): @@ -107,7 +107,7 @@ def test_completed_true(self): self.prepare_response("GET", "/ws/sci/123", EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED, 200) t = AsyncRequestProxy(123, self.dc.get_sci_api()._conn) self.assertIs(t.completed, True) - self.assertEqual(t.response, EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED) + self.assertEqual(t.response, six.b(EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED)) def test_completed_already(self): self.prepare_response("GET", "/ws/sci/123", EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED, 200) From a1bf79939888068fa123e4e7dcc3c4d4ffee4684 Mon Sep 17 00:00:00 2001 From: Dan Harrison Date: Tue, 31 Mar 2015 14:15:37 -0500 Subject: [PATCH 036/140] sci.py: fixed docstring typo --- devicecloud/sci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devicecloud/sci.py b/devicecloud/sci.py index a244f11..31bb4fe 100644 --- a/devicecloud/sci.py +++ b/devicecloud/sci.py @@ -74,7 +74,7 @@ class AsyncRequestProxy(object): It has three properties: - job_id => the ID in device cloud of the job - response => the response to the request if completed - - complete => True/False value indicating whether request has completed + - completed => True/False value indicating whether request has completed """ def __init__(self, job_id, conn): self.job_id = job_id From ababaaa5c949499153f1c9df995fec14e0ba3ca1 Mon Sep 17 00:00:00 2001 From: Dan Harrison Date: Tue, 31 Mar 2015 14:24:41 -0500 Subject: [PATCH 037/140] sci.py: improved docstrings --- devicecloud/sci.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/devicecloud/sci.py b/devicecloud/sci.py index 31bb4fe..72f8956 100644 --- a/devicecloud/sci.py +++ b/devicecloud/sci.py @@ -68,8 +68,9 @@ def to_xml(self): class AsyncRequestProxy(object): - """ An object representing an asynychronous SCI request. Can be used for - polling the status of the corresponding request. + """An object representing an asynychronous SCI request. + + Can be used for polling the status of the corresponding request. It has three properties: - job_id => the ID in device cloud of the job @@ -83,7 +84,7 @@ def __init__(self, job_id, conn): @property def completed(self): - """ Return True if the request has completed, False otherwise. """ + """Return True if the request has completed, False otherwise""" if self.response is not None: return True resp = self._conn.get('/ws/sci/{0}'.format(self.job_id)) @@ -100,15 +101,28 @@ class ServerCommandInterfaceAPI(APIBase): """Encapsulate Server Command Interface API""" def get_async_job(self, job_id): - """ Query an asynchronous SCI job that was not created with - send_sci_async. """ + """Query an asynchronous SCI job by ID + + This is useful if the job was not created with send_sci_async(). + + :param int job_id: The job ID to query + :returns: The SCI response from GETting the job information + """ uri = "/ws/sci/{0}".format(job_id) # TODO: do parsing here? return self._conn.get(uri) def send_sci_async(self, operation, target, payload, **sci_options): - """ Sends an asynchronous SCI request, and wraps the job in an object - to manage it. """ + """Send an asynchronous SCI request, and wraps the job in an object + to manage it + :param str operation: The operation is one of {send_message, update_firmware, disconnect, query_firmware_targets, + file_system, data_service, and reboot} + :param target: The device(s) to be targeted with this request + :type target: :class:`~.TargetABC` or list of :class:`~.TargetABC` instances + + TODO: document other params + + """ sci_options['synchronous'] = False resp = self.send_sci(operation, target, payload, **sci_options) dom = ET.fromstring(resp.content) From ba0c4e74a3b9c1a98f0eb7f764d91b18113de279 Mon Sep 17 00:00:00 2001 From: Dan Harrison Date: Tue, 31 Mar 2015 14:34:19 -0500 Subject: [PATCH 038/140] sci.py: prettied up sphinx docstrings --- devicecloud/sci.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/devicecloud/sci.py b/devicecloud/sci.py index 72f8956..d619865 100644 --- a/devicecloud/sci.py +++ b/devicecloud/sci.py @@ -72,10 +72,9 @@ class AsyncRequestProxy(object): Can be used for polling the status of the corresponding request. - It has three properties: - - job_id => the ID in device cloud of the job - - response => the response to the request if completed - - completed => True/False value indicating whether request has completed + :ivar job_id: the ID in device cloud of the job + :ivar response: the response to the request if completed + :ivar completed: True if the request has completed, False otherwise; queries on read """ def __init__(self, job_id, conn): self.job_id = job_id @@ -84,7 +83,6 @@ def __init__(self, job_id, conn): @property def completed(self): - """Return True if the request has completed, False otherwise""" if self.response is not None: return True resp = self._conn.get('/ws/sci/{0}'.format(self.job_id)) @@ -115,6 +113,7 @@ def get_async_job(self, job_id): def send_sci_async(self, operation, target, payload, **sci_options): """Send an asynchronous SCI request, and wraps the job in an object to manage it + :param str operation: The operation is one of {send_message, update_firmware, disconnect, query_firmware_targets, file_system, data_service, and reboot} :param target: The device(s) to be targeted with this request From 59e43317bf03fd0b950d197896085be81090ce9b Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Thu, 4 Jun 2015 18:35:52 -0500 Subject: [PATCH 039/140] docs: fix bagdes by migrating to using shields.io --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0c1d0de..07f3242 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,10 @@ Python Device Cloud Library =========================== -[![Build Status](https://travis-ci.org/Etherios/python-devicecloud.svg?branch=master)](https://travis-ci.org/Etherios/python-devicecloud) -[![Coverage Status](https://img.shields.io/coveralls/Etherios/python-devicecloud.svg)](https://coveralls.io/r/Etherios/python-devicecloud) -[![Latest Version](https://pypip.in/version/devicecloud/badge.svg)](https://pypi.python.org/pypi/devicecloud/) -[![Supported Python versions](https://pypip.in/py_versions/devicecloud/badge.svg)](https://pypi.python.org/pypi/devicecloud/) -[![License](https://pypip.in/license/devicecloud/badge.svg)](https://pypi.python.org/pypi/devicecloud/) +[![Build Status](https://img.shields.io/travis/digidotcom/python-devicecloud.svg)](https://travis-ci.org/digidotcom/python-devicecloud) +[![Coverage Status](https://img.shields.io/coveralls/digidotcom/python-devicecloud.svg)](https://coveralls.io/r/digidotcom/python-devicecloud) +[![Latest Version](https://img.shields.io/pypi/v/devicecloud.svg)](https://pypi.python.org/pypi/devicecloud/) +[![License](https://img.shields.io/badge/license-MPL%202.0-blue.svg)](https://github.com/digidotcom/python-devicecloud/blob/master/LICENSE) Be sure to check out the [full documentation](http://etherios.github.io/python-devicecloud). A [Changelog](https://github.com/etherios/python-devicecloud/blob/master/CHANGELOG.md) From 568cdbb0db27df050a17859614ef602ef82132c0 Mon Sep 17 00:00:00 2001 From: Dan Harrison Date: Thu, 16 Apr 2015 18:12:49 -0500 Subject: [PATCH 040/140] filedata.py: added a raw option to disable XML generation, which corrupts binary files --- devicecloud/filedata.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/devicecloud/filedata.py b/devicecloud/filedata.py index 87f9f61..d7ba805 100644 --- a/devicecloud/filedata.py +++ b/devicecloud/filedata.py @@ -54,7 +54,8 @@ def get_filedata(self, condition=None, page_size=1000): for fd_json in self._conn.iter_json_pages("/ws/FileData", page_size=page_size, **params): yield FileDataObject.from_json(self, fd_json) - def write_file(self, path, name, data, content_type=None, archive=False): + def write_file(self, path, name, data, content_type=None, archive=False, + raw=False): """Write a file to the file data store at the given path :param str path: The path (directory) into which the file should be written. @@ -66,6 +67,7 @@ def write_file(self, path, name, data, content_type=None, archive=False): :type content_type: str or None :param bool archive: If true, history will be retained for various revisions of this file. If this is not required, leave as false. + :param bool raw: If true, skip the FileData XML headers (necessary for binary files) """ path = validate_type(path, *six.string_types) @@ -80,19 +82,22 @@ def write_file(self, path, name, data, content_type=None, archive=False): path += "/" name = name.lstrip("/") - if six.PY3: - base64_encoded_data = base64.encodebytes(data).decode('utf-8') - else: - base64_encoded_data = base64.encodestring(data) - sio = six.moves.StringIO() - sio.write("") - if content_type is not None: - sio.write("{}".format(content_type)) - sio.write("file") - sio.write("{}".format(base64_encoded_data)) - sio.write("{}".format(archive_str)) - sio.write("") + if not raw: + if six.PY3: + base64_encoded_data = base64.encodebytes(data).decode('utf-8') + else: + base64_encoded_data = base64.encodestring(data) + + sio.write("") + if content_type is not None: + sio.write("{}".format(content_type)) + sio.write("file") + sio.write("{}".format(base64_encoded_data)) + sio.write("{}".format(archive_str)) + sio.write("") + else: + sio.write(data) params = { "type": "file", From d6b8369b4360efb8294f65863765910c329625d3 Mon Sep 17 00:00:00 2001 From: Dan Harrison Date: Tue, 5 May 2015 16:51:59 -0500 Subject: [PATCH 041/140] devicecore.py: added delete_device function and unit tests --- devicecloud/devicecore.py | 9 +++++++ devicecloud/test/unit/test_devicecore.py | 33 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index 3a42a4b..c8767f4 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -157,6 +157,15 @@ def get_groups(self, condition=None, page_size=1000): for group_data in self._conn.iter_json_pages("/ws/Group", page_size=page_size, **query_kwargs): yield Group.from_json(group_data) + def delete_device(self, dev): + """ Delete a from the cloud account associated with the handle. + + :raises DeviceCloudHttpException: If there is an unexpected error reported by the device cloud. + :param dev: Device object of the device to delete. + :return: the Response from the delete request. + """ + return self._conn.delete('/ws/DeviceCore/%s' % dev.get_device_id()) + def provision_device(self, **kwargs): """Provision a single device with the specified information diff --git a/devicecloud/test/unit/test_devicecore.py b/devicecloud/test/unit/test_devicecore.py index cdc24c0..3375c53 100644 --- a/devicecloud/test/unit/test_devicecore.py +++ b/devicecloud/test/unit/test_devicecore.py @@ -15,6 +15,7 @@ import httpretty from devicecloud.devicecore import ADD_GROUP_TEMPLATE import six +import mock EXAMPLE_GET_DEVICES = { @@ -337,6 +338,38 @@ def test_mixed_error_success_response(self): }) +class TestDeviceCoreDeleting(HttpTestBase): + + def test_delete_device_good(self): + fake_device = mock.MagicMock() + fake_device.get_device_id.return_value = '1234' + self.prepare_response("DELETE", "/ws/DeviceCore/1234", "1 items deleted", status=200) + self.dc.devicecore.delete_device(fake_device) + req = self._get_last_request() + self.assertEqual(req.path, "/ws/DeviceCore/1234") + + def test_delete_device_not_exist(self): + fake_device = mock.MagicMock() + fake_device.get_device_id.return_value = '1234' + self.prepare_response("DELETE", "/ws/DeviceCore/1234", "0 items deleted", status=200) + self.dc.devicecore.delete_device(fake_device) + req = self._get_last_request() + self.assertEqual(req.path, "/ws/DeviceCore/1234") + + def test_delete_device_bad_status(self): + fake_device = mock.MagicMock() + fake_device.get_device_id.return_value = '1234' + self.prepare_response("DELETE", "/ws/DeviceCore/1234", "I pity da foo' who don' know about API changes.", status=400) + try: + self.dc.devicecore.delete_device(fake_device) + except DeviceCloudHttpException: + pass + else: + assert False, "should have thrown exception" + req = self._get_last_request() + self.assertEqual(req.path, "/ws/DeviceCore/1234") + + class TestDeviceCoreDevices(HttpTestBase): def test_dc_get_devices(self): From 7a5d3467fcb7aa8b6cf8fed2f9e4c937ca304d81 Mon Sep 17 00:00:00 2001 From: Stephen Stack Date: Tue, 5 May 2015 17:09:28 -0500 Subject: [PATCH 042/140] docs: update README with supported features --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 07f3242..13d5362 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,7 @@ between the device cloud API and this library. For now, however, that is not the case. The current features are supported by the library: * Getting basic device information via DeviceCore +* Provision and Delete devices via DeviceCore * Listing devices associated with a device cloud account * Interacting with Device Cloud Data Streams * Create Streams From d05604be7928f723742c66bd4a3ded42e04fc537 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Thu, 4 Jun 2015 18:54:58 -0500 Subject: [PATCH 043/140] coverage: newer coverage, python-coveralls, and passenv This change fixes coveralls.io integration by changing a few things: 1. passenv is now required in order for environment variables to make it to the coveralls script as described in the readme for python-coveralls. This is how the script knows how to authenticate with coveralls.io 2. We moved to using python-coveralls instead of coveralls. 3. Due to #2, we can use a newer coverage --- tox.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 065fdbb..81bd799 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,7 @@ envlist = py27,py32,py33,py34,pypy [testenv] +passenv = TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH deps= -rtest-requirements.txt commands=nosetests -m '^(int|unit)?[Tt]est' @@ -9,8 +10,8 @@ commands=nosetests -m '^(int|unit)?[Tt]est' [testenv:coverage] deps= {[testenv]deps} - coverage>=3.6,<3.999 - coveralls + coverage + python-coveralls commands = coverage run --branch --omit={envdir}/* {envbindir}/nosetests coveralls From 3b52b6000391cc00d939fd456f531ffaa3304735 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Thu, 4 Jun 2015 19:49:47 -0500 Subject: [PATCH 044/140] quality: add codeclimate badge and exclude tests There were a few files in tests that, understandably, contain some level of code duplication. According to the codeclimate docs and common sense, tests should be excluded from the judgement of the overall health of the codebase. https://codeclimate.com/github/digidotcom/python-devicecloud --- .codeclimate.yml | 20 ++++++++++++++++++++ README.md | 1 + 2 files changed, 21 insertions(+) create mode 100644 .codeclimate.yml diff --git a/.codeclimate.yml b/.codeclimate.yml new file mode 100644 index 0000000..67e2e7c --- /dev/null +++ b/.codeclimate.yml @@ -0,0 +1,20 @@ +# +# ---Choose Your Languages--- +# To disable analysis for a certain language, set the language to `false`. +# For help setting your languages: +# http://docs.codeclimate.com/article/169-configuring-analysis-languages +# +languages: + Ruby: false + Javascript: false + PHP: false + Python: true + +# +# ---Exclude Files or Directories--- +# List the files or directories you would like excluded from analysis. +# For help setting your exclude paths: +# http://docs.codeclimate.com/article/166-excluding-files-folders +# +exclude_paths: + - devicecloud/test/** diff --git a/README.md b/README.md index 13d5362..e9445b2 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ Python Device Cloud Library [![Build Status](https://img.shields.io/travis/digidotcom/python-devicecloud.svg)](https://travis-ci.org/digidotcom/python-devicecloud) [![Coverage Status](https://img.shields.io/coveralls/digidotcom/python-devicecloud.svg)](https://coveralls.io/r/digidotcom/python-devicecloud) +[![Code Climate](https://img.shields.io/codeclimate/github/digidotcom/python-devicecloud.svg)](https://codeclimate.com/github/digidotcom/python-devicecloud) [![Latest Version](https://img.shields.io/pypi/v/devicecloud.svg)](https://pypi.python.org/pypi/devicecloud/) [![License](https://img.shields.io/badge/license-MPL%202.0-blue.svg)](https://github.com/digidotcom/python-devicecloud/blob/master/LICENSE) From 1b030e4dbd919eee26f608d4f192b2eef14e432d Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Sat, 13 Jun 2015 01:04:02 -0500 Subject: [PATCH 045/140] legal: update copyright to use Digi name instead of Etherios/Digi Signed-off-by: Paul Osborne --- devicecloud/__init__.py | 3 +-- devicecloud/apibase.py | 3 +-- devicecloud/conditions.py | 3 +-- devicecloud/devicecore.py | 3 +-- devicecloud/examples/__init__.py | 3 +-- devicecloud/examples/devicecore_playground.py | 3 +-- devicecloud/examples/filedata_playground.py | 3 +-- devicecloud/examples/streams_playground.py | 3 +-- devicecloud/filedata.py | 3 +-- devicecloud/sci.py | 3 +-- devicecloud/streams.py | 3 +-- devicecloud/test/__init__.py | 3 +-- devicecloud/test/unit/test_core.py | 3 +-- devicecloud/test/unit/test_devicecore.py | 3 +-- devicecloud/test/unit/test_sci.py | 3 +-- devicecloud/test/unit/test_streams.py | 3 +-- devicecloud/test/unit/test_utilities.py | 3 +-- devicecloud/util.py | 3 +-- devicecloud/version.py | 3 +-- 19 files changed, 19 insertions(+), 38 deletions(-) diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index b263921..97ae270 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. import logging import time import json diff --git a/devicecloud/apibase.py b/devicecloud/apibase.py index 96b4ba2..06a582c 100644 --- a/devicecloud/apibase.py +++ b/devicecloud/apibase.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. class APIBase(object): def __init__(self, conn): diff --git a/devicecloud/conditions.py b/devicecloud/conditions.py index a038ae8..81bbf1f 100644 --- a/devicecloud/conditions.py +++ b/devicecloud/conditions.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. """Module with functionality for building queries against cloud resources diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index c8767f4..f1366a1 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. import sys import xml.etree.ElementTree as ET diff --git a/devicecloud/examples/__init__.py b/devicecloud/examples/__init__.py index 5cb2679..02c23fd 100644 --- a/devicecloud/examples/__init__.py +++ b/devicecloud/examples/__init__.py @@ -2,5 +2,4 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. diff --git a/devicecloud/examples/devicecore_playground.py b/devicecloud/examples/devicecore_playground.py index 7056115..7b14c60 100644 --- a/devicecloud/examples/devicecore_playground.py +++ b/devicecloud/examples/devicecore_playground.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. from getpass import getpass from devicecloud import DeviceCloud diff --git a/devicecloud/examples/filedata_playground.py b/devicecloud/examples/filedata_playground.py index 1555761..633dbf4 100644 --- a/devicecloud/examples/filedata_playground.py +++ b/devicecloud/examples/filedata_playground.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. from getpass import getpass from devicecloud import DeviceCloud diff --git a/devicecloud/examples/streams_playground.py b/devicecloud/examples/streams_playground.py index c479170..117fb51 100644 --- a/devicecloud/examples/streams_playground.py +++ b/devicecloud/examples/streams_playground.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. from getpass import getpass from math import pi import pprint diff --git a/devicecloud/filedata.py b/devicecloud/filedata.py index d7ba805..9cc05dd 100644 --- a/devicecloud/filedata.py +++ b/devicecloud/filedata.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. """Provide access to the device cloud filedata API""" diff --git a/devicecloud/sci.py b/devicecloud/sci.py index d619865..7f8a970 100644 --- a/devicecloud/sci.py +++ b/devicecloud/sci.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. """Server Command Interface functionality""" from devicecloud.apibase import APIBase diff --git a/devicecloud/streams.py b/devicecloud/streams.py index 5196ec4..b866afe 100644 --- a/devicecloud/streams.py +++ b/devicecloud/streams.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. r"""Module providing classes for interacting with device cloud data streams""" diff --git a/devicecloud/test/__init__.py b/devicecloud/test/__init__.py index 5cb2679..02c23fd 100644 --- a/devicecloud/test/__init__.py +++ b/devicecloud/test/__init__.py @@ -2,5 +2,4 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. diff --git a/devicecloud/test/unit/test_core.py b/devicecloud/test/unit/test_core.py index 4578fc9..e896fe1 100644 --- a/devicecloud/test/unit/test_core.py +++ b/devicecloud/test/unit/test_core.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. import unittest from devicecloud import DeviceCloudHttpException diff --git a/devicecloud/test/unit/test_devicecore.py b/devicecloud/test/unit/test_devicecore.py index 3375c53..f1b1f33 100644 --- a/devicecloud/test/unit/test_devicecore.py +++ b/devicecloud/test/unit/test_devicecore.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. import copy import datetime import unittest diff --git a/devicecloud/test/unit/test_sci.py b/devicecloud/test/unit/test_sci.py index 87b8c3e..d275caa 100644 --- a/devicecloud/test/unit/test_sci.py +++ b/devicecloud/test/unit/test_sci.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. import unittest diff --git a/devicecloud/test/unit/test_streams.py b/devicecloud/test/unit/test_streams.py index fbfe58c..30fb243 100644 --- a/devicecloud/test/unit/test_streams.py +++ b/devicecloud/test/unit/test_streams.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. import unittest import datetime diff --git a/devicecloud/test/unit/test_utilities.py b/devicecloud/test/unit/test_utilities.py index 3b4786c..73b4818 100644 --- a/devicecloud/test/unit/test_utilities.py +++ b/devicecloud/test/unit/test_utilities.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. import unittest import json diff --git a/devicecloud/util.py b/devicecloud/util.py index 7b7cc50..509ba0c 100644 --- a/devicecloud/util.py +++ b/devicecloud/util.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. import datetime import arrow diff --git a/devicecloud/version.py b/devicecloud/version.py index 851cca9..a045ca6 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -2,7 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2014 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. __version__ = "0.2" From 929adf7442bc18801920e660e7d6a5cc5d2968f4 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Sun, 14 Jun 2015 22:36:32 -0500 Subject: [PATCH 046/140] core: optimization: reuse http sessions and connections Using a requests HTTP session gives us HTTP Session and connection reuse without too much effort. This is in keeping with the device cloud programming guide: http://goo.gl/I1ChjF. Resolves https://jira.digi.com/browse/PYTHONDC-100 Signed-off-by: Paul Osborne --- devicecloud/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index 97ae270..b59a5b0 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -123,6 +123,8 @@ def __init__(self, auth, base_url, self._throttle_delay_init = throttle_delay_init self._throttle_delay_max = throttle_delay_max self._throttle_delay_backoff_coefficient = throttle_delay_backoff_coefficient + self._session = requests.Session() + self._session.auth = auth def _make_url(self, path): if not path.startswith("/"): @@ -142,7 +144,7 @@ def _make_request(self, method, url, **kwargs): remaining_attempts = throttle_retries + 1 retry_delay = throttle_delay_init while remaining_attempts > 0: - response = requests.request(method, url, auth=self._auth, **kwargs) + response = self._session.request(method, url, **kwargs) if response.status_code in SUCCESSFUL_STATUS_CODES: return response elif response.status_code in HTTP_THROTTLED_CODES: From 519ed6886002634d2308311d8d43d68cf5daa76f Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 12 Jun 2015 17:15:54 -0500 Subject: [PATCH 047/140] monitor: initial, untested support for tcp push monitors This commit adds support for TCP push monitors based on the original work done by Andrew Tolbert (@weak) in the idigi-python-monitor-api: https://github.com/digidotcom/idigi-python-monitor-api Most of the code for doing the actual TCP connection are copied from the original project. The code for getting and creating monitors has been updated to use the DeviceCloudConnection pervasive in the library. Relates to: - Monitor Support: https://jira.digi.com/browse/PYTHONDC-54 - TCP Mon Support: https://jira.digi.com/browse/PYTHONDC-55 - Delete Monitors: https://jira.digi.com/browse/PYTHONDC-56 Signed-off-by: Paul Osborne --- devicecloud/__init__.py | 36 ++ devicecloud/apibase.py | 4 + devicecloud/data/__init__.py | 6 + devicecloud/data/devicecloud.crt | 94 ++++ devicecloud/examples/monitor_playground.py | 51 ++ devicecloud/monitor.py | 124 +++++ devicecloud/monitor_tcp.py | 585 +++++++++++++++++++++ 7 files changed, 900 insertions(+) create mode 100644 devicecloud/data/__init__.py create mode 100644 devicecloud/data/devicecloud.crt create mode 100644 devicecloud/examples/monitor_playground.py create mode 100644 devicecloud/monitor.py create mode 100644 devicecloud/monitor_tcp.py diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index b59a5b0..7972e59 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -6,6 +6,7 @@ import logging import time import json +import urlparse from devicecloud.util import validate_type from requests.auth import HTTPBasicAuth @@ -126,6 +127,20 @@ def __init__(self, auth, base_url, self._session = requests.Session() self._session.auth = auth + @property + def hostname(self): + """Get the hostname that this connection is associated with""" + from six.moves.urllib.parse import urlparse + return urlparse(self._base_url).netloc.split(':', 1)[0] + + @property + def username(self): + return self._auth.username + + @property + def password(self): + return self._auth.password + def _make_url(self, path): if not path.startswith("/"): path = "/" + path @@ -354,6 +369,7 @@ def __init__(self, username, password, base_url=None, self._filedata_api = None # filedata property api ref self._devicecore_api = None # devicecore property api ref self._sci_api = None # sci property api ref + self._monitor_api = None # monitor property of api ref self._legacy_api = None # legacy property api ref def has_valid_credentials(self): @@ -401,6 +417,12 @@ def sci(self): self._sci_api = self.get_sci_api() return self._sci_api + @property + def monitor(self): + if self._monitor_api is None: + self._monitor_api = self.get_monitor_api() + return self._monitor_api + @property def ws(self): """Property providing access to the :class:`.WebServiceStub` with a base of ``/ws``""" @@ -471,6 +493,20 @@ def get_sci_api(self): return ServerCommandInterfaceAPI(self._conn) + def get_monitor_api(self): + """Returns a :class:`.MonitorAPI` bound to this device cloud instance + + This provides access to the same API as :attr:`.DeviceCloud.monitor` but will create + a new object (with a new cache) each time called. + + :return: Monitor API object bound to this device cloud account + :rtype: :class:`.MonitorAPI` + + """ + from devicecloud.monitor import MonitorAPI + + return MonitorAPI(self._conn) + def get_web_service_stub(self): """Returns a :class:`.WebServiceStub` bound to this device cloud instance diff --git a/devicecloud/apibase.py b/devicecloud/apibase.py index 06a582c..3e67f02 100644 --- a/devicecloud/apibase.py +++ b/devicecloud/apibase.py @@ -5,5 +5,9 @@ # Copyright (c) 2015 Digi International, Inc. class APIBase(object): + """Base class for all API Classes + + :type _conn: devicecloud.DeviceCloudConnection + """ def __init__(self, conn): self._conn = conn diff --git a/devicecloud/data/__init__.py b/devicecloud/data/__init__.py new file mode 100644 index 0000000..cbfb263 --- /dev/null +++ b/devicecloud/data/__init__.py @@ -0,0 +1,6 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015 Digi International, Inc. All rights reserved. +# diff --git a/devicecloud/data/devicecloud.crt b/devicecloud/data/devicecloud.crt new file mode 100644 index 0000000..9641564 --- /dev/null +++ b/devicecloud/data/devicecloud.crt @@ -0,0 +1,94 @@ +-----BEGIN CERTIFICATE----- +MIID7jCCAtagAwIBAgIQHywezy3AzTcnh1pWVDNUujANBgkqhkiG9w0BAQUFADA8 +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMVGhhd3RlLCBJbmMuMRYwFAYDVQQDEw1U +aGF3dGUgU1NMIENBMB4XDTEyMDEyNTAwMDAwMFoXDTE3MDEyMzIzNTk1OVowgYYx +CzAJBgNVBAYTAlVTMRIwEAYDVQQIEwlNaW5uZXNvdGExEzARBgNVBAcUCk1pbm5l +dG9ua2ExGzAZBgNVBAoUEkRpZ2kgSW50ZXJuYXRpb25hbDEbMBkGA1UECxQSRGln +aSBJbnRlcm5hdGlvbmFsMRQwEgYDVQQDFAsqLmlkaWdpLmNvbTCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAMhPXg6GdwDI/z8K13CQzyIPbebrjG1KrV1F +qKBOshm9Iqf4DrZ+pNSA/ckOa2WrSZjMHJPTprAaRL+gh/Ymh3WDCLCKIZy3fm7Q +jqTNRLDJipNypBrc8OJmaz7I7wBfAKo6mJU+GlyTvR3fyBaqeihVbqE+aOhtMQvq +I+PjcvNXRx9c4YZrz1swKDwZo36A3zDtefQo/ZK08W+eGP4HEasJeRl63hfo24CT +Se4r02c4TRRJcNbTPG2jueiUrawPy4hh9f2y/luQ6UZfnqxO/CTep5AVGQcDZt+O +ItDl42EMmPvhYFy74fHYlEOVWkIR836uExPHBAD/YMEQuq3BeckCAwEAAaOBoDCB +nTAMBgNVHRMBAf8EAjAAMDoGA1UdHwQzMDEwL6AtoCuGKWh0dHA6Ly9zdnItb3Yt +Y3JsLnRoYXd0ZS5jb20vVGhhd3RlT1YuY3JsMB0GA1UdJQQWMBQGCCsGAQUFBwMB +BggrBgEFBQcDAjAyBggrBgEFBQcBAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly9v +Y3NwLnRoYXd0ZS5jb20wDQYJKoZIhvcNAQEFBQADggEBADOWE49SbClVY/X7TWDf +UIJk6ItfQBS+h2i9e5uKmiAGu9akkM9weVOLGfs+jsGSJnqKlossfqfMKB74hO8q +gmYgYtSx+LLjzSxkcHkGSYGErtAs2EemddB4zkUurK+oZ3qNpfCYPnWbzboNM4Ip +nJ7I7K9bNkbEa9Q0m4iu06Uc+yUmA/DvgStmBHHRsLfVB+KiuuwXcLPSxr0mi7IQ +eaj8GKGhBMGPVr3mvhFrKusCNEX4gxiX35s6JiF5gKnZ3AQitWez5gpOjhi14Yv3 +UyRLm2WhxcYY+zzWlfzfyJyNL+rlfoC1IQbfwISvBW14VjbCqgiOmPfXuoYrhCAX +EHk= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEbDCCA1SgAwIBAgIQTV8sNAiyTCDNbVB+JE3J7DANBgkqhkiG9w0BAQUFADCB +qTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjEoMCYGA1UECxMf +Q2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYGA1UECxMvKGMpIDIw +MDYgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxHzAdBgNV +BAMTFnRoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EwHhcNMTAwMjA4MDAwMDAwWhcNMjAw +MjA3MjM1OTU5WjA8MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMVGhhd3RlLCBJbmMu +MRYwFAYDVQQDEw1UaGF3dGUgU1NMIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAmeSFW3ZJfS8F2MWsyMip09yY5tc0pi8M8iIm2KPJFEyPBaRF6BQM +WJAFGrfFwQalgK+7HUlrUjSIw1nn72vEJ0GMK2Yd0OCjl5gZNEtB1ZjVxwWtouTX +7QytT8G1sCH9PlBTssSQ0NQwZ2ya8Q50xMLciuiX/8mSrgGKVgqYMrAAI+yQGmDD +7bs6yw9jnw1EyVLhJZa/7VCViX9WFLG3YR0cB4w6LPf/gN45RdWvGtF42MdxaqMZ +pzJQIenyDqHGEwNESNFmqFJX1xG0k4vlmZ9d53hR5U32t1m0drUJN00GOBN6HAiY +XMRISstSoKn4sZ2Oe3mwIC88lqgRYke7EQIDAQABo4H7MIH4MDIGCCsGAQUFBwEB +BCYwJDAiBggrBgEFBQcwAYYWaHR0cDovL29jc3AudGhhd3RlLmNvbTASBgNVHRMB +Af8ECDAGAQH/AgEAMDQGA1UdHwQtMCswKaAnoCWGI2h0dHA6Ly9jcmwudGhhd3Rl +LmNvbS9UaGF3dGVQQ0EuY3JsMA4GA1UdDwEB/wQEAwIBBjAoBgNVHREEITAfpB0w +GzEZMBcGA1UEAxMQVmVyaVNpZ25NUEtJLTItOTAdBgNVHQ4EFgQUp6KDuzRFQD38 +1TBPErk+oQGf9tswHwYDVR0jBBgwFoAUe1tFz6/Oy3r9MZIaarbzRutXSFAwDQYJ +KoZIhvcNAQEFBQADggEBAIAigOBsyJUW11cmh/NyNNvGclYnPtOW9i4lkaU+M5en +S+Uv+yV9Lwdh+m+DdExMU3IgpHrPUVFWgYiwbR82LMgrsYiZwf5Eq0hRfNjyRGQq +2HGn+xov+RmNNLIjv8RMVR2OROiqXZrdn/0Dx7okQ40tR0Tb9tiYyLL52u/tKVxp +EvrRI5YPv5wN8nlFUzeaVi/oVxBw9u6JDEmJmsEj9cIqzEHPIqtlbreUgm0vQF9Y +3uuVK6ZyaFIZkSqudZ1OkubK3lTqGKslPOZkpnkfJn1h7X3S5XFV2JMXfBQ4MDzf +huNMrUnjl1nOG5srztxl1Asoa06ERlFE9zMILViXIa4= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIERTCCA66gAwIBAgIQM2VQCHmtc+IwueAdDX+skTANBgkqhkiG9w0BAQUFADCB +zjELMAkGA1UEBhMCWkExFTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJ +Q2FwZSBUb3duMR0wGwYDVQQKExRUaGF3dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UE +CxMfQ2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjEhMB8GA1UEAxMYVGhh +d3RlIFByZW1pdW0gU2VydmVyIENBMSgwJgYJKoZIhvcNAQkBFhlwcmVtaXVtLXNl +cnZlckB0aGF3dGUuY29tMB4XDTA2MTExNzAwMDAwMFoXDTIwMTIzMDIzNTk1OVow +gakxCzAJBgNVBAYTAlVTMRUwEwYDVQQKEwx0aGF3dGUsIEluYy4xKDAmBgNVBAsT +H0NlcnRpZmljYXRpb24gU2VydmljZXMgRGl2aXNpb24xODA2BgNVBAsTLyhjKSAy +MDA2IHRoYXd0ZSwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MR8wHQYD +VQQDExZ0aGF3dGUgUHJpbWFyeSBSb290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEArKDw+4BZ1JzHpM+doVlzCRBFDA0sbmjxbFtIaElZN/wLMxnC +d3/MEC2VNBzm600JpxzSuMmXNgK3idQkXwbAzESUlI0CYm/rWt0RjSiaXISQEHoN +vXRmL2o4oOLVVETrHQefB7pv7un9Tgsp9T6EoAHxnKv4HH6JpOih2HFlDaNRe+68 +0iJgDblbnd+6/FFbC6+Ysuku6QToYofeK8jXTsFMZB7dz4dYukpPymgHHRydSsbV +L5HMfHFyHMXAZ+sy/cmSXJTahcCbv1N9Kwn0jJ2RH5dqUsveCTakd9h7h1BE1T5u +KWn7OUkmHgmlgHtALevoJ4XJ/mH9fuZ8lx3VnQIDAQABo4HCMIG/MA8GA1UdEwEB +/wQFMAMBAf8wOwYDVR0gBDQwMjAwBgRVHSAAMCgwJgYIKwYBBQUHAgEWGmh0dHBz +Oi8vd3d3LnRoYXd0ZS5jb20vY3BzMA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQU +e1tFz6/Oy3r9MZIaarbzRutXSFAwQAYDVR0fBDkwNzA1oDOgMYYvaHR0cDovL2Ny +bC50aGF3dGUuY29tL1RoYXd0ZVByZW1pdW1TZXJ2ZXJDQS5jcmwwDQYJKoZIhvcN +AQEFBQADgYEAhKhMyT4qvJrizI8LsiV3xGGJiWNa1KMVQNT7Xj+0Q+pjFytrmXSe +Cajd1FYVLnp5MV9jllMbNNkV6k9tcMq+9oKp7dqFd8x2HGqBCiHYQZl/Xi6Cweiq +95OBBaqStB+3msAHF/XLxrRMDtdW3HEgdDjWdMbWj2uvi42gbCkLYeA= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDJzCCApCgAwIBAgIBATANBgkqhkiG9w0BAQQFADCBzjELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYD +VQQKExRUaGF3dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlv +biBTZXJ2aWNlcyBEaXZpc2lvbjEhMB8GA1UEAxMYVGhhd3RlIFByZW1pdW0gU2Vy +dmVyIENBMSgwJgYJKoZIhvcNAQkBFhlwcmVtaXVtLXNlcnZlckB0aGF3dGUuY29t +MB4XDTk2MDgwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgc4xCzAJBgNVBAYTAlpB +MRUwEwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEdMBsG +A1UEChMUVGhhd3RlIENvbnN1bHRpbmcgY2MxKDAmBgNVBAsTH0NlcnRpZmljYXRp +b24gU2VydmljZXMgRGl2aXNpb24xITAfBgNVBAMTGFRoYXd0ZSBQcmVtaXVtIFNl +cnZlciBDQTEoMCYGCSqGSIb3DQEJARYZcHJlbWl1bS1zZXJ2ZXJAdGhhd3RlLmNv +bTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA0jY2aovXwlue2oFBYo847kkE +VdbQ7xwblRZH7xhINTpS9CtqBo87L+pW46+GjZ4X9560ZXUCTe/LCaIhUdib0GfQ +ug2SBhRz1JPLlyoAnFxODLz6FVL88kRu2hFKbgifLy3j+ao6hnO2RlNYyIkFvYMR +uHM/qgeN9EJN50CdHDcCAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG +9w0BAQQFAAOBgQAmSCwWwlj66BZ0DKqqX1Q/8tfJeGBeXm43YyJ3Nn6yF8Q0ufUI +hfzJATj/Tb7yFkJD57taRvvBxhEf8UqwKEbJw8RCfbz6q1lu1bdRiBHjpIUZa4JM +pAwSremkrj/xw0llmozFyD4lt5SZu5IycQfwhl7tUCemDaYj+bvLpgcUQg== +-----END CERTIFICATE----- diff --git a/devicecloud/examples/monitor_playground.py b/devicecloud/examples/monitor_playground.py new file mode 100644 index 0000000..61b971a --- /dev/null +++ b/devicecloud/examples/monitor_playground.py @@ -0,0 +1,51 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (C) 2015, Digi International, Inc.. +from getpass import getpass +import random +import time + +from devicecloud import DeviceCloud +from devicecloud.filedata import fd_name, fd_size, fd_type, fd_path +from devicecloud.streams import DataPoint +import six +from six.moves import input + +def get_authenticated_dc(): + while True: + user = input("username: ") + password = getpass("password: ") + dc = DeviceCloud(user, password, base_url="https://login.etherios.com") + if dc.has_valid_credentials(): + print("Credentials accepted!") + return dc + else: + print("Invalid username or password provided, try again") + + +if __name__ == '__main__': + dc = get_authenticated_dc() + + # Create a fresh monitor over a pretty broad set of topics + topics = ['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint'] + mon = dc.monitor.get_monitor(topics) + if mon is not None: + mon.delete() + mon = dc.monitor.create_monitor(topics) + + def listener(*args, **kwargs): + print args, kwargs + + mon.add_listener(listener) + + test_stream = dc.streams.get_stream("test") + try: + while True: + test_stream.write(DataPoint(random.random())) + time.sleep(3.14) + except KeyboardInterrupt: + print("Shutting down threads...") + + dc.monitor.stop_listeners() diff --git a/devicecloud/monitor.py b/devicecloud/monitor.py new file mode 100644 index 0000000..928c21d --- /dev/null +++ b/devicecloud/monitor.py @@ -0,0 +1,124 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015 Digi International, Inc. All rights reserved. +# +# This code is originally from another Digi Open Source Library: +# https://github.com/digidotcom/idigi-python-monitor-api +import xml.etree.ElementTree as ET +import logging +import textwrap +from devicecloud.apibase import APIBase +from devicecloud.conditions import Attribute +from devicecloud.monitor_tcp import TCPClientManager + +logger = logging.getLogger(__name__) + + +class MonitorAPI(APIBase): + """Provide access to the device cloud Monitor API for receiving push notifiactions + + The Monitor API in the device cloud allows for the creation and destructions of + multiple "monitors." Each monitor is registered against one or more "topics" + which describe the data in which it is interested. + + There are, in turn, two main ways to receive data matching the topics for a + given monitor + + 1. Stream: The device cloud supports a protocol over TCP (optionally with SSL) over which + the batches of events will be sent when they are received. + 2. HTTP: When batches of events are received, a configured web + service endpoint will received a POST request with the new data. + + Currently, this library supports setting up both types of monitors, but there + is no special support provided for parsing HTTP postback requests. + """ + + def __init__(self, conn): + APIBase.__init__(self, conn) + # TODO: determine best way to expose additional options + self._tcp_client_manager = TCPClientManager(self._conn, secure=True) + + def create_monitor(self, topics, batch_size=1, batch_duration=0, transport_type='tcp', + compression='gzip', format_type='json'): + """Creates a Monitor instance in the device cloud for a given list of topics + + :param topics: a string list of topics (e.g. ['DeviceCore[U]', + 'FileDataCore']). + :param batch_size: How many Msgs received before sending data. + :param batch_duration: How long to wait before sending batch if it + does not exceed batch_size. + :param transport_type: Either 'tcp' or 'http' + :param compression: Compression value (i.e. 'gzip'). + :param format_type: What format server should send data in (i.e. + 'xml' or 'json'). + + Returns a string of the created Monitor Id (e.g.. 9001) + """ + + monitor_xml = """\ + + {topics} + {batch_size} + {format_type} + {transport_type} + {compression} + + """.format( + topics=','.join(topics), + batch_size=batch_size, + batch_duration=batch_duration, + format_type=format_type, + transport_type=transport_type, + compression=compression, + ) + monitor_xml = textwrap.dedent(monitor_xml) + + response = self._conn.post("/ws/Monitor", monitor_xml) + location = ET.fromstring(response.text).find('.//location').text + monitor_id = int(location.split('/')[-1]) + return DeviceCloudMonitor(self._conn, self._tcp_client_manager, monitor_id) + + def get_monitor(self, topics): + """Attempts to find a Monitor in device cloud that matches the provided topics + + :param topics: a string list of topics (e.g. ``['DeviceCore[U]', 'FileDataCore'])`` + + Returns a :class:`DeviceCloudMonitor` if found, otherwise None. + """ + condition = (Attribute("monTopic") == ",".join(topics)) + for monitor_data in self._conn.iter_json_pages("/ws/Monitor", + condition=condition.compile()): + # just return the first one + return DeviceCloudMonitor.from_json(self._conn, self._tcp_client_manager, monitor_data) + return None + + def stop_listeners(self): + """Stop any listener threads that may be running and join on them""" + self._tcp_client_manager.stop() + + +class DeviceCloudMonitor(object): + """Provides access to a single monitor instance on the device cloud + + :type _tcp_client_manager: devicecloud.monitor_tcp.TCPClientManager + :type _conn: devicecloud.DeviceCloudConnection + """ + + @classmethod + def from_json(cls, conn, tcp_client_manager, monitor_data): + return cls(conn, tcp_client_manager, monitor_data['monId']) + + def __init__(self, conn, tcp_client_manager, monitor_id): + self._conn = conn + self._tcp_client_manager = tcp_client_manager + self._id = monitor_id + + def delete(self): + """Delete this monitor form the device cloud""" + self._conn.delete("/ws/Monitor/{id}".format(id=self._id)) + + def add_listener(self, callback): + """Create a secure SSL/TCP listen session to the device cloud""" + self._tcp_client_manager.create_session(callback, self._id) diff --git a/devicecloud/monitor_tcp.py b/devicecloud/monitor_tcp.py new file mode 100644 index 0000000..bf17021 --- /dev/null +++ b/devicecloud/monitor_tcp.py @@ -0,0 +1,585 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015 Digi International, Inc. All rights reserved. +# +# This code is originally from another Digi Open Source Library: +# https://github.com/digidotcom/idigi-python-monitor-api + +from Queue import Empty, Queue +import logging +import socket +import struct +from threading import Thread +import errno +import select +import zlib +import ssl + +import pkg_resources + +DEFAULT_CRT_NAME = "devicecloud.crt" + +# Push Opcodes. +CONNECTION_REQUEST = 0x01 +CONNECTION_RESPONSE = 0x02 +PUBLISH_MESSAGE = 0x03 +PUBLISH_MESSAGE_RECEIVED = 0x04 + +# Data has not been completely read. +INCOMPLETE = -1 +# No Data Received on Socket. +NO_DATA = -2 + +# Possible Responses from iDigi with respect to Push. +STATUS_OK = 200 +STATUS_UNAUTHORIZED = 403 +STATUS_BAD_REQUEST = 400 + +# Ports to Connect on for Push. +PUSH_OPEN_PORT = 3200 +PUSH_SECURE_PORT = 3201 + + +def _read_msg_header(session): + """ + Perform a read on input socket to consume headers and then return + a tuple of message type, message length. + + :param session: Push Session to read data for. + + Returns response type (i.e. PUBLISH_MESSAGE) if header was completely + read, otherwise None if header was not completely read. + """ + try: + data = session.socket.recv(6 - len(session.data)) + if len(data) == 0: # No Data on Socket. Likely closed. + return NO_DATA + session.data += data + # Data still not completely read. + if len(session.data) < 6: + return INCOMPLETE + + except ssl.SSLError: + # This can happen when select gets triggered + # for an SSL socket and data has not yet been + # read. + return INCOMPLETE + + session.message_length = struct.unpack('!i', session.data[2:6])[0] + response_type = struct.unpack('!H', session.data[0:2])[0] + + # Clear out session data as header is consumed. + session.data = "" + return response_type + + +def _read_msg(session): + """ + Perform a read on input socket to consume message and then return the + payload and block_id in a tuple. + + :param session: Push Session to read data for. + """ + if len(session.data) == session.message_length: + # Data Already completely read. Return + return True + + try: + data = session.socket.recv(session.message_length - len(session.data)) + if len(data) == 0: + raise PushException("No Data on Socket!") + session.data += data + except ssl.SSLError: + # This can happen when select gets triggered + # for an SSL socket and data has not yet been + # read. Wait for it to get triggered again. + return False + + # Whether or not all data was read. + return len(session.data) == session.message_length + + +class PushException(Exception): + """ + Indicates an issue interacting with iDigi Push Functionality. + """ + pass + + +class PushSession(object): + """ + A PushSession is responsible for establishing a socket connection + with iDigi to receive events generated by Devices connected to + iDigi. + """ + + def __init__(self, callback, monitor_id, client): + """ + Creates a PushSession for use with interacting with iDigi's + Push Functionality. + + :param callback: The callback function to invoke when data received. + Must have 1 required parameter that will contain the payload. + :param monitor_id: The id of the Monitor to observe. + :param client: The client object this session is derived from. + """ + self.callback = callback + self.monitor_id = monitor_id + self.client = client + self.socket = None + self.log = logging.getLogger("push_session[%s]" % monitor_id) + + # Received protocol data holders. + self.data = "" + self.message_length = 0 + + def send_connection_request(self): + """ + Sends a ConnectionRequest to the iDigi server using the credentials + established with the id of the monitor as defined in the monitor + member. + """ + try: + self.log.info("Sending ConnectionRequest for Monitor %s." + % self.monitor_id) + # Send connection request and perform a receive to ensure + # request is authenticated. + # Protocol Version = 1. + payload = struct.pack('!H', 0x01) + # Username Length. + payload += struct.pack('!H', len(self.client.username)) + # Username. + payload += self.client.username + # Password Length. + payload += struct.pack('!H', len(self.client.password)) + # Password. + payload += self.client.password + # Monitor ID. + payload += struct.pack('!L', int(self.monitor_id)) + + # Header 6 Bytes : Type [2 bytes] & Length [4 Bytes] + # ConnectionRequest is Type 0x01. + data = struct.pack("!HL", CONNECTION_REQUEST, len(payload)) + + # The full payload. + data += payload + + # Send Connection Request. + self.socket.send(data) + + # Set a 60 second blocking on recv, if we don't get any data + # within 60 seconds, timeout which will throw an exception. + self.socket.settimeout(60) + + # Should receive 10 bytes with ConnectionResponse. + response = self.socket.recv(10) + + # Make socket blocking. + self.socket.settimeout(0) + + if len(response) != 10: + raise PushException("Length of Connection Request Response " + "(%d) is not 10." % len(response)) + + # Type + response_type = int(struct.unpack("!H", response[0:2])[0]) + if response_type != CONNECTION_RESPONSE: + raise PushException( + "Connection Response Type (%d) is not " + "ConnectionResponse Type (%d)." % (response_type, CONNECTION_RESPONSE)) + + status_code = struct.unpack("!H", response[6:8])[0] + self.log.info("Got ConnectionResponse for Monitor %s. Status %s." + % (self.monitor_id, status_code)) + if status_code != STATUS_OK: + raise PushException("Connection Response Status Code (%d) is \ +not STATUS_OK (%d)." % (status_code, STATUS_OK)) + except Exception, exception: + # TODO(posborne): This is bad! It isn't necessarily a socket exception! + # Likely a socket exception, close it and raise an exception. + self.socket.close() + self.socket = None + raise exception + + def start(self): + """ + Creates a TCP connection to the iDigi Server and sends a + ConnectionRequest message. + """ + self.log.info("Starting Insecure Session for Monitor %s." + % self.monitor_id) + if self.socket is not None: + raise Exception("Socket already established for %s." % self) + + try: + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.socket.connect((self.client.hostname, PUSH_OPEN_PORT)) + self.socket.setblocking(0) + except Exception, exception: + self.socket.close() + self.socket = None + raise exception + + self.send_connection_request() + + def stop(self): + """ + Closes the socket associated with this session and puts Session + into a state such that it can be re-established later. + """ + if self.socket is not None: + self.socket.close() + self.socket = None + self.data = None + + +class SecurePushSession(PushSession): + """ + SecurePushSession extends PushSession by wrapping the socket connection + in SSL. It expects the certificate to match any of those in the passed + in ca_certs member file. + """ + + def __init__(self, callback, monitor_id, client, ca_certs=None): + """ + Creates a PushSession wrapped in SSL for use with interacting with + the device cloud push functionality. + + :param callback: The callback function to invoke when data is received. + Must have 1 required parameter that will contain the + payload. + :param monitor_id: The id of the Monitor to observe. + :param client: The client object this session is derived from. + :param ca_certs: Path to a file containing Certificates. + If not provided, the devicecloud.crt file provided with the module will + be used. In most cases, the devicecloud.crt file should be acceptable. + """ + PushSession.__init__(self, callback, monitor_id, client) + # Fall back on devicecloud.crt in the same path as this module if not + # specified. + if ca_certs is None: + ca_certs = pkg_resources.resource_filename("devicecloud.data", DEFAULT_CRT_NAME) + self.ca_certs = ca_certs + + def start(self): + """ + Creates a SSL connection to the iDigi Server and sends a + ConnectionRequest message. + """ + self.log.info("Starting SSL Session for Monitor %s." + % self.monitor_id) + if self.socket is not None: + raise Exception("Socket already established for %s." % self) + + try: + # Create socket, wrap in SSL and connect. + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # Validate that certificate server uses matches what we expect. + if self.ca_certs is not None: + self.socket = ssl.wrap_socket(self.socket, + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=self.ca_certs) + else: + self.socket = ssl.wrap_socket(self.socket) + + self.socket.connect((self.client.hostname, PUSH_SECURE_PORT)) + self.socket.setblocking(0) + except Exception, exception: + self.socket.close() + self.socket = None + raise exception + + self.send_connection_request() + + +class CallbackWorkerPool(object): + """ + A Worker Pool implementation that creates a number of predefined threads + used for invoking Session callbacks. + """ + + def __init__(self, write_queue=None, size=1): + """ + Creates a Callback Worker Pool for use in invoking Session Callbacks + when data is received by a push client. + + :param write_queue: Queue used for queueing up socket write events + for when a payload message is received and processed. + :param size: The number of worker threads to invoke callbacks. + """ + # Used to queue up PublishMessageReceived events to be sent back to + # the iDigi server. + self.__write_queue = write_queue + # Used to queue up sessions and data to callback with. + self.__queue = Queue(size) + # Number of workers to create. + self.size = size + self.log = logging.getLogger('callback_worker_pool') + + for _ in range(size): + worker = Thread(target=self._consume_queue) + worker.daemon = True + worker.start() + + def _consume_queue(self): + """ + Continually blocks until data is on the internal queue, then calls + the session's registered callback and sends a PublishMessageReceived + if callback returned True. + """ + while True: + session, block_id, data = self.__queue.get() + try: + if session.callback(data): + # Send a Successful PublishMessageReceived with the + # block id sent in request + if self.__write_queue is not None: + response_message = struct.pack('!HHH', + PUBLISH_MESSAGE_RECEIVED, + block_id, 200) + self.__write_queue.put((session.socket, + response_message)) + except Exception, exception: + self.log.exception(exception) + + self.__queue.task_done() + + def queue_callback(self, session, block_id, data): + """ + Queues up a callback event to occur for a session with the given + payload data. Will block if the queue is full. + + :param session: the session with a defined callback function to call. + :param block_id: the block_id of the message received. + :param data: the data payload of the message received. + """ + self.__queue.put((session, block_id, data)) + + +class TCPClientManager(object): + """A Client for the 'Push' feature in the device cloud""" + + def __init__(self, conn, secure=True, ca_certs=None, workers=1): + """ + Arbitrator for multiple TCP Client Sessions + + :param conn: The :class:`devicecloud.DeviceCloudConnection` to use + :param secure: Whether or not to create a secure SSL wrapped session. + :param ca_certs: Path to a file containing Certificates. + If not provided, the devicecloud.crt file provided with the module will + be used. In most cases, the devicecloud.crt file should be acceptable. + :param workers: Number of workers threads to process callback calls. + """ + self._conn = conn + self._secure = secure + self._ca_certs = ca_certs + + # A dict mapping Sockets to their PushSessions + self.sessions = {} + # IO thread is used monitor sockets and consume data. + self._io_thread = None + # Writer thread is used to send data on sockets. + self._writer_thread = None + # Write queue is used to queue up data to write to sockets. + self._write_queue = Queue() + # A pool that monitors callback events and invokes them. + self._callback_pool = CallbackWorkerPool(self._write_queue, size=workers) + + self.closed = False + self.log = logging.getLogger(__name__) + + @property + def hostname(self): + return self._conn.hostname + + @property + def username(self): + return self._conn.username + + @property + def password(self): + return self._conn.password + + def _restart_session(self, session): + """ + Restarts and re-establishes session. + + :param session: The session to restart. + """ + # remove old session key, if socket is None, that means the + # session was closed by user and there is no need to restart. + if session.socket is not None: + self.log.info("Attempting restart session for Monitor Id %s." + % session.monitor_id) + del self.sessions[session.socket.fileno()] + session.stop() + session.start() + self.sessions[session.socket.fileno()] = session + + def _writer(self): + """ + Indefinitely checks the writer queue for data to write + to socket. + """ + while not self.closed: + try: + sock, data = self._write_queue.get(timeout=0.1) + self._write_queue.task_done() + sock.send(data) + except Empty: + pass # nothing to write after timeout + except socket.error, err: + if err.errno == errno.EBADF: + self._clean_dead_sessions() + + def _clean_dead_sessions(self): + """ + Traverses sessions to determine if any sockets + were removed (indicates a stopped session). + In these cases, remove the session. + """ + for sck in self.sessions.keys(): + session = self.sessions[sck] + if session.socket is None: + del self.sessions[sck] + + def _select(self): + """ + While the client is not marked as closed, performs a socket select + on all PushSession sockets. If any data is received, parses and + forwards it on to the callback function. If the callback is + successful, a PublishMessageReceived message is sent. + """ + try: + while not self.closed: + try: + inputready = \ + select.select(self.sessions.keys(), [], [], 0.1)[0] + for sock in inputready: + session = self.sessions[sock] + sck = session.socket + + if sck is None: + # Socket has since been deleted, continue + continue + + # If no defined message length, nothing has been + # consumed yet, parse the header. + if session.message_length == 0: + # Read header information before receiving rest of + # message. + response_type = _read_msg_header(session) + if response_type == NO_DATA: + # No data could be read, assume socket closed. + if session.socket is not None: + self.log.error("Socket closed for Monitor %s." % session.monitor_id) + self._restart_session(session) + continue + elif response_type == INCOMPLETE: + # More Data to be read. Continue. + continue + elif response_type != PUBLISH_MESSAGE: + self.log.warn("Response Type (%x) does " \ + "not match PublishMessage (%x)" \ + % (response_type, PUBLISH_MESSAGE)) + continue + + try: + if not _read_msg(session): + # Data not completely read, continue. + continue + except PushException, err: + # If Socket is None, it was closed, + # otherwise it was closed when it shouldn't + # have been restart it. + session.data = "" + session.message_length = 0 + + if session.socket is None: + del self.sessions[sck] + else: + self.log.exception(err) + self._restart_session(session) + continue + + # We received full payload, + # clear session data and parse it. + data = session.data + session.data = "" + session.message_length = 0 + block_id = struct.unpack('!H', data[0:2])[0] + compression = struct.unpack('!B', data[4:5])[0] + payload = data[10:] + + if compression == 0x01: + # Data is compressed, uncompress it. + payload = zlib.decompress(payload) + + # Enqueue payload into a callback queue to be + # invoked. + self._callback_pool.queue_callback(session, + block_id, payload) + except select.error, err: + # Evaluate sessions if we get a bad file descriptor, if + # socket is gone, delete the session. + if err.args[0] == errno.EBADF: + self._clean_dead_sessions() + except Exception, err: + self.log.exception(err) + finally: + for session in self.sessions.values(): + if session is not None: + session.stop() + + def _init_threads(self): + """Initializes the IO and Writer threads""" + if self._io_thread is None: + self._io_thread = Thread(target=self._select) + self._io_thread.start() + + if self._writer_thread is None: + self._writer_thread = Thread(target=self._writer) + self._writer_thread.start() + + def create_session(self, callback, monitor_id): + """ + Creates and Returns a PushSession instance based on the input monitor + and callback. When data is received, callback will be invoked. + If neither monitor or monitor_id are specified, throws an Exception. + + :param callback: Callback function to call when PublishMessage + messages are received. Expects 1 argument which will contain the + payload of the pushed message. Additionally, expects + function to return True if callback was able to process + the message, False or None otherwise. + :param monitor_id: The id of the Monitor, will be queried + to understand parameters of the monitor. + """ + self.log.info("Creating Session for Monitor %s." % monitor_id) + session = SecurePushSession(callback, monitor_id, self, self._ca_certs) \ + if self._secure else PushSession(callback, monitor_id, self) + + session.start() + self.sessions[session.socket.fileno()] = session + + self._init_threads() + return session + + def stop(self): + """Stops all session activity. + + Blocks until io and writer thread dies + """ + if self._io_thread is not None: + self.log.info("Waiting for I/O thread to stop...") + self.closed = True + self._io_thread.join() + + if self._writer_thread is not None: + self.log.info("Waiting for Writer Thread to stop...") + self.closed = True + self._writer_thread.join() + + self.log.info("All worker threads stopped.") From 5692b532cb0495b10618fb888f2d3e6c409b2df9 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 12 Jun 2015 17:34:24 -0500 Subject: [PATCH 048/140] monitor: deserialize json data on reception This removes the burden of performing JSON unpacking from the user. This assumes you are using JSON which is probably a reasonable assumption but not limited by the API at present. Some logging was also modified to be more clear with regards to the return value from callbacks. Relates to: - Monitor Support: https://jira.digi.com/browse/PYTHONDC-54 - TCP Mon Support: https://jira.digi.com/browse/PYTHONDC-55 - Delete Monitors: https://jira.digi.com/browse/PYTHONDC-56 Signed-off-by: Paul Osborne --- devicecloud/examples/monitor_playground.py | 6 +++-- devicecloud/monitor_tcp.py | 30 ++++++++++++---------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/devicecloud/examples/monitor_playground.py b/devicecloud/examples/monitor_playground.py index 61b971a..b059a33 100644 --- a/devicecloud/examples/monitor_playground.py +++ b/devicecloud/examples/monitor_playground.py @@ -4,6 +4,7 @@ # # Copyright (C) 2015, Digi International, Inc.. from getpass import getpass +import pprint import random import time @@ -35,8 +36,9 @@ def get_authenticated_dc(): mon.delete() mon = dc.monitor.create_monitor(topics) - def listener(*args, **kwargs): - print args, kwargs + def listener(data): + pprint.pprint(data) + return True # we got it! mon.add_listener(listener) diff --git a/devicecloud/monitor_tcp.py b/devicecloud/monitor_tcp.py index bf17021..30da8f2 100644 --- a/devicecloud/monitor_tcp.py +++ b/devicecloud/monitor_tcp.py @@ -8,6 +8,7 @@ # https://github.com/digidotcom/idigi-python-monitor-api from Queue import Empty, Queue +import json import logging import socket import struct @@ -311,12 +312,12 @@ def __init__(self, write_queue=None, size=1): """ # Used to queue up PublishMessageReceived events to be sent back to # the iDigi server. - self.__write_queue = write_queue + self._write_queue = write_queue # Used to queue up sessions and data to callback with. - self.__queue = Queue(size) + self._queue = Queue(size) # Number of workers to create. self.size = size - self.log = logging.getLogger('callback_worker_pool') + self.log = logging.getLogger('{}.callback_worker_pool'.format(__name__)) for _ in range(size): worker = Thread(target=self._consume_queue) @@ -330,21 +331,25 @@ def _consume_queue(self): if callback returned True. """ while True: - session, block_id, data = self.__queue.get() + session, block_id, raw_data = self._queue.get() + data = json.loads(raw_data) # decode as JSON try: - if session.callback(data): + result = session.callback(data) + if result is None: + self.log.warn("Callback %r returned None, expected boolean. Messages " + "are not marked as received unless True is returned", session.callback) + elif result: # Send a Successful PublishMessageReceived with the # block id sent in request - if self.__write_queue is not None: + if self._write_queue is not None: response_message = struct.pack('!HHH', PUBLISH_MESSAGE_RECEIVED, block_id, 200) - self.__write_queue.put((session.socket, - response_message)) + self._write_queue.put((session.socket, response_message)) except Exception, exception: self.log.exception(exception) - self.__queue.task_done() + self._queue.task_done() def queue_callback(self, session, block_id, data): """ @@ -355,7 +360,7 @@ def queue_callback(self, session, block_id, data): :param block_id: the block_id of the message received. :param data: the data payload of the message received. """ - self.__queue.put((session, block_id, data)) + self._queue.put((session, block_id, data)) class TCPClientManager(object): @@ -518,9 +523,8 @@ def _select(self): payload = zlib.decompress(payload) # Enqueue payload into a callback queue to be - # invoked. - self._callback_pool.queue_callback(session, - block_id, payload) + # invoked + self._callback_pool.queue_callback(session, block_id, payload) except select.error, err: # Evaluate sessions if we get a bad file descriptor, if # socket is gone, delete the session. From 079de0ed6d0a6c7a82675d308213ecab22d4ac16 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Sat, 13 Jun 2015 18:14:57 -0500 Subject: [PATCH 049/140] examples: extract get_authenticated_dc to shared location This is used in all the examples and was starting to get more complex. It is now shared. This makes each example a little bit less standalone but probably still clear enough. get_authenticated_dc now optionally uses environment variables for username, password, and hostname. Signed-off-by: Paul Osborne --- devicecloud/examples/cookbook_streams.py | 19 +++---------- devicecloud/examples/devicecore_playground.py | 19 ++----------- devicecloud/examples/example_helpers.py | 28 +++++++++++++++++++ devicecloud/examples/filedata_playground.py | 21 ++------------ devicecloud/examples/monitor_playground.py | 20 ++----------- devicecloud/examples/streams_playground.py | 16 ++--------- 6 files changed, 43 insertions(+), 80 deletions(-) create mode 100644 devicecloud/examples/example_helpers.py diff --git a/devicecloud/examples/cookbook_streams.py b/devicecloud/examples/cookbook_streams.py index 2708d92..5436fa7 100644 --- a/devicecloud/examples/cookbook_streams.py +++ b/devicecloud/examples/cookbook_streams.py @@ -1,23 +1,12 @@ -from devicecloud.streams import STREAM_TYPE_STRING, DataPoint, STREAM_TYPE_INTEGER -from devicecloud import DeviceCloud -from getpass import getpass import datetime import pprint import random import time import json +from devicecloud.examples.example_helpers import get_authenticated_dc -def get_authenticated_dc(): - while True: - user = raw_input("username: ") - password = getpass("password: ") - dc = DeviceCloud(user, password) - if dc.has_valid_credentials(): - print ("Credentials accepted!") - return dc - else: - print ("Invalid username or password provided, try again") +from devicecloud.streams import STREAM_TYPE_STRING, DataPoint, STREAM_TYPE_INTEGER def get_or_create_classroom(datatype): @@ -76,7 +65,7 @@ def example_1(): classroom.bulk_write_datapoints(datapoints) most_recent_dp = classroom.get_current_value() - print json.loads(most_recent_dp.get_data())['name'] + print(json.loads(most_recent_dp.get_data())['name']) def example_2(): @@ -91,4 +80,4 @@ def example_2(): example_2() -print 'done.' \ No newline at end of file +print('done.') \ No newline at end of file diff --git a/devicecloud/examples/devicecore_playground.py b/devicecloud/examples/devicecore_playground.py index 7b14c60..f94bb40 100644 --- a/devicecloud/examples/devicecore_playground.py +++ b/devicecloud/examples/devicecore_playground.py @@ -3,27 +3,13 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # # Copyright (c) 2015 Digi International, Inc. -from getpass import getpass - -from devicecloud import DeviceCloud from devicecloud.devicecore import dev_mac, group_path - - -def get_authenticated_dc(): - while True: - user = raw_input("username: ") - password = getpass("password: ") - dc = DeviceCloud(user, password, - base_url="https://login.etherios.com") - if dc.has_valid_credentials(): - print ("Credentials accepted!") - return dc - else: - print ("Invalid username or password provided, try again") +from devicecloud.examples.example_helpers import get_authenticated_dc def show_group_tree(dc): stats = {} # group -> devices count including children + def count_nodes(group): count_for_this_node = \ len(list(dc.devicecore.get_devices(group_path == group.get_path()))) @@ -33,6 +19,7 @@ def count_nodes(group): total = count_for_this_node + subnode_count stats[group] = total return total + count_nodes(dc.devicecore.get_group_tree_root()) print(stats) dc.devicecore.get_group_tree_root().print_subtree() diff --git a/devicecloud/examples/example_helpers.py b/devicecloud/examples/example_helpers.py new file mode 100644 index 0000000..7caf53a --- /dev/null +++ b/devicecloud/examples/example_helpers.py @@ -0,0 +1,28 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015 Digi International, Inc. +from getpass import getpass +import os +from devicecloud import DeviceCloud + + +def get_authenticated_dc(): + while True: + base_url = os.environ.get('DC_BASE_URL', 'https://login.etherios.com') + + username = os.environ.get('DC_USERNAME', None) + if not username: + username = input("username: ") + + password = os.environ.get('DC_PASSWORD', None) + if not password: + password = getpass("password: ") + + dc = DeviceCloud(username, password, base_url=base_url) + if dc.has_valid_credentials(): + print("Credentials accepted!") + return dc + else: + print("Invalid username or password provided, try again") diff --git a/devicecloud/examples/filedata_playground.py b/devicecloud/examples/filedata_playground.py index 633dbf4..5c4bd42 100644 --- a/devicecloud/examples/filedata_playground.py +++ b/devicecloud/examples/filedata_playground.py @@ -3,24 +3,10 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # # Copyright (c) 2015 Digi International, Inc. -from getpass import getpass -from devicecloud import DeviceCloud -from devicecloud.filedata import fd_name, fd_size, fd_type, fd_path +from devicecloud.examples.example_helpers import get_authenticated_dc +from devicecloud.filedata import fd_path import six -from six.moves import input - -def get_authenticated_dc(): - while True: - user = input("username: ") - password = getpass("password: ") - dc = DeviceCloud(user, password, base_url="https://login-etherios-com-2v5p9uat81qu.runscope.net") - if dc.has_valid_credentials(): - print("Credentials accepted!") - return dc - else: - print("Invalid username or password provided, try again") - if __name__ == '__main__': dc = get_authenticated_dc() @@ -32,6 +18,5 @@ def get_authenticated_dc(): for fd_file in files: print(fd_file) - - for fd in dc.filedata.get_filedata(fd_path=="~/"): + for fd in dc.filedata.get_filedata(fd_path == "~/"): print (fd) diff --git a/devicecloud/examples/monitor_playground.py b/devicecloud/examples/monitor_playground.py index b059a33..5bc8069 100644 --- a/devicecloud/examples/monitor_playground.py +++ b/devicecloud/examples/monitor_playground.py @@ -3,28 +3,13 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # # Copyright (C) 2015, Digi International, Inc.. -from getpass import getpass import pprint import random import time -from devicecloud import DeviceCloud -from devicecloud.filedata import fd_name, fd_size, fd_type, fd_path -from devicecloud.streams import DataPoint -import six -from six.moves import input - -def get_authenticated_dc(): - while True: - user = input("username: ") - password = getpass("password: ") - dc = DeviceCloud(user, password, base_url="https://login.etherios.com") - if dc.has_valid_credentials(): - print("Credentials accepted!") - return dc - else: - print("Invalid username or password provided, try again") +from devicecloud.examples.example_helpers import get_authenticated_dc +from devicecloud.streams import DataPoint if __name__ == '__main__': dc = get_authenticated_dc() @@ -35,6 +20,7 @@ def get_authenticated_dc(): if mon is not None: mon.delete() mon = dc.monitor.create_monitor(topics) + pprint.pprint(mon.get_metadata()) def listener(data): pprint.pprint(data) diff --git a/devicecloud/examples/streams_playground.py b/devicecloud/examples/streams_playground.py index 117fb51..c56e2a8 100644 --- a/devicecloud/examples/streams_playground.py +++ b/devicecloud/examples/streams_playground.py @@ -3,25 +3,13 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # # Copyright (c) 2015 Digi International, Inc. -from getpass import getpass from math import pi import pprint import time -from devicecloud import DeviceCloud -from devicecloud.streams import DataPoint, NoSuchStreamException, STREAM_TYPE_INTEGER - +from devicecloud.examples.example_helpers import get_authenticated_dc -def get_authenticated_dc(): - while True: - user = raw_input("username: ") - password = getpass("password: ") - dc = DeviceCloud(user, password, base_url="https://login-etherios-com-2v5p9uat81qu.runscope.net") - if dc.has_valid_credentials(): - print ("Credentials accepted!") - return dc - else: - print ("Invalid username or password provided, try again") +from devicecloud.streams import DataPoint, NoSuchStreamException, STREAM_TYPE_INTEGER def create_stream_and_delete(dc): From df1ddd936b1d247b35526e183009203b9b39ec4c Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Sat, 13 Jun 2015 21:04:29 -0500 Subject: [PATCH 050/140] monitor: test: unit/integration tests for monitor implementation This commit adds unit and integration tests for monitors. The code as-is for the TCP push monitor is hard to unit test as-is, so for now an integration test has been put in place to provide some level of automated coverage for this code. It does not test edge cases well but it is something. This commit also includes a fair number of bug fixes for python3 with the new TCP push monitor implementation which previously was written to support python 2 only. Relates to: - Monitor Support: https://jira.digi.com/browse/PYTHONDC-54 - TCP Mon Support: https://jira.digi.com/browse/PYTHONDC-55 - Delete Monitors: https://jira.digi.com/browse/PYTHONDC-56 Signed-off-by: Paul Osborne --- devicecloud/__init__.py | 1 - devicecloud/examples/cookbook_streams.py | 2 +- devicecloud/monitor.py | 29 ++- devicecloud/monitor_tcp.py | 78 ++++---- .../test/integration/inttest_monitor_tcp.py | 61 ++++++ .../test/integration/inttest_streams.py | 13 +- devicecloud/test/unit/test_monitor.py | 186 ++++++++++++++++++ devicecloud/test/unit/test_monitor_tcp.py | 29 +++ 8 files changed, 351 insertions(+), 48 deletions(-) create mode 100644 devicecloud/test/integration/inttest_monitor_tcp.py create mode 100644 devicecloud/test/unit/test_monitor.py create mode 100644 devicecloud/test/unit/test_monitor_tcp.py diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index 7972e59..be5d9b6 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -6,7 +6,6 @@ import logging import time import json -import urlparse from devicecloud.util import validate_type from requests.auth import HTTPBasicAuth diff --git a/devicecloud/examples/cookbook_streams.py b/devicecloud/examples/cookbook_streams.py index 5436fa7..303813a 100644 --- a/devicecloud/examples/cookbook_streams.py +++ b/devicecloud/examples/cookbook_streams.py @@ -80,4 +80,4 @@ def example_2(): example_2() -print('done.') \ No newline at end of file +print('done.') diff --git a/devicecloud/monitor.py b/devicecloud/monitor.py index 928c21d..d735883 100644 --- a/devicecloud/monitor.py +++ b/devicecloud/monitor.py @@ -106,15 +106,42 @@ class DeviceCloudMonitor(object): :type _conn: devicecloud.DeviceCloudConnection """ + # TODO: consider adding getters/setters for each metadata + @classmethod def from_json(cls, conn, tcp_client_manager, monitor_data): - return cls(conn, tcp_client_manager, monitor_data['monId']) + monitor_id = int(monitor_data['monId']) + return cls(conn, tcp_client_manager, monitor_id) def __init__(self, conn, tcp_client_manager, monitor_id): self._conn = conn self._tcp_client_manager = tcp_client_manager self._id = monitor_id + def get_id(self): + """Get the ID of this monitor as an integer""" + return self._id + + def get_metadata(self): + """Get additional information about this monitor + + This method returns a dictionary where the keys contain information about the + monitor. The returned data will look something like this:: + + { + 'cstId': '7603', + 'monBatchDuration': '10', + 'monBatchSize': '1', + 'monCompression': 'zlib', + 'monFormatType': 'json', + 'monId': '178023', + 'monStatus': 'INACTIVE', + 'monTopic': 'DeviceCore,FileDataCore,FileData,DataPoint', + 'monTransportType': 'tcp' + } + """ + return self._conn.get_json("/ws/Monitor/{id}".format(id=self._id))["items"][0] + def delete(self): """Delete this monitor form the device cloud""" self._conn.delete("/ws/Monitor/{id}".format(id=self._id)) diff --git a/devicecloud/monitor_tcp.py b/devicecloud/monitor_tcp.py index 30da8f2..94954a6 100644 --- a/devicecloud/monitor_tcp.py +++ b/devicecloud/monitor_tcp.py @@ -7,7 +7,6 @@ # This code is originally from another Digi Open Source Library: # https://github.com/digidotcom/idigi-python-monitor-api -from Queue import Empty, Queue import json import logging import socket @@ -19,6 +18,8 @@ import ssl import pkg_resources +from six.moves.queue import Queue, Empty +import six DEFAULT_CRT_NAME = "devicecloud.crt" @@ -72,7 +73,7 @@ def _read_msg_header(session): response_type = struct.unpack('!H', session.data[0:2])[0] # Clear out session data as header is consumed. - session.data = "" + session.data = six.b("") return response_type @@ -103,10 +104,7 @@ def _read_msg(session): class PushException(Exception): - """ - Indicates an issue interacting with iDigi Push Functionality. - """ - pass + """Indicates an issue interacting with Push Functionality.""" class PushSession(object): @@ -117,9 +115,7 @@ class PushSession(object): """ def __init__(self, callback, monitor_id, client): - """ - Creates a PushSession for use with interacting with iDigi's - Push Functionality. + """Creates a PushSession for use with the device cloud :param callback: The callback function to invoke when data received. Must have 1 required parameter that will contain the payload. @@ -130,10 +126,10 @@ def __init__(self, callback, monitor_id, client): self.monitor_id = monitor_id self.client = client self.socket = None - self.log = logging.getLogger("push_session[%s]" % monitor_id) + self.log = logging.getLogger("%s.push_session.%s" % (__name__, monitor_id)) # Received protocol data holders. - self.data = "" + self.data = six.b("") self.message_length = 0 def send_connection_request(self): @@ -152,11 +148,11 @@ def send_connection_request(self): # Username Length. payload += struct.pack('!H', len(self.client.username)) # Username. - payload += self.client.username + payload += six.b(self.client.username) # Password Length. payload += struct.pack('!H', len(self.client.password)) # Password. - payload += self.client.password + payload += six.b(self.client.password) # Monitor ID. payload += struct.pack('!L', int(self.monitor_id)) @@ -195,9 +191,9 @@ def send_connection_request(self): self.log.info("Got ConnectionResponse for Monitor %s. Status %s." % (self.monitor_id, status_code)) if status_code != STATUS_OK: - raise PushException("Connection Response Status Code (%d) is \ -not STATUS_OK (%d)." % (status_code, STATUS_OK)) - except Exception, exception: + raise PushException("Connection Response Status Code (%d) is " + "not STATUS_OK (%d)." % (status_code, STATUS_OK)) + except Exception as exception: # TODO(posborne): This is bad! It isn't necessarily a socket exception! # Likely a socket exception, close it and raise an exception. self.socket.close() @@ -205,12 +201,8 @@ def send_connection_request(self): raise exception def start(self): - """ - Creates a TCP connection to the iDigi Server and sends a - ConnectionRequest message. - """ - self.log.info("Starting Insecure Session for Monitor %s." - % self.monitor_id) + """Creates a TCP connection to the device cloud and sends a ConnectionRequest message""" + self.log.info("Starting Insecure Session for Monitor %s" % self.monitor_id) if self.socket is not None: raise Exception("Socket already established for %s." % self) @@ -218,16 +210,17 @@ def start(self): self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.connect((self.client.hostname, PUSH_OPEN_PORT)) self.socket.setblocking(0) - except Exception, exception: + except socket.error as exception: self.socket.close() self.socket = None - raise exception + raise self.send_connection_request() def stop(self): - """ - Closes the socket associated with this session and puts Session + """Stop/Close this session + + Close the socket associated with this session and puts Session into a state such that it can be re-established later. """ if self.socket is not None: @@ -287,7 +280,7 @@ def start(self): self.socket.connect((self.client.hostname, PUSH_SECURE_PORT)) self.socket.setblocking(0) - except Exception, exception: + except Exception as exception: self.socket.close() self.socket = None raise exception @@ -332,7 +325,7 @@ def _consume_queue(self): """ while True: session, block_id, raw_data = self._queue.get() - data = json.loads(raw_data) # decode as JSON + data = json.loads(raw_data.decode('utf-8')) # decode as JSON try: result = session.callback(data) if result is None: @@ -346,7 +339,7 @@ def _consume_queue(self): PUBLISH_MESSAGE_RECEIVED, block_id, 200) self._write_queue.put((session.socket, response_message)) - except Exception, exception: + except Exception as exception: self.log.exception(exception) self._queue.task_done() @@ -408,10 +401,9 @@ def password(self): return self._conn.password def _restart_session(self, session): - """ - Restarts and re-establishes session. + """Restarts and re-establishes session - :param session: The session to restart. + :param session: The session to restart """ # remove old session key, if socket is None, that means the # session was closed by user and there is no need to restart. @@ -435,7 +427,7 @@ def _writer(self): sock.send(data) except Empty: pass # nothing to write after timeout - except socket.error, err: + except socket.error as err: if err.errno == errno.EBADF: self._clean_dead_sessions() @@ -445,7 +437,7 @@ def _clean_dead_sessions(self): were removed (indicates a stopped session). In these cases, remove the session. """ - for sck in self.sessions.keys(): + for sck in list(self.sessions.keys()): session = self.sessions[sck] if session.socket is None: del self.sessions[sck] @@ -460,8 +452,7 @@ def _select(self): try: while not self.closed: try: - inputready = \ - select.select(self.sessions.keys(), [], [], 0.1)[0] + inputready = select.select(self.sessions.keys(), [], [], 0.1)[0] for sock in inputready: session = self.sessions[sock] sck = session.socket @@ -486,8 +477,7 @@ def _select(self): # More Data to be read. Continue. continue elif response_type != PUBLISH_MESSAGE: - self.log.warn("Response Type (%x) does " \ - "not match PublishMessage (%x)" \ + self.log.warn("Response Type (%x) does not match PublishMessage (%x)" % (response_type, PUBLISH_MESSAGE)) continue @@ -495,11 +485,11 @@ def _select(self): if not _read_msg(session): # Data not completely read, continue. continue - except PushException, err: + except PushException as err: # If Socket is None, it was closed, # otherwise it was closed when it shouldn't # have been restart it. - session.data = "" + session.data = six.b("") session.message_length = 0 if session.socket is None: @@ -511,8 +501,8 @@ def _select(self): # We received full payload, # clear session data and parse it. - data = session.data - session.data = "" + data = session.data + session.data = six.b("") session.message_length = 0 block_id = struct.unpack('!H', data[0:2])[0] compression = struct.unpack('!B', data[4:5])[0] @@ -525,12 +515,12 @@ def _select(self): # Enqueue payload into a callback queue to be # invoked self._callback_pool.queue_callback(session, block_id, payload) - except select.error, err: + except select.error as err: # Evaluate sessions if we get a bad file descriptor, if # socket is gone, delete the session. if err.args[0] == errno.EBADF: self._clean_dead_sessions() - except Exception, err: + except Exception as err: self.log.exception(err) finally: for session in self.sessions.values(): diff --git a/devicecloud/test/integration/inttest_monitor_tcp.py b/devicecloud/test/integration/inttest_monitor_tcp.py new file mode 100644 index 0000000..8775e57 --- /dev/null +++ b/devicecloud/test/integration/inttest_monitor_tcp.py @@ -0,0 +1,61 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015 Digi International, Inc. +import pprint +import time +from devicecloud.streams import DataPoint + +from devicecloud.test.integration.inttest_utilities import DeviceCloudIntegrationTestCase +import six + + +class StreamsIntegrationTestCase(DeviceCloudIntegrationTestCase): + + def test_event_reception(self): + rx = [] + + def receive_notification(notification): + rx.append(notification) + return True + + topics = ['DataPoint', 'FileData'] + monitor = self._dc.monitor.get_monitor(topics) + if monitor: + monitor.delete() + monitor = self._dc.monitor.create_monitor(topics) + monitor.add_listener(receive_notification) + + self._dc.filedata.write_file("/~/inttest/monitor_tcp/", "test_file.txt", six.b("Hello, world!"), "text/plain") + self._dc.streams.get_stream("inttest/monitor_tcp").write(DataPoint(10)) + + # Wait for the evenets to come in from the cloud + time.sleep(3) + self._dc.monitor.stop_listeners() + + try: + fd_push_seen = False + dp_push_seen = False + for rec in rx: + msg = rec['Document']['Msg'] + fd = msg.get('FileData', None) + if fd: + if (fd['id']['fdName'] == 'test_file.txt' and + fd['id']['fdPath'] == '/db/7603_Etherios/inttest/monitor_tcp/'): + fd_push_seen = True + dp = msg.get('DataPoint') + if dp: + if (dp['streamId'] == 'inttest/monitor_tcp'): + dp_push_seen = True + + self.assertTrue(fd_push_seen) + self.assertTrue(dp_push_seen) + except: + # add some additional debugging information + pprint.pprint(rx) + raise + +if __name__ == '__main__': + import unittest + unittest.main() \ No newline at end of file diff --git a/devicecloud/test/integration/inttest_streams.py b/devicecloud/test/integration/inttest_streams.py index 4c47b50..4e9694f 100644 --- a/devicecloud/test/integration/inttest_streams.py +++ b/devicecloud/test/integration/inttest_streams.py @@ -59,7 +59,7 @@ def test_bulk_write_datapoints_multiple_streams(self): # This test verifies that we can write in bulk a bunch of datapoints to several # datastreams and read them back. # - SID_FMT="pythondc-inttest/test_bulk_write_datapoints_multiple_streams-{}" + SID_FMT = "pythondc-inttest/test_bulk_write_datapoints_multiple_streams-{}" datapoints = [] dt = datetime.datetime.now() for i in range(300): @@ -70,6 +70,13 @@ def test_bulk_write_datapoints_multiple_streams(self): timestamp=dt - datetime.timedelta(seconds=300 - i), data=i, )) + + # remove any existing data before starting out + for i in range(3): + s = self._dc.streams.get_stream_if_exists(SID_FMT.format(i % 3)) + if s: + s.delete() + self._dc.streams.bulk_write_datapoints(datapoints) for i in range(3): @@ -93,6 +100,10 @@ def test_bulk_write_datapoints_single_stream(self): data=i, )) + stream = self._dc.streams.get_stream_if_exists("pythondc-inttest/test_bulk_write_datapoints_single_stream") + if stream: + stream.delete() + stream = self._dc.streams.get_stream("pythondc-inttest/test_bulk_write_datapoints_single_stream") stream.bulk_write_datapoints(datapoints) stream_contents_asc = list(stream.read(newest_first=False)) diff --git a/devicecloud/test/unit/test_monitor.py b/devicecloud/test/unit/test_monitor.py new file mode 100644 index 0000000..289204b --- /dev/null +++ b/devicecloud/test/unit/test_monitor.py @@ -0,0 +1,186 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015 Digi International, Inc. +from devicecloud.test.unit.test_utilities import HttpTestBase +import six + +CREATE_MONITOR_GOOD_REQUEST = """\ + + topA,topB + 10 + json + tcp + gzip + +""" + +CREATE_MONITOR_GOOD_RESPONSE = """\ + + + Monitor/178008 + +""" + +GET_MONITOR_SINGLE_FOUND = """\ +{ + "resultTotalRows": "1", + "requestedStartRow": "0", + "resultSize": "1", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { + "monId": "178007", + "cstId": "7603", + "monTopic": "DeviceCore,FileDataCore,FileData,DataPoint", + "monTransportType": "tcp", + "monFormatType": "json", + "monBatchSize": "1", + "monCompression": "zlib", + "monStatus": "INACTIVE", + "monBatchDuration": "10" + } + ] +} +""" + +GET_MONITOR_METADTATA = """\ +{ + "resultTotalRows": "1", + "requestedStartRow": "0", + "resultSize": "1", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { + "monId": "178007", + "cstId": "7603", + "monTopic": "DeviceCore,FileDataCore,FileData,DataPoint", + "monTransportType": "tcp", + "monFormatType": "json", + "monBatchSize": "1", + "monCompression": "zlib", + "monStatus": "INACTIVE", + "monBatchDuration": "10" + } + ] +} +""" + +GET_MONITOR_MULTIPLE_FOUND = """\ +{ + "resultTotalRows": "2", + "requestedStartRow": "0", + "resultSize": "2", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { + "monId": "178007", + "cstId": "7603", + "monTopic": "DeviceCore,FileDataCore,FileData,DataPoint", + "monTransportType": "tcp", + "monFormatType": "json", + "monBatchSize": "1", + "monCompression": "zlib", + "monStatus": "INACTIVE", + "monBatchDuration": "10" + }, + { + "monId": "198765", + "cstId": "7603", + "monTopic": "DeviceCore,FileDataCore,FileData,DataPoint", + "monTransportType": "tcp", + "monFormatType": "json", + "monBatchSize": "1", + "monCompression": "", + "monStatus": "INACTIVE", + "monBatchDuration": "10" + } + ] +} +""" + +GET_MONITOR_NONE_FOUND = """\ +{ + "resultTotalRows": "0", + "requestedStartRow": "0", + "resultSize": "0", + "requestedSize": "1000", + "remainingSize": "0", + "items": [] +} +""" + + + +class TestMonitorAPI(HttpTestBase): + + def test_create_monitor(self): + self.prepare_response("POST", "/ws/Monitor", data=CREATE_MONITOR_GOOD_RESPONSE) + mon = self.dc.monitor.create_monitor(['topA', 'topB'], batch_size=10, batch_duration=0, + transport_type='tcp', compression='gzip', format_type='json') + self.assertEqual(self._get_last_request().body, six.b(CREATE_MONITOR_GOOD_REQUEST)) + self.assertEqual(mon.get_id(), 178008) + + def test_get_monitor_present(self): + self.prepare_response("GET", "/ws/Monitor", data=GET_MONITOR_SINGLE_FOUND) + mon = self.dc.monitor.get_monitor(['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint']) + self.assertEqual(mon.get_id(), 178007) + self.assertEqual(self._get_last_request_params(), { + 'condition': "monTopic='DeviceCore,FileDataCore,FileData,DataPoint'", + 'start': '0', + 'size': '1000' + }) + + def test_get_monitor_multiple(self): + # Should just pick the first result (currently), so results are the same as ever + self.prepare_response("GET", "/ws/Monitor", data=GET_MONITOR_MULTIPLE_FOUND) + mon = self.dc.monitor.get_monitor(['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint']) + self.assertEqual(mon.get_id(), 178007) + self.assertEqual(self._get_last_request_params(), { + 'condition': "monTopic='DeviceCore,FileDataCore,FileData,DataPoint'", + 'start': '0', + 'size': '1000' + }) + + def test_get_monitor_does_not_exist(self): + # Should just pick the first result (currently), so results are the same as ever + self.prepare_response("GET", "/ws/Monitor", data=GET_MONITOR_NONE_FOUND) + mon = self.dc.monitor.get_monitor(['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint']) + self.assertEqual(mon, None) + + +class TestDeviceCloudMonitor(HttpTestBase): + + def setUp(self): + HttpTestBase.setUp(self) + self.prepare_response("GET", "/ws/Monitor", data=GET_MONITOR_SINGLE_FOUND) + mon = self.dc.monitor.get_monitor(['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint']) + self.mon = mon + + def test_get_metadata(self): + self.prepare_response("GET", "/ws/Monitor/178007", data=GET_MONITOR_METADTATA) + self.assertEqual(self.mon.get_metadata(), { + "monId": "178007", + "cstId": "7603", + "monTopic": "DeviceCore,FileDataCore,FileData,DataPoint", + "monTransportType": "tcp", + "monFormatType": "json", + "monBatchSize": "1", + "monCompression": "zlib", + "monStatus": "INACTIVE", + "monBatchDuration": "10" + }) + + def test_delete(self): + self.prepare_response("DELETE", "/ws/Monitor/178007") + self.mon.delete() + req = self._get_last_request() + self.assertEqual(req.method, "DELETE") + self.assertEqual(req.path, "/ws/Monitor/178007") + + def test_get_id(self): + self.assertEqual(self.mon.get_id(), 178007) diff --git a/devicecloud/test/unit/test_monitor_tcp.py b/devicecloud/test/unit/test_monitor_tcp.py new file mode 100644 index 0000000..2c85f78 --- /dev/null +++ b/devicecloud/test/unit/test_monitor_tcp.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015 Digi International, Inc. + +from devicecloud.monitor_tcp import TCPClientManager +from devicecloud.test.unit.test_utilities import HttpTestBase + + +class TestTCPClientManager(HttpTestBase): + + # NOTE: currently only integration tests exist to test several parts of + # the basic device cloud push client functionality for historical reasons. + # In the future, it would be nice to extended the unit test coverage + # for this code. + + def setUp(self): + HttpTestBase.setUp(self) + self.client_manager = TCPClientManager(self.dc.get_connection()) + + def test_hostname(self): + self.assertEqual(self.client_manager.hostname, "login.etherios.com") + + def test_username(self): + self.assertEqual(self.client_manager.username, "user") + + def test_password(self): + self.assertEqual(self.client_manager.password, "pass") From 456a77f98453bb7830bc50edf9dfc4eaceaaf80c Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Mon, 15 Jun 2015 11:24:13 -0500 Subject: [PATCH 051/140] monitor: docs: add Sphinx docs for the new monitor API Signed-off-by: Paul Osborne --- devicecloud/__init__.py | 1 + devicecloud/monitor.py | 52 ++++++++++++++++--- .../test/integration/inttest_monitor_tcp.py | 2 +- docs/index.rst | 1 + docs/monitor.rst | 18 +++++++ 5 files changed, 67 insertions(+), 7 deletions(-) create mode 100644 docs/monitor.rst diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index be5d9b6..e507cf5 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -418,6 +418,7 @@ def sci(self): @property def monitor(self): + """Property providing access to the :class:`.MonitorAPI`""" if self._monitor_api is None: self._monitor_api = self.get_monitor_api() return self._monitor_api diff --git a/devicecloud/monitor.py b/devicecloud/monitor.py index d735883..8e3adc7 100644 --- a/devicecloud/monitor.py +++ b/devicecloud/monitor.py @@ -17,14 +17,14 @@ class MonitorAPI(APIBase): - """Provide access to the device cloud Monitor API for receiving push notifiactions + """Provide access to the device cloud Monitor API for receiving push notifications - The Monitor API in the device cloud allows for the creation and destructions of + The Monitor API in the device cloud allows for the creation and destruction of multiple "monitors." Each monitor is registered against one or more "topics" which describe the data in which it is interested. There are, in turn, two main ways to receive data matching the topics for a - given monitor + given monitor: 1. Stream: The device cloud supports a protocol over TCP (optionally with SSL) over which the batches of events will be sent when they are received. @@ -33,6 +33,47 @@ class MonitorAPI(APIBase): Currently, this library supports setting up both types of monitors, but there is no special support provided for parsing HTTP postback requests. + + More information on the format for topic strings can be found in the `device + cloud documentation for monitors `_. + + Here's a quick example showing a typical pattern used for creating a push monitor + and associated listener that triggers a callback. Deletion of existing monitors + matching the same topics is not necessary but sometimes done in order to ensure + that changes to the monitor configuration in code always make it to the monitor + configuration in the device cloud:: + + def monitor_callback(json_data): + print(json_data) + return True # message received + + # Listen for DataPoint updates + topics = ['DataPoint[U]'] + monitor = dc.monitor.get_monitor(topics) + if monitor: + monitor.delete() + monitor = dc.monitor.create_monitor(topics) + monitor.add_listener(monitor_callback) + + # later... + dc.monitor.stop_listeners() + + When updates to any DataPoint in the device cloud occurs, the callback will be called + with a data structure like this one:: + + {'Document': {'Msg': {'DataPoint': {'cstId': 7603, + 'data': 0.411700824929, + 'description': '', + 'id': '684572e0-12c4-11e5-8507-fa163ed4cf14', + 'quality': 0, + 'serverTimestamp': 1434307047694, + 'streamId': 'test', + 'streamUnits': '', + 'timestamp': 1434307047694}, + 'group': '*', + 'operation': 'INSERTION', + 'timestamp': '2015-06-14T18:37:27.815Z', + 'topic': '7603/DataPoint/test'}}} """ def __init__(self, conn): @@ -51,8 +92,7 @@ def create_monitor(self, topics, batch_size=1, batch_duration=0, transport_type= does not exceed batch_size. :param transport_type: Either 'tcp' or 'http' :param compression: Compression value (i.e. 'gzip'). - :param format_type: What format server should send data in (i.e. - 'xml' or 'json'). + :param format_type: What format server should send data in (i.e. 'xml' or 'json'). Returns a string of the created Monitor Id (e.g.. 9001) """ @@ -83,7 +123,7 @@ def create_monitor(self, topics, batch_size=1, batch_duration=0, transport_type= def get_monitor(self, topics): """Attempts to find a Monitor in device cloud that matches the provided topics - :param topics: a string list of topics (e.g. ``['DeviceCore[U]', 'FileDataCore'])`` + :param topics: a string list of topics (e.g. ``['DeviceCore[U]', 'FileDataCore'])``) Returns a :class:`DeviceCloudMonitor` if found, otherwise None. """ diff --git a/devicecloud/test/integration/inttest_monitor_tcp.py b/devicecloud/test/integration/inttest_monitor_tcp.py index 8775e57..5bcfb6b 100644 --- a/devicecloud/test/integration/inttest_monitor_tcp.py +++ b/devicecloud/test/integration/inttest_monitor_tcp.py @@ -46,7 +46,7 @@ def receive_notification(notification): fd_push_seen = True dp = msg.get('DataPoint') if dp: - if (dp['streamId'] == 'inttest/monitor_tcp'): + if dp['streamId'] == 'inttest/monitor_tcp': dp_push_seen = True self.assertTrue(fd_push_seen) diff --git a/docs/index.rst b/docs/index.rst index 445eaac..d65b302 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,6 +12,7 @@ Documention Map streams filedata sci + monitor ws cookbook diff --git a/docs/monitor.rst b/docs/monitor.rst new file mode 100644 index 0000000..df57f47 --- /dev/null +++ b/docs/monitor.rst @@ -0,0 +1,18 @@ +Monitor API +=========== + +Monitor Overview +---------------- + +Provide access to the device cloud monitor API which can be used to +subscribe to topics to receive notifications when data is received +on the device cloud. + +SCI API Documentation +--------------------- + +.. automodule:: devicecloud.monitor + :members: + +.. automodule:: devicecloud.monitor_tcp + :members: From 9d28cbf408e58f96ae347d0aa380e2e74f36ea16 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Mon, 15 Jun 2015 13:31:33 -0500 Subject: [PATCH 052/140] monitor: rename create_monitor -> create_tcp_monitor There are several options for monitors that only apply to either TCP monitors or HTTP monitors. As such, having separate create functions for each should make it easier for clients to not end up with a bad combination of options. Signed-off-by: Paul Osborne --- devicecloud/examples/monitor_playground.py | 2 +- devicecloud/monitor.py | 12 +++++------- devicecloud/test/integration/inttest_monitor_tcp.py | 2 +- devicecloud/test/unit/test_monitor.py | 6 +++--- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/devicecloud/examples/monitor_playground.py b/devicecloud/examples/monitor_playground.py index 5bc8069..7e85820 100644 --- a/devicecloud/examples/monitor_playground.py +++ b/devicecloud/examples/monitor_playground.py @@ -19,7 +19,7 @@ mon = dc.monitor.get_monitor(topics) if mon is not None: mon.delete() - mon = dc.monitor.create_monitor(topics) + mon = dc.monitor.create_tcp_monitor(topics) pprint.pprint(mon.get_metadata()) def listener(data): diff --git a/devicecloud/monitor.py b/devicecloud/monitor.py index 8e3adc7..4359f44 100644 --- a/devicecloud/monitor.py +++ b/devicecloud/monitor.py @@ -52,7 +52,7 @@ def monitor_callback(json_data): monitor = dc.monitor.get_monitor(topics) if monitor: monitor.delete() - monitor = dc.monitor.create_monitor(topics) + monitor = dc.monitor.create_tcp_monitor(topics) monitor.add_listener(monitor_callback) # later... @@ -81,16 +81,15 @@ def __init__(self, conn): # TODO: determine best way to expose additional options self._tcp_client_manager = TCPClientManager(self._conn, secure=True) - def create_monitor(self, topics, batch_size=1, batch_duration=0, transport_type='tcp', - compression='gzip', format_type='json'): - """Creates a Monitor instance in the device cloud for a given list of topics + def create_tcp_monitor(self, topics, batch_size=1, batch_duration=0, + compression='gzip', format_type='json'): + """Creates a TCP Monitor instance in the device cloud for a given list of topics :param topics: a string list of topics (e.g. ['DeviceCore[U]', 'FileDataCore']). :param batch_size: How many Msgs received before sending data. :param batch_duration: How long to wait before sending batch if it does not exceed batch_size. - :param transport_type: Either 'tcp' or 'http' :param compression: Compression value (i.e. 'gzip'). :param format_type: What format server should send data in (i.e. 'xml' or 'json'). @@ -102,7 +101,7 @@ def create_monitor(self, topics, batch_size=1, batch_duration=0, transport_type= {topics} {batch_size} {format_type} - {transport_type} + tcp {compression} """.format( @@ -110,7 +109,6 @@ def create_monitor(self, topics, batch_size=1, batch_duration=0, transport_type= batch_size=batch_size, batch_duration=batch_duration, format_type=format_type, - transport_type=transport_type, compression=compression, ) monitor_xml = textwrap.dedent(monitor_xml) diff --git a/devicecloud/test/integration/inttest_monitor_tcp.py b/devicecloud/test/integration/inttest_monitor_tcp.py index 5bcfb6b..03afa0a 100644 --- a/devicecloud/test/integration/inttest_monitor_tcp.py +++ b/devicecloud/test/integration/inttest_monitor_tcp.py @@ -24,7 +24,7 @@ def receive_notification(notification): monitor = self._dc.monitor.get_monitor(topics) if monitor: monitor.delete() - monitor = self._dc.monitor.create_monitor(topics) + monitor = self._dc.monitor.create_tcp_monitor(topics) monitor.add_listener(receive_notification) self._dc.filedata.write_file("/~/inttest/monitor_tcp/", "test_file.txt", six.b("Hello, world!"), "text/plain") diff --git a/devicecloud/test/unit/test_monitor.py b/devicecloud/test/unit/test_monitor.py index 289204b..cfc4141 100644 --- a/devicecloud/test/unit/test_monitor.py +++ b/devicecloud/test/unit/test_monitor.py @@ -118,10 +118,10 @@ class TestMonitorAPI(HttpTestBase): - def test_create_monitor(self): + def test_create_tcp_monitor(self): self.prepare_response("POST", "/ws/Monitor", data=CREATE_MONITOR_GOOD_RESPONSE) - mon = self.dc.monitor.create_monitor(['topA', 'topB'], batch_size=10, batch_duration=0, - transport_type='tcp', compression='gzip', format_type='json') + mon = self.dc.monitor.create_tcp_monitor(['topA', 'topB'], batch_size=10, batch_duration=0, + compression='gzip', format_type='json') self.assertEqual(self._get_last_request().body, six.b(CREATE_MONITOR_GOOD_REQUEST)) self.assertEqual(mon.get_id(), 178008) From ea31776900158baeec264e2afa525d6cb57e4b24 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Mon, 15 Jun 2015 13:43:30 -0500 Subject: [PATCH 053/140] monitor: add get_monitors(condition) interface and attributes This commits adds support for a new `get_monitors` method that takes a condition as an argument. This allows for a number of queries that were previously not possible which accomadates a broad set of use cases including the ones called out by the following issues: - Getting All Monitors: https://jira.digi.com/browse/PYTHONDC-57 - Removing Inactive Monitors: https://jira.digi.com/browse/PYTHONDC-58 Signed-off-by: Paul Osborne --- devicecloud/conditions.py | 4 +- devicecloud/monitor.py | 129 +++++++++++++++++++++++++- devicecloud/test/unit/test_monitor.py | 14 +++ docs/core.rst | 10 +- 4 files changed, 148 insertions(+), 9 deletions(-) diff --git a/devicecloud/conditions.py b/devicecloud/conditions.py index 81bbf1f..4231f3b 100644 --- a/devicecloud/conditions.py +++ b/devicecloud/conditions.py @@ -106,8 +106,8 @@ def compile(self): class Attribute(object): """An attribute is a piece of data on which we may perform comparisons - Comparisons performed to attributes generated :class:`~Condition`s. - + Comparisons performed to attributes will in turn generate new + :class:`.Comparison` instances. """ def __init__(self, name): diff --git a/devicecloud/monitor.py b/devicecloud/monitor.py index 4359f44..1f8b7f3 100644 --- a/devicecloud/monitor.py +++ b/devicecloud/monitor.py @@ -15,6 +15,97 @@ logger = logging.getLogger(__name__) +#: System-generated identifier for the monitor. +MON_ID_ATTR = Attribute("monId") + +#: Device Cloud customer identifier. +MON_CST_ID_ATTR = Attribute("cstId") + +#: One or more topics to monitor separated by comma. See the device cloud +#: documentation for more details +MON_TOPIC_ATTR = Attribute("monTopic") + +#: Format for delivered event data: xml, json +MON_FORMAT_TYPE_ATTR = Attribute("monFormatType") + +#: Transport method used to deliver push notifications to the +#: client application: tcp, http +MON_TRANSPORT_TYPE_ATTR = Attribute("monTransportType") + +#: For HTTP transport type only. URL of the customer web server. For http URLs, +#: the default listening port is 80; for https URLs, the default listening +#: port is 443. +MON_HTTP_TRANSPORT_URL_ATTR = Attribute("monTransportUrl") + +#: For HTTP transport type only. Credentials for basic authentication +#: in the following format: ``username:password`` +MON_HTTP_TRANSPORT_TOKEN_ATTR = Attribute("monTransportToken") + +#: For HTTP transport type only. HTTP method to use for sending +#: data: PUT or POST. The default is PUT. +MON_HTTP_TRANSPORT_METHOD_ATTR = Attribute("monTransportMethod") + +#: For HTTP transport type only. Time in milliseconds Device Cloud waits +#: when attempting to connect to the destination http server. A value of +#: 0 means use the system default of 5000 (5 seconds). Most monitors do +#: not need to configure this setting. +MON_HTTP_CONNECT_TIMEOUT_ATTR = Attribute("monConnectTimeout") + +#: For HTTP transport type only. Time in milliseconds Device Cloud waits +#: for a response for pushed events from the http server. A value of 0 means +#: use the system default of 5000 (5 seconds). Most monitors do not need to +#: configure this setting. +MON_HTTP_RESPONSE_TIMEOUT_ATTR = Attribute("monResponseTimeout") + +#: For TCP transport type only. Indicates whether the client will explicitly +#: acknowledge TCP push events or allow Device Cloud to automatically acknowledge +#: events when sent. Options include: explicit or off. The default is off. +MON_TCP_ACK_OPTION_ATTR = Attribute("monAckOption") + +#: Specifies an upper bound on how many messages are aggregated before sending +#: a batch. The default is 100. +MON_BATCH_SIZE_ATTR = Attribute("monBatchSize") + +#: Specifies an upper bound on the number of seconds messages are aggregated +#: before sending. The default is 10. +MON_BATCH_DURATION_ATTR = Attribute("monBatchDuration") + +#: Keyword that specifies the method used to compress messages. Options include: +#: zlib or none. The default is none. For zlib, the deflate algorithm is used to +#: compress the data; use inflate to decompress the data. +#: +#: Note: For backwards compatibility, gzip is accepted as a valid keyword. +#: Compression has always been done using the deflate algorithm. +MON_COMPRESSION_ATTR = Attribute("monCompression") + +#: Boolean value that specifies whether Device Cloud replays any missed +#: published events before any new published events are forwarded. True +#: indicates missed published events are replayed. False indicates missed +#: published events are not replayed. The default is false. +MON_AUTO_REPLAY_ON_CONNECT = Attribute("monAutoReplayOnConnect") + +#: Optional text field used to label or describe the monitor. +MON_DESCRIPTION_ATTR = Attribute("monDescription") + +#: Specifies last connection time to the client application. +MON_LAST_CONNECT_ATTR = Attribute("monLastConnect") + +#: Specifies the last message pushed to the client application +MON_LAST_SENT_ATTR = Attribute("monLastSent") + +#: Specifies the current connection status to the client application: +#: +#: - CONNECTING: For HTTP monitors only. Device Cloud is attempting +#: to connect to the configured HTTP server. Once connected, the state changes to ACTIVE. +#: - ACTIVE: Monitor is connected and publishing events. +#: - INACTIVE: Monitor is not connected and events are not published or recorded. +#: - SUSPENDED: For monitors with monAutoReplayOnConnect = True. Monitor has disconnected, +#: but publish events are recorded for later replay. +#: - DISABLED: For HTTP monitors only. If a monitor has not connected for 24 hours, +#: the state is set to DISABLED, and publish events are not recorded for replay. +#: A disabled monitor must be reconfigured via the Monitor web service. +MON_STATUS_ATTR = Attribute("monStatus") + class MonitorAPI(APIBase): """Provide access to the device cloud Monitor API for receiving push notifications @@ -118,6 +209,37 @@ def create_tcp_monitor(self, topics, batch_size=1, batch_duration=0, monitor_id = int(location.split('/')[-1]) return DeviceCloudMonitor(self._conn, self._tcp_client_manager, monitor_id) + def get_monitors(self, condition=None, page_size=1000): + """Return an iterator over all monitors matching the provided condition + + Get all inactive monitors and print id:: + + for mon in dc.monitor.get_monitors(MON_STATUS_ATTR == "DISABLED"): + print(mon.get_id()) + + Get all the HTTP monitors and print id:: + + for mon in dc.monitor.get_monitors(MON_TRANSPORT_TYPE_ATTR == "http"): + print(mon.get_id()) + + Many other possibilities exist. See the :mod:`devicecloud.condition` documention + for additional details on building compound expressions. + + :param condition: An :class:`.Expression` which defines the condition + which must be matched on the monitor that will be retrieved from + the device cloud. If a condition is unspecified, an iterator over + all monitors for this account will be returned. + :type condition: :class:`.Expression` or None + :param int page_size: The number of results to fetch in a single page. + :return: Generator yielding :class:`.DeviceCloudMonitor` instances matching the + provided conditions. + """ + req_kwargs = {} + if condition: + req_kwargs['condition'] = condition.compile() + for monitor_data in self._conn.iter_json_pages("/ws/Monitor", **req_kwargs): + yield DeviceCloudMonitor.from_json(self._conn, self._tcp_client_manager, monitor_data) + def get_monitor(self, topics): """Attempts to find a Monitor in device cloud that matches the provided topics @@ -125,11 +247,8 @@ def get_monitor(self, topics): Returns a :class:`DeviceCloudMonitor` if found, otherwise None. """ - condition = (Attribute("monTopic") == ",".join(topics)) - for monitor_data in self._conn.iter_json_pages("/ws/Monitor", - condition=condition.compile()): - # just return the first one - return DeviceCloudMonitor.from_json(self._conn, self._tcp_client_manager, monitor_data) + for monitor in self.get_monitors(MON_TOPIC_ATTR == ",".join(topics)): + return monitor # return the first one, even if there are multiple return None def stop_listeners(self): diff --git a/devicecloud/test/unit/test_monitor.py b/devicecloud/test/unit/test_monitor.py index cfc4141..dc3e609 100644 --- a/devicecloud/test/unit/test_monitor.py +++ b/devicecloud/test/unit/test_monitor.py @@ -3,6 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # # Copyright (c) 2015 Digi International, Inc. +from devicecloud.monitor import MON_TOPIC_ATTR, MON_TRANSPORT_TYPE_ATTR from devicecloud.test.unit.test_utilities import HttpTestBase import six @@ -125,6 +126,19 @@ def test_create_tcp_monitor(self): self.assertEqual(self._get_last_request().body, six.b(CREATE_MONITOR_GOOD_REQUEST)) self.assertEqual(mon.get_id(), 178008) + def test_get_monitors(self): + self.prepare_response("GET", "/ws/Monitor", data=GET_MONITOR_SINGLE_FOUND) + mons = list(self.dc.monitor.get_monitors((MON_TOPIC_ATTR == "DeviceCore") & + (MON_TRANSPORT_TYPE_ATTR == "tcp"))) + self.assertEqual(len(mons), 1) + mon = mons[0] + self.assertEqual(mon.get_id(), 178007) + self.assertEqual(self._get_last_request_params(), { + 'condition': "monTopic='DeviceCore' and monTransportType='tcp'", + 'start': '0', + 'size': '1000' + }) + def test_get_monitor_present(self): self.prepare_response("GET", "/ws/Monitor", data=GET_MONITOR_SINGLE_FOUND) mon = self.dc.monitor.get_monitor(['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint']) diff --git a/docs/core.rst b/docs/core.rst index fa77ad1..169bf21 100644 --- a/docs/core.rst +++ b/docs/core.rst @@ -1,11 +1,17 @@ Core API ======== -API Documentation ------------------ +DeviceCloud Core API +-------------------- The :class:`devicecloud.DeviceCloud` class contains the core interface which will be used by all clients using the devicecloud library. .. automodule:: devicecloud :members: + +Conditions API +-------------- + +.. automodule:: devicecloud.conditions + :members: From 857306781c1c97ebb4feccd7ab6c4a5c71da3798 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Mon, 15 Jun 2015 14:21:24 -0500 Subject: [PATCH 054/140] release: version 0.3 release preparation --- CHANGELOG.md | 29 +++++++++++++++++++++++++++-- README.md | 4 ++-- devicecloud/version.py | 2 +- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f79e80b..50eb84a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,32 @@ ## Python Devicecloud Library Changelog +### 0.3 / 2015-06-15 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.2...0.3) + +Enhancements: + +* monitor: Support for the Monitor API was added allowing for + querying, adding, and removing of monitors (limiited to TCP right + now). +* monitor: Support for listening for changes and receiving callbacks + when monitors are triggered has been added. Support is limited to + SSL/TCP for now. +* streams: get_stream() now optionally accepts a ``stream_prefix`` + that allow for restricting which streams are returned based on path. +* sci: support added for ``get_job_async`` +* devicecore: provisioning: support added for removing devices from an + account. +* core: HTTP sessions are now used in order to allow for HTTP + connection reuse and credential reuse (via cookies) + +Bug Fixes: + +* Unit tests for fixed for Python 3.4 +* filedata: ``raw`` option added to work around some issues when + retrieving binary data as base64 + ### 0.2 / 2015-01-23 -[Full Changelog](https://github.com/etherios/python-devicecloud/compare/0.1.1...0.2) +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.1.1...0.2) Enhancements: @@ -36,7 +61,7 @@ Thanks to Dan Harrison, Steve Stack, Tom Manley, and Paul Osborne for contributi going into this release. ### 0.1.1 / 2014-09-14 -[Full Changelog](https://github.com/etherios/python-devicecloud/compare/0.1...0.1.1) +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.1...0.1.1) Enhancements: diff --git a/README.md b/README.md index e9445b2..7f9c6de 100644 --- a/README.md +++ b/README.md @@ -113,14 +113,14 @@ is not the case. The current features are supported by the library: * APIs to make direct web service calls to the device cloud with some details handled by the library (see DeviceCloudConnection and 'ws' documentation) * Device Provisioning via Mac Address, IMEI or Device ID +* Monitors +* Creating a TCP or HTTP monitor The following features are *not* supported at this time. Feedback on which features should be highest priority is always welcome. * Alarms -* Monitors * Scheduled Operations -* Creating a TCP or HTTP monitor * Asynchronous SCI requests * High level access to many SCI/RCI operations * DeviceMetaData diff --git a/devicecloud/version.py b/devicecloud/version.py index a045ca6..f9695f4 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -4,4 +4,4 @@ # # Copyright (c) 2015 Digi International, Inc. -__version__ = "0.2" +__version__ = "0.3" From c41479a32b0b8a90b949f48ec8c0c881c959652a Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Mon, 15 Jun 2015 14:45:23 -0500 Subject: [PATCH 055/140] docs: replace reference to old "etherios" name with digi/digidotcom --- README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7f9c6de..6a9d7bf 100644 --- a/README.md +++ b/README.md @@ -7,15 +7,15 @@ Python Device Cloud Library [![Latest Version](https://img.shields.io/pypi/v/devicecloud.svg)](https://pypi.python.org/pypi/devicecloud/) [![License](https://img.shields.io/badge/license-MPL%202.0-blue.svg)](https://github.com/digidotcom/python-devicecloud/blob/master/LICENSE) -Be sure to check out the [full documentation](http://etherios.github.io/python-devicecloud). -A [Changelog](https://github.com/etherios/python-devicecloud/blob/master/CHANGELOG.md) +Be sure to check out the [full documentation](http://digidotcom.github.io/python-devicecloud). +A [Changelog](https://github.com/digidotcom/python-devicecloud/blob/master/CHANGELOG.md) is also available. Overview -------- Python-devicecloud is a library providing simple, intuitive access to -the [Device Cloud by Etherios](http://www.etherios.com/products/devicecloud/) +the [Digi Device Cloud](http://www.digi.com/cloud/digi-device-cloud) for clients written in Python. The library wraps the Device Cloud REST API and hides the details of @@ -61,7 +61,7 @@ for stream in dc.streams.get_streams(): ``` For more examples and detailed documentation, be sure to checkout out -the [Full API Documentation](https://etherios.github.io/python-devicecloud). +the [Full API Documentation](https://digidotcom.github.io/python-devicecloud). Installation ------------ @@ -141,20 +141,19 @@ Contributions to the library are very welcome in whatever form can be provided. This could include issue reports, bug fixes, or features additions. For issue reports, please [create an issue against the Github -project](https://github.com/Etherios/python-devicecloud/issues). +project](https://github.com/digidotcom/python-devicecloud/issues). For code changes, feel free to fork the project on Github and submit a pull request with your changes. Additional instructions for developers contributing to the project can be found in the [Developer's -Guide](https://github.com/Etherios/python-devicecloud/blob/master/HACKING.md). +Guide](https://github.com/digidotcom/python-devicecloud/blob/master/HACKING.md). License ------- This software is open-source software. -Copyright (c) 2014, Etherios, Inc. All rights reserved. -Etherios, Inc. is a Division of Digi International. +Copyright (c) 2015 Digi International, Inc. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, From 097b208e74225c27ab075a643883f57abc606ebd Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Mon, 15 Jun 2015 14:48:34 -0500 Subject: [PATCH 056/140] setup: update etherios -> digi with name changes --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index d6b00ad..34b9629 100644 --- a/setup.py +++ b/setup.py @@ -51,11 +51,11 @@ def get_long_description(): setup( name="devicecloud", version=get_version(), - description="Python API to the Device Cloud by Etherios", + description="Python API to the Digi Device Cloud", long_description=get_long_description(), - url="https://github.com/etherios/python-devicecloud", - author="Etherios, Inc.", - author_email="paul.osborne@etherios.com", # TODO: mailing list? + url="https://github.com/digidotcom/python-devicecloud", + author="Digi International, Inc.", + author_email="paul.osborne@digi.com", packages=find_packages(), install_requires=open('requirements.txt').read().split(), classifiers=[ From 9d0a36c2feb0d86c20b48edbcff975a3ad92e96f Mon Sep 17 00:00:00 2001 From: Zach Varberg Date: Thu, 2 Jul 2015 15:13:46 -0500 Subject: [PATCH 057/140] Fix typo in Hacking.md --- HACKING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HACKING.md b/HACKING.md index 43e5c5b..7143fc8 100644 --- a/HACKING.md +++ b/HACKING.md @@ -20,7 +20,7 @@ Running the Unit Tests ### Running Tests with Nose -Running the tests is easy with nose (including in +Running the tests is easy with nose (included in test-requirements.txt). From the project root: $ nosetests . From 1e5102d8bafb3b4d2cb07822129397aa56f30bbe Mon Sep 17 00:00:00 2001 From: Zach Varberg Date: Thu, 2 Jul 2015 15:17:09 -0500 Subject: [PATCH 058/140] Handle using the input function in python 2 for getting username for examples Previously this used the builtin input function to get the username. In python 3 this is fine, but if python 2 this is equivalent to eval(raw_input(prompt)) and thus tried to evaluate the username as a variable and typically failed. --- devicecloud/examples/example_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/devicecloud/examples/example_helpers.py b/devicecloud/examples/example_helpers.py index 7caf53a..ea1b7af 100644 --- a/devicecloud/examples/example_helpers.py +++ b/devicecloud/examples/example_helpers.py @@ -5,6 +5,7 @@ # Copyright (c) 2015 Digi International, Inc. from getpass import getpass import os +from six.moves import input from devicecloud import DeviceCloud From c15a626e0be8344ffdd6b3a180878075f5728b82 Mon Sep 17 00:00:00 2001 From: Zach Varberg Date: Mon, 6 Jul 2015 13:13:44 -0500 Subject: [PATCH 059/140] Fix the formatting of the message in the creation of NoSuchStreamExcptions --- devicecloud/streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devicecloud/streams.py b/devicecloud/streams.py index b866afe..9c91920 100644 --- a/devicecloud/streams.py +++ b/devicecloud/streams.py @@ -613,7 +613,7 @@ def _get_stream_metadata(self, use_cached): self._cached_data = self._conn.get_json("/ws/DataStream/%s" % self._stream_id)["items"][0] except DeviceCloudHttpException as http_exception: if http_exception.response.status_code == 404: - raise NoSuchStreamException("Stream with id %r has not been created", self._stream_id) + raise NoSuchStreamException("Stream with id %r has not been created" % self._stream_id) raise http_exception return self._cached_data From 76aceafcc0f77550cfab1719fdee0e822f513a7e Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Mon, 6 Jul 2015 18:50:02 -0500 Subject: [PATCH 060/140] testing: fix inttest.sh by passing all environment variables to testenv Tox 2.0 added a new passenv option which prevents environment variables from being automatically being passed to the test environment. This resulted in environment variables like the DC username/password not making it to the tests when run with inttest.sh. For now, we just tell tox to pass all environment variables along with a wildcard. This fixes https://jira.digi.com/browse/PYTHONDC-102 Signed-off-by: Paul Osborne --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 81bd799..2187929 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ envlist = py27,py32,py33,py34,pypy [testenv] -passenv = TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH +passenv = * deps= -rtest-requirements.txt commands=nosetests -m '^(int|unit)?[Tt]est' From 130efa6a626174113bf51c6f16d8a748b18c90ce Mon Sep 17 00:00:00 2001 From: Adam Schulz Date: Mon, 6 Jul 2015 17:10:43 -0500 Subject: [PATCH 061/140] monitors: added http monitor support Added monitor support for http. Added unit tests to support http monitor. Refactored existing TCP tests to be less generic to accommodate an addition monitor (http). https://jira.digi.com/browse/PYTHONDC-54 --- devicecloud/examples/monitor_playground.py | 34 ++++- devicecloud/monitor.py | 89 ++++++++++-- .../test/integration/inttest_monitor_tcp.py | 2 +- devicecloud/test/unit/test_monitor.py | 130 ++++++++++++++++-- 4 files changed, 228 insertions(+), 27 deletions(-) diff --git a/devicecloud/examples/monitor_playground.py b/devicecloud/examples/monitor_playground.py index 7e85820..eeadba5 100644 --- a/devicecloud/examples/monitor_playground.py +++ b/devicecloud/examples/monitor_playground.py @@ -11,9 +11,7 @@ from devicecloud.streams import DataPoint -if __name__ == '__main__': - dc = get_authenticated_dc() - +def test_tcp_monitor(dc): # Create a fresh monitor over a pretty broad set of topics topics = ['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint'] mon = dc.monitor.get_monitor(topics) @@ -26,7 +24,7 @@ def listener(data): pprint.pprint(data) return True # we got it! - mon.add_listener(listener) + mon.add_callback(listener) test_stream = dc.streams.get_stream("test") try: @@ -37,3 +35,31 @@ def listener(data): print("Shutting down threads...") dc.monitor.stop_listeners() + +def test_http_monitor(dc): + # Create a fresh monitor over a pretty broad set of topics + topics = ['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint'] + mon = dc.monitor.get_monitor(topics) + if mon is not None: + mon.delete() + mon = dc.monitor.create_http_monitor(topics, 'http://digi.com', transport_token=None, transport_method='PUT', + connect_timeout=0, response_timeout=0, batch_size=1, batch_duration=0, + compression='none', format_type='json') + pprint.pprint(mon.get_metadata()) + + def listener(data): + pprint.pprint(data) + return True # we got it! + + test_stream = dc.streams.get_stream("test") + try: + while True: + test_stream.write(DataPoint(random.random())) + time.sleep(3.14) + except KeyboardInterrupt: + print("Shutting down threads...") + +if __name__ == '__main__': + dc = get_authenticated_dc() + test_http_monitor(dc) + diff --git a/devicecloud/monitor.py b/devicecloud/monitor.py index 1f8b7f3..449b9e9 100644 --- a/devicecloud/monitor.py +++ b/devicecloud/monitor.py @@ -184,7 +184,7 @@ def create_tcp_monitor(self, topics, batch_size=1, batch_duration=0, :param compression: Compression value (i.e. 'gzip'). :param format_type: What format server should send data in (i.e. 'xml' or 'json'). - Returns a string of the created Monitor Id (e.g.. 9001) + Returns an object of the created Monitor """ monitor_xml = """\ @@ -207,7 +207,59 @@ def create_tcp_monitor(self, topics, batch_size=1, batch_duration=0, response = self._conn.post("/ws/Monitor", monitor_xml) location = ET.fromstring(response.text).find('.//location').text monitor_id = int(location.split('/')[-1]) - return DeviceCloudMonitor(self._conn, self._tcp_client_manager, monitor_id) + return TCPDeviceCloudMonitor(self._conn, monitor_id, self._tcp_client_manager) + + def create_http_monitor(self, topics, transport_url, transport_token=None, transport_method='PUT',connect_timeout=0, + response_timeout=0, batch_size=1, batch_duration=0, compression='none', format_type='json'): + """Creates a HTTP Monitor instance in the device cloud for a given list of topics + + :param topics: a string list of topics (e.g. ['DeviceCore[U]', + 'FileDataCore']). + :param transport_url: URL of the customer web server. + :param transport_token: Credentials for basic authentication in the following format: username:password + :param transport_method: HTTP method to use for sending data: PUT or POST. The default is PUT. + :param connect_timeout: A value of 0 means use the system default of 5000 (5 seconds). + :param response_timeout: A value of 0 means use the system default of 5000 (5 seconds). + :param batch_size: How many Msgs received before sending data. + :param batch_duration: How long to wait before sending batch if it + does not exceed batch_size. + :param compression: Compression value (i.e. 'gzip'). + :param format_type: What format server should send data in (i.e. 'xml' or 'json'). + + Returns an object of the created Monitor + """ + + monitor_xml = """\ + + {topics} + {batch_size} + {format_type} + http + {transport_url} + {transport_token} + {transport_method} + {connect_timeout} + {response_timeout} + {compression} + + """.format( + topics=','.join(topics), + transport_url=transport_url, + transport_token=transport_token, + transport_method=transport_method, + connect_timeout=connect_timeout, + response_timeout=response_timeout, + batch_size=batch_size, + batch_duration=batch_duration, + format_type=format_type, + compression=compression, + ) + monitor_xml = textwrap.dedent(monitor_xml) + + response = self._conn.post("/ws/Monitor", monitor_xml) + location = ET.fromstring(response.text).find('.//location').text + monitor_id = int(location.split('/')[-1]) + return HTTPDeviceCloudMonitor(self._conn, monitor_id) def get_monitors(self, condition=None, page_size=1000): """Return an iterator over all monitors matching the provided condition @@ -238,7 +290,7 @@ def get_monitors(self, condition=None, page_size=1000): if condition: req_kwargs['condition'] = condition.compile() for monitor_data in self._conn.iter_json_pages("/ws/Monitor", **req_kwargs): - yield DeviceCloudMonitor.from_json(self._conn, self._tcp_client_manager, monitor_data) + yield DeviceCloudMonitor.from_json(self._conn, monitor_data, self._tcp_client_manager) def get_monitor(self, topics): """Attempts to find a Monitor in device cloud that matches the provided topics @@ -259,6 +311,8 @@ def stop_listeners(self): class DeviceCloudMonitor(object): """Provides access to a single monitor instance on the device cloud + This is a base class that should not be instantiated directly. + :type _tcp_client_manager: devicecloud.monitor_tcp.TCPClientManager :type _conn: devicecloud.DeviceCloudConnection """ @@ -266,13 +320,19 @@ class DeviceCloudMonitor(object): # TODO: consider adding getters/setters for each metadata @classmethod - def from_json(cls, conn, tcp_client_manager, monitor_data): + def from_json(cls, conn, monitor_data, tcp_client_manager): monitor_id = int(monitor_data['monId']) - return cls(conn, tcp_client_manager, monitor_id) - - def __init__(self, conn, tcp_client_manager, monitor_id): + transport_type = monitor_data['monTransportType'] + kls = { + "tcp": TCPDeviceCloudMonitor, + "http": HTTPDeviceCloudMonitor, + }.get(transport_type.lower()) + if kls is None: + raise ValueError("Unexpected monTransportType %r" % transport_type) + return kls(conn, monitor_id, tcp_client_manager) + + def __init__(self, conn, monitor_id, *args, **kwargs): self._conn = conn - self._tcp_client_manager = tcp_client_manager self._id = monitor_id def get_id(self): @@ -303,6 +363,17 @@ def delete(self): """Delete this monitor form the device cloud""" self._conn.delete("/ws/Monitor/{id}".format(id=self._id)) - def add_listener(self, callback): +class HTTPDeviceCloudMonitor(DeviceCloudMonitor): + """Device Cloud Monitor with HTTP transport type""" + + +class TCPDeviceCloudMonitor(DeviceCloudMonitor): + """Device Cloud Monitor with TCP transport type""" + + def __init__(self, conn, monitor_id, tcp_client_manager): + DeviceCloudMonitor.__init__(self, conn, monitor_id) + self._tcp_client_manager = tcp_client_manager + + def add_callback(self, callback): """Create a secure SSL/TCP listen session to the device cloud""" self._tcp_client_manager.create_session(callback, self._id) diff --git a/devicecloud/test/integration/inttest_monitor_tcp.py b/devicecloud/test/integration/inttest_monitor_tcp.py index 03afa0a..e06eecd 100644 --- a/devicecloud/test/integration/inttest_monitor_tcp.py +++ b/devicecloud/test/integration/inttest_monitor_tcp.py @@ -25,7 +25,7 @@ def receive_notification(notification): if monitor: monitor.delete() monitor = self._dc.monitor.create_tcp_monitor(topics) - monitor.add_listener(receive_notification) + monitor.add_callback(receive_notification) self._dc.filedata.write_file("/~/inttest/monitor_tcp/", "test_file.txt", six.b("Hello, world!"), "text/plain") self._dc.streams.get_stream("inttest/monitor_tcp").write(DataPoint(10)) diff --git a/devicecloud/test/unit/test_monitor.py b/devicecloud/test/unit/test_monitor.py index dc3e609..98cd629 100644 --- a/devicecloud/test/unit/test_monitor.py +++ b/devicecloud/test/unit/test_monitor.py @@ -7,7 +7,7 @@ from devicecloud.test.unit.test_utilities import HttpTestBase import six -CREATE_MONITOR_GOOD_REQUEST = """\ +CREATE_TCP_MONITOR_GOOD_REQUEST = """\ topA,topB 10 @@ -17,6 +17,21 @@ """ +CREATE_HTTP_MONITOR_GOOD_REQUEST = """\ + + topA,topB + 1 + json + http + http://digi.com + None + PUT + 0 + 0 + none + +""" + CREATE_MONITOR_GOOD_RESPONSE = """\ @@ -24,7 +39,7 @@ """ -GET_MONITOR_SINGLE_FOUND = """\ +GET_TCP_MONITOR_SINGLE_FOUND = """\ { "resultTotalRows": "1", "requestedStartRow": "0", @@ -47,7 +62,30 @@ } """ -GET_MONITOR_METADTATA = """\ +GET_HTTP_MONITOR_SINGLE_FOUND = """\ +{ + "resultTotalRows": "1", + "requestedStartRow": "0", + "resultSize": "1", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { + "monId": "178007", + "cstId": "7603", + "monTopic": "DeviceCore,FileDataCore,FileData,DataPoint", + "monTransportType": "http", + "monFormatType": "json", + "monBatchSize": "1", + "monCompression": "none", + "monStatus": "INACTIVE", + "monBatchDuration": "0" + } + ] +} +""" + +GET_TCP_MONITOR_METADTATA = """\ { "resultTotalRows": "1", "requestedStartRow": "0", @@ -70,6 +108,29 @@ } """ +GET_HTTP_MONITOR_METADTATA = """\ +{ + "resultTotalRows": "1", + "requestedStartRow": "0", + "resultSize": "1", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { + "monId": "178007", + "cstId": "7603", + "monTopic": "DeviceCore,FileDataCore,FileData,DataPoint", + "monTransportType": "http", + "monFormatType": "json", + "monBatchSize": "1", + "monCompression": "none", + "monStatus": "INACTIVE", + "monBatchDuration": "0" + } + ] +} +""" + GET_MONITOR_MULTIPLE_FOUND = """\ { "resultTotalRows": "2", @@ -90,15 +151,26 @@ "monBatchDuration": "10" }, { - "monId": "198765", + "monId": "178007", "cstId": "7603", "monTopic": "DeviceCore,FileDataCore,FileData,DataPoint", "monTransportType": "tcp", "monFormatType": "json", "monBatchSize": "1", - "monCompression": "", + "monCompression": "zlib", "monStatus": "INACTIVE", "monBatchDuration": "10" + }, + { + "monId": "178007", + "cstId": "7603", + "monTopic": "DeviceCore,FileDataCore,FileData,DataPoint", + "monTransportType": "http", + "monFormatType": "json", + "monBatchSize": "1", + "monCompression": "none", + "monStatus": "INACTIVE", + "monBatchDuration": "0" } ] } @@ -123,11 +195,20 @@ def test_create_tcp_monitor(self): self.prepare_response("POST", "/ws/Monitor", data=CREATE_MONITOR_GOOD_RESPONSE) mon = self.dc.monitor.create_tcp_monitor(['topA', 'topB'], batch_size=10, batch_duration=0, compression='gzip', format_type='json') - self.assertEqual(self._get_last_request().body, six.b(CREATE_MONITOR_GOOD_REQUEST)) + self.assertEqual(self._get_last_request().body, six.b(CREATE_TCP_MONITOR_GOOD_REQUEST)) self.assertEqual(mon.get_id(), 178008) - def test_get_monitors(self): - self.prepare_response("GET", "/ws/Monitor", data=GET_MONITOR_SINGLE_FOUND) + def test_create_http_monitor(self): + self.prepare_response("POST", "/ws/Monitor", data=CREATE_MONITOR_GOOD_RESPONSE) + mon = self.dc.monitor.create_http_monitor(['topA', 'topB'], 'http://digi.com', transport_token=None, + transport_method='PUT', connect_timeout=0, response_timeout=0, + batch_size=1, batch_duration=0, compression='none', + format_type='json') + self.assertEqual(self._get_last_request().body, six.b(CREATE_HTTP_MONITOR_GOOD_REQUEST)) + self.assertEqual(mon.get_id(), 178008) + + def test_get_tcp_monitors(self): + self.prepare_response("GET", "/ws/Monitor", data=GET_TCP_MONITOR_SINGLE_FOUND) mons = list(self.dc.monitor.get_monitors((MON_TOPIC_ATTR == "DeviceCore") & (MON_TRANSPORT_TYPE_ATTR == "tcp"))) self.assertEqual(len(mons), 1) @@ -139,8 +220,31 @@ def test_get_monitors(self): 'size': '1000' }) - def test_get_monitor_present(self): - self.prepare_response("GET", "/ws/Monitor", data=GET_MONITOR_SINGLE_FOUND) + def test_get_http_monitors(self): + self.prepare_response("GET", "/ws/Monitor", data=GET_TCP_MONITOR_SINGLE_FOUND) + mons = list(self.dc.monitor.get_monitors((MON_TOPIC_ATTR == "DeviceCore") & + (MON_TRANSPORT_TYPE_ATTR == "http"))) + self.assertEqual(len(mons), 1) + mon = mons[0] + self.assertEqual(mon.get_id(), 178007) + self.assertEqual(self._get_last_request_params(), { + 'condition': "monTopic='DeviceCore' and monTransportType='http'", + 'start': '0', + 'size': '1000' + }) + + def test_tcp_get_monitor_present(self): + self.prepare_response("GET", "/ws/Monitor", data=GET_TCP_MONITOR_SINGLE_FOUND) + mon = self.dc.monitor.get_monitor(['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint']) + self.assertEqual(mon.get_id(), 178007) + self.assertEqual(self._get_last_request_params(), { + 'condition': "monTopic='DeviceCore,FileDataCore,FileData,DataPoint'", + 'start': '0', + 'size': '1000' + }) + + def test_http_get_monitor_present(self): + self.prepare_response("GET", "/ws/Monitor", data=GET_HTTP_MONITOR_SINGLE_FOUND) mon = self.dc.monitor.get_monitor(['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint']) self.assertEqual(mon.get_id(), 178007) self.assertEqual(self._get_last_request_params(), { @@ -171,12 +275,12 @@ class TestDeviceCloudMonitor(HttpTestBase): def setUp(self): HttpTestBase.setUp(self) - self.prepare_response("GET", "/ws/Monitor", data=GET_MONITOR_SINGLE_FOUND) + self.prepare_response("GET", "/ws/Monitor", data=GET_TCP_MONITOR_SINGLE_FOUND) mon = self.dc.monitor.get_monitor(['DeviceCore', 'FileDataCore', 'FileData', 'DataPoint']) self.mon = mon - def test_get_metadata(self): - self.prepare_response("GET", "/ws/Monitor/178007", data=GET_MONITOR_METADTATA) + def test_get_tcp_metadata(self): + self.prepare_response("GET", "/ws/Monitor/178007", data=GET_TCP_MONITOR_METADTATA) self.assertEqual(self.mon.get_metadata(), { "monId": "178007", "cstId": "7603", From 5ac6c15cddf010709361c16b69e622aca93d6b28 Mon Sep 17 00:00:00 2001 From: Zach Varberg Date: Wed, 8 Jul 2015 10:47:39 -0500 Subject: [PATCH 062/140] Fix an issue with encoding and decoding data types between the devicecloud and python Previously the data was decoded (from dc type to python type) every time get_data was called on the DataPoint. This was not correct as for some data types it may not be possible to call the decode method on the already decoded data. Further, the encode method was never properly used when creating the XML to push data to the device cloud. This issue was masked by the fact that for currently supported data types the default str method used was sufficient. Signed-off-by: Zach Varberg --- devicecloud/streams.py | 49 +++++++++++++++++++++------ devicecloud/test/unit/test_streams.py | 38 ++++++++++++++++++++- 2 files changed, 76 insertions(+), 11 deletions(-) diff --git a/devicecloud/streams.py b/devicecloud/streams.py index b866afe..0ed49f8 100644 --- a/devicecloud/streams.py +++ b/devicecloud/streams.py @@ -61,6 +61,34 @@ logger = logging.getLogger("devicecloud.streams") +def _get_encoder_method(stream_type): + """A function to get the python type to device cloud type converter function. + + :param stream_type: The streams data type + :return: A function that when called with the python object will return the serializable + type for sending to the cloud. If there is no function for the given type, or the `stream_type` + is `None` the returned function will simply return the object unchanged. + """ + if stream_type is not None: + return DSTREAM_TYPE_MAP.get(stream_type.upper(), (lambda x: x, lambda x: x))[1] + else: + return lambda x: x + + +def _get_decoder_method(stream_type): + """ A function to get the device cloud type to python type converter function. + + :param stream_type: The streams data type + :return: A function that when called with the device cloud object will return the python + native type. If there is no function for the given type, or the `stream_type` is `None` + the returned function will simply return the object unchanged. + """ + if stream_type is not None: + return DSTREAM_TYPE_MAP.get(stream_type.upper(), (lambda x: x, lambda x: x))[0] + else: + return lambda x: x + + class StreamException(DeviceCloudException): """Base class for stream related exceptions""" @@ -272,6 +300,8 @@ def from_json(cls, stream, json_data): :return: (:class:`~DataPoint`) newly created :class:`~DataPoint` """ + type_converter = _get_decoder_method(stream.get_data_type()) + data = type_converter(json_data.get("data")) return cls( # these are actually properties of the stream, not the data point stream_id=stream.get_stream_id(), @@ -279,7 +309,7 @@ def from_json(cls, stream, json_data): units=stream.get_units(), # and these are part of the data point itself - data=json_data.get("data"), + data=data, description=json_data.get("description"), timestamp=json_data.get("timestampISO"), server_timestamp=json_data.get("serverTimestampISO"), @@ -303,8 +333,8 @@ def from_rollup_json(cls, stream, json_data): timestamp = isoformat(dc_utc_timestamp_to_dt(int(json_data.get("timestamp")))) # Special handling for data, all rollup data is float type - type_converter = DSTREAM_TYPE_MAP[dp.get_data_type()] - data = type_converter[0](float(json_data.get("data"))) + type_converter = _get_decoder_method(stream.get_data_type()) + data = type_converter(float(json_data.get("data"))) # Update the special fields dp.set_timestamp(timestamp) @@ -379,12 +409,7 @@ def get_id(self): def get_data(self): """Get the actual data value associated with this data point""" - data = self._data - if self._data_type is not None: - type_converters = DSTREAM_TYPE_MAP.get(self._data_type.upper()) - if type_converters: - data = type_converters[0](self._data) - return data + return self._data def set_data(self, data): """Set the data for this data point @@ -550,10 +575,14 @@ def to_xml(self): set on this datapoint. Values not set (e.g. quality) will be ommitted. """ + type_converter = _get_encoder_method(self._data_type) + # Convert from python native to device cloud + encoded_data = type_converter(self._data) + out = StringIO() out.write("") out.write("{}".format(self.get_stream_id())) - out.write("{}".format(self.get_data())) + out.write("{}".format(encoded_data)) conditional_write(out, "{}", self.get_description()) if self.get_timestamp() is not None: out.write("{}".format(isoformat(self.get_timestamp()))) diff --git a/devicecloud/test/unit/test_streams.py b/devicecloud/test/unit/test_streams.py index 30fb243..b10202a 100644 --- a/devicecloud/test/unit/test_streams.py +++ b/devicecloud/test/unit/test_streams.py @@ -10,13 +10,14 @@ from dateutil.tz import tzutc from devicecloud.streams import DataStream, STREAM_TYPE_FLOAT, DataPoint, NoSuchStreamException, ROLLUP_INTERVAL_HALF, \ - ROLLUP_METHOD_COUNT, STREAM_TYPE_INTEGER + ROLLUP_METHOD_COUNT, STREAM_TYPE_INTEGER, DSTREAM_TYPE_MAP from devicecloud.test.unit.test_utilities import HttpTestBase from devicecloud import DeviceCloudHttpException # Example HTTP Responses import httpretty +import mock import six CREATE_DATA_STREAM = { @@ -770,6 +771,41 @@ def test_accessors(self): self.assertEqual(dp.get_stream_id(), "test") self.assertEqual(dp.get_data_type(), "FLOAT") + def test_from_json_conversion(self): + stream = self._get_stream("test", with_cached_data=False) + self.prepare_response("GET", "/ws/DataStream/test", GET_TEST_DATA_STREAM) + test_json_data = {six.u('description'): six.u('Test'), + six.u('quality'): six.u('20'), + six.u('timestamp'): six.u('1404683207981'), + six.u('data'): six.u('3.14159265358'), + six.u('serverTimestampISO'): six.u('2014-07-06T21:46:47.981Z'), + six.u('location'): six.u('1.0,2.0,3.0'), + six.u('timestampISO'): six.u('2014-07-06T21:46:47.981Z'), + six.u('serverTimestamp'): six.u('1404683207981'), + six.u('id'): six.u('07d77854-0557-11e4-ab44-fa163e7ebc6b')} + + dp = DataPoint.from_json(stream, test_json_data) + self.assertEqual(3.14159265358, dp.get_data()) + + def test_get_data_no_conversion(self): + # This is to prove that the issue of calling the DSTREAM_TYPE_MAP conversion + # methods is no longer done on the get_data call. Previously this could + # result in already converted data trying to be converted again. For most + # types this is not an issue (i.e. calling float on a float) however, some + # conversions could have typing issues when run on their own results. + old_float_conversion = DSTREAM_TYPE_MAP[STREAM_TYPE_FLOAT] + mfloat = mock.Mock(side_effect=float) + DSTREAM_TYPE_MAP[STREAM_TYPE_FLOAT] = (mfloat, str) + my_float = 3.14159265358 + dp = DataPoint( + data_type=STREAM_TYPE_FLOAT, + data=my_float, + quality=0 + ) + self.assertEqual(my_float, dp.get_data()) + self.assertFalse(mfloat.called) + DSTREAM_TYPE_MAP[STREAM_TYPE_FLOAT] = old_float_conversion + def test_bad_location_string(self): dp = DataPoint(123) self.assertRaises(ValueError, dp.set_location, "0,1") From 8819de736deaafc6a5ac5266da071c79915884a0 Mon Sep 17 00:00:00 2001 From: Zach Varberg Date: Wed, 8 Jul 2015 10:48:02 -0500 Subject: [PATCH 063/140] Add support for the JSON data type This allows for the creating and pushing JSON type DataPoints to the devicecloud as well as reading and converting JSON DataPoints from the devicecloud Resolves https://jira.digi.com/browse/PYTHONDC-101 Signed-off-by: Zach Varberg --- devicecloud/examples/streams_playground.py | 33 ++++++++++++- devicecloud/streams.py | 4 +- devicecloud/test/unit/test_streams.py | 55 +++++++++++++++++++++- 3 files changed, 88 insertions(+), 4 deletions(-) diff --git a/devicecloud/examples/streams_playground.py b/devicecloud/examples/streams_playground.py index c56e2a8..91e1d04 100644 --- a/devicecloud/examples/streams_playground.py +++ b/devicecloud/examples/streams_playground.py @@ -9,7 +9,7 @@ from devicecloud.examples.example_helpers import get_authenticated_dc -from devicecloud.streams import DataPoint, NoSuchStreamException, STREAM_TYPE_INTEGER +from devicecloud.streams import DataPoint, NoSuchStreamException, STREAM_TYPE_INTEGER, STREAM_TYPE_JSON def create_stream_and_delete(dc): @@ -143,8 +143,39 @@ def bulk_write_datapoints_multiple_streams(dc): stream.delete() +def create_and_use_json_stream(dc): + # get a test stream reference + test_stream = dc.streams.get_stream_if_exists("test-json") + + # we want a clean stream to work with. If the stream exists, nuke it + if test_stream is not None: + test_stream.delete() + + test_stream = dc.streams.create_stream( + stream_id="test-json", + data_type=STREAM_TYPE_JSON, + description='a stream used for testing json', + units='international json standard unit (IJSU)', + ) + + test_stream.write(DataPoint( + data_type=STREAM_TYPE_JSON, + data = {'key1': 'value1', + 2: 2, + 'key3': [1, 2, 3]}, + description="Some JSON data in IJSUs", + ) + ) + + time.sleep(5) + + print(test_stream.get_current_value()) + + test_stream.delete() + if __name__ == '__main__': dc = get_authenticated_dc() + create_and_use_json_stream(dc) create_stream_and_delete(dc) attempt_to_delete_non_existant(dc) write_points_and_delete_some(dc) diff --git a/devicecloud/streams.py b/devicecloud/streams.py index 0ed49f8..8aaa140 100644 --- a/devicecloud/streams.py +++ b/devicecloud/streams.py @@ -5,7 +5,7 @@ # Copyright (c) 2015 Digi International, Inc. r"""Module providing classes for interacting with device cloud data streams""" - +import json import logging import datetime @@ -25,6 +25,7 @@ STREAM_TYPE_DOUBLE = "DOUBLE" STREAM_TYPE_STRING = "STRING" STREAM_TYPE_BINARY = "BINARY" +STREAM_TYPE_JSON = "JSON" STREAM_TYPE_UNKNOWN = "UNKNOWN" ROLLUP_INTERVAL_HALF = "half" @@ -53,6 +54,7 @@ STREAM_TYPE_STRING: (str, str), STREAM_TYPE_BINARY: (str, str), STREAM_TYPE_UNKNOWN: (str, str), + STREAM_TYPE_JSON: (json.loads, json.dumps) } diff --git a/devicecloud/test/unit/test_streams.py b/devicecloud/test/unit/test_streams.py index b10202a..71dff09 100644 --- a/devicecloud/test/unit/test_streams.py +++ b/devicecloud/test/unit/test_streams.py @@ -10,7 +10,7 @@ from dateutil.tz import tzutc from devicecloud.streams import DataStream, STREAM_TYPE_FLOAT, DataPoint, NoSuchStreamException, ROLLUP_INTERVAL_HALF, \ - ROLLUP_METHOD_COUNT, STREAM_TYPE_INTEGER, DSTREAM_TYPE_MAP + ROLLUP_METHOD_COUNT, STREAM_TYPE_INTEGER, DSTREAM_TYPE_MAP, STREAM_TYPE_JSON from devicecloud.test.unit.test_utilities import HttpTestBase from devicecloud import DeviceCloudHttpException @@ -18,6 +18,7 @@ # Example HTTP Responses import httpretty import mock +import re import six CREATE_DATA_STREAM = { @@ -144,6 +145,36 @@ } """ +GET_TEST_DATA_STREAM_JSON = """\ +{ +"resultSize": "1", +"requestedSize": "1000", +"pageCursor": "88afa98e-1-7efbf125", +"items": [ +{ + "cstId": "7603", + "streamId": "test", + "dataType": "JSON", + "forwardTo": "", + "currentValue": { + "id": "07d77854-0557-11e4-ab44-fa163e7ebc6b", + "timestamp": "1404683207981", + "timestampISO": "2014-07-06T21:46:47.981Z", + "serverTimestamp": "1404683207981", + "serverTimestampISO": "2014-07-06T21:46:47.981Z", + "data": "{\\"key3\\": [1, 2, 3], \\"key1\\": \\"value1\\", \\"2\\": 2}", + "description": "Test", + "quality": "20", + "location": "1.0,2.0,3.0" + }, + "description": "some description", + "units": "light years", + "dataTtl": "172800", + "rollupTtl": "432000"} +] +} +""" + GET_TEST_DATA_STREAM_NO_CURRENT_VALUE = """\ { "resultSize": "1", @@ -564,7 +595,6 @@ def parse_for_stream_id(response): self.assertEqual(parse_for_stream_id(requests[1].body), {'test'}) - class TestDataStreamDeleteDataPoints(HttpTestBase): def test_delete_datapoint(self): @@ -771,6 +801,27 @@ def test_accessors(self): self.assertEqual(dp.get_stream_id(), "test") self.assertEqual(dp.get_data_type(), "FLOAT") + def test_get_json_data(self): + stream = self._get_stream("test", with_cached_data=True) + self.prepare_response("GET", "/ws/DataStream/test", GET_TEST_DATA_STREAM_JSON) + + dp = stream.get_current_value() + expected_dict = {'key1': 'value1', '2': 2, 'key3': [1, 2, 3]} + self.assertDictEqual(expected_dict, dp.get_data()) + + def test_json_encode_to_xml(self): + my_dict = {'key1': 'value1', '2': 2, 'key3': [1, 2, 3]} + dp = DataPoint( + data_type=STREAM_TYPE_JSON, + data=my_dict, + ) + xml = dp.to_xml() + + self.assertIsNotNone(re.search('\{[ ",a-zA-Z0-9:[\]]+\}', xml)) + self.assertIsNotNone(re.search('"key1": "value1"', xml)) + self.assertIsNotNone(re.search('"2": 2', xml)) + self.assertIsNotNone(re.search('"key3": \[1, 2, 3\]', xml)) + def test_from_json_conversion(self): stream = self._get_stream("test", with_cached_data=False) self.prepare_response("GET", "/ws/DataStream/test", GET_TEST_DATA_STREAM) From 4295ce77520528d30c582f424f32e3422cca6924 Mon Sep 17 00:00:00 2001 From: Ryan Zoeller Date: Mon, 13 Jul 2015 18:55:57 -0500 Subject: [PATCH 064/140] Fixed link --- HACKING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/HACKING.md b/HACKING.md index 7143fc8..bd988d9 100644 --- a/HACKING.md +++ b/HACKING.md @@ -76,8 +76,8 @@ activated): The docs that are built will be located at docs/_build/html/index.html. -The documentation for the project is published on github using a (Github -Pages)[https://pages.github.com/] Project Site. The process for +The documentation for the project is published on github using a [Github +Pages](https://pages.github.com/) Project Site. The process for releasing a new set of documentation is the following: 1. Create a fresh clone of the project and checkout the `gh-pages` From c0069985bde7a7bdd9547eff1dc444944fba9498 Mon Sep 17 00:00:00 2001 From: Zach Varberg Date: Tue, 14 Jul 2015 15:53:46 -0500 Subject: [PATCH 065/140] Add functionality for file system service commands This adds the functionality to write, read, and delete files on a device. This does not add the functionality to create or delete directories. This resolves the following: https://jira.digi.com/browse/PYTHONDC-28 https://jira.digi.com/browse/PYTHONDC-29 https://jira.digi.com/browse/PYTHONDC-30 https://jira.digi.com/browse/PYTHONDC-31 This contains work towards the following https://jira.digi.com/browse/PYTHONDC-59 Signed-off-by: Zach Varberg --- devicecloud/__init__.py | 21 + devicecloud/apibase.py | 9 + .../file_system_service_playground.py | 66 +++ devicecloud/file_system_service.py | 430 +++++++++++++++++ devicecloud/sci.py | 4 +- .../test/unit/test_file_system_service.py | 451 ++++++++++++++++++ docs/filesystem.rst | 14 + docs/index.rst | 1 + 8 files changed, 994 insertions(+), 2 deletions(-) create mode 100644 devicecloud/examples/file_system_service_playground.py create mode 100644 devicecloud/file_system_service.py create mode 100644 devicecloud/test/unit/test_file_system_service.py create mode 100644 docs/filesystem.rst diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index e507cf5..0065d1c 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -416,6 +416,13 @@ def sci(self): self._sci_api = self.get_sci_api() return self._sci_api + @property + def file_system_service(self): + """Property providing access to the :class:`.FileSystemServiceAPI`""" + if self._fss_api is None: + self._fss_api = self.get_fss_api() + return self._fss_api + @property def monitor(self): """Property providing access to the :class:`.MonitorAPI`""" @@ -493,6 +500,20 @@ def get_sci_api(self): return ServerCommandInterfaceAPI(self._conn) + def get_fss_api(self): + """Returns a :class:`.FileSystemServiceAPI` bound to this device cloud instance + + This provides access to the same API as :attr:`.DeviceCloud.file_system_service` but will create + a new object (with a new cache) each time called. + + :return: FSS API object bound to this device cloud account + :rtype: :class:`.FileSystemServiceAPI` + + """ + from devicecloud.file_system_service import FileSystemServiceAPI + + return FileSystemServiceAPI(self.sci) + def get_monitor_api(self): """Returns a :class:`.MonitorAPI` bound to this device cloud instance diff --git a/devicecloud/apibase.py b/devicecloud/apibase.py index 3e67f02..56708df 100644 --- a/devicecloud/apibase.py +++ b/devicecloud/apibase.py @@ -11,3 +11,12 @@ class APIBase(object): """ def __init__(self, conn): self._conn = conn + + +class SCIAPIBase(object): + """Base class for API classes using SCI to communicate + + :type _sci_api: devicecloud.sci.ServerCommandInterfaceAPI + """ + def __init__(self, sci_api): + self._sci_api = sci_api diff --git a/devicecloud/examples/file_system_service_playground.py b/devicecloud/examples/file_system_service_playground.py new file mode 100644 index 0000000..252b72a --- /dev/null +++ b/devicecloud/examples/file_system_service_playground.py @@ -0,0 +1,66 @@ +import six +from devicecloud.examples.example_helpers import get_authenticated_dc +from devicecloud.file_system_service import FileSystemServiceAPI +from devicecloud.sci import DeviceTarget + + +def use_filesystem(dc, target, base_dir): + fssapi = dc.get_fss_api() + fd = dc.get_filedata_api() + + tmp_file = '{}/test.txt'.format(base_dir) + tmp_str = six.b('testing string') + + print("\nWriting temp file {}".format(tmp_file)) + print(fssapi.put_file(target, tmp_file, file_data=tmp_str)) + + tmp_server_file = 'test_file.txt' + print("\nWriting temp file to server {}".format(tmp_server_file)) + fd.write_file("/~/test_dir/", "test_file.txt", six.b("Hello, world!"), "text/plain") + + tmp_server_device_file = '{}/{}'.format(base_dir, tmp_server_file) + print("\nWriting temp file from server {}".format(tmp_server_device_file)) + fssapi.put_file(target, tmp_server_device_file, server_file='/~/test_dir/{}'.format(tmp_server_file)) + + print("\nList of files in {}".format(base_dir)) + out_dict = fssapi.list_files(target, base_dir) + print(out_dict) + + for device, (dirs, files) in out_dict.iteritems(): + for f in files: + if f.path.endswith('test.txt'): + print("\nUsing file info object to get data") + print(f.get_data()) + + print("\nUsing API to get file data") + print(fssapi.get_file(target, tmp_file)) + + print("\nUsing API to get other file data") + print(fssapi.get_file(target, tmp_server_device_file)) + + print("\nUsing API to get partial file data") + print(fssapi.get_file(target, tmp_file, offset=3, length=4)) + + print("\nUsing API to write part of a file") + fssapi.put_file(target, tmp_file, file_data=six.b("what"), offset=4) + print(fssapi.get_file(target, tmp_file)) + + print("\nUsing API to write part of a file and truncating") + fssapi.put_file(target, tmp_file, file_data=six.b("why"), offset=4, truncate=True) + print(fssapi.get_file(target, tmp_file)) + + print("\nDeleting temp file") + print(fssapi.delete_file(target, tmp_file)) + print(fssapi.delete_file(target, tmp_server_device_file)) + + print("\nList of files in {}".format(base_dir)) + out_dict = fssapi.list_files(target, base_dir) + print(out_dict) + + +if __name__ == "__main__": + dc = get_authenticated_dc() + device_id = "your-device-id-here" + target = DeviceTarget(device_id) + base_dir = '/a/directory/on/your/device' + use_filesystem(dc, target, base_dir) diff --git a/devicecloud/file_system_service.py b/devicecloud/file_system_service.py new file mode 100644 index 0000000..e2bf3f3 --- /dev/null +++ b/devicecloud/file_system_service.py @@ -0,0 +1,430 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015 Digi International, Inc. + +"""Provide access to the device cloud file system service API""" +import base64 +from collections import namedtuple +import xml.etree.ElementTree as ET + +from devicecloud.sci import DeviceTarget +import six +from devicecloud.apibase import SCIAPIBase + + +class FileSystemServiceException(Exception): + """A file system service exception""" + + +class ResponseParseError(FileSystemServiceException): + """An Exception when receiving an unexpected SCI response format""" + + +def _parse_command_response(response): + """Parse an SCI command response into ElementTree XML + + This is a helper method that takes a Requests Response object + of an SCI command response and will parse it into an ElementTree Element + representing the root of the XML response. + + :param response: The requests response object + :return: An ElementTree Element that is the root of the response XML + :raises ResponseParseError: If the response XML is not well formed + """ + try: + root = ET.fromstring(response.text) + except ET.ParseError: + raise ResponseParseError( + "Unexpected response format, could not parse XML. Response: {}".format(response.text)) + + return root + + +def _parse_error_tree(error): + """Parse an error ElementTree Node to create an ErrorInfo object + + :param error: The ElementTree error node + :return: An ErrorInfo object containing the error ID and the message. + """ + errinf = ErrorInfo(error.get('id'), None) + if error.text is not None: + errinf.message = error.text + else: + desc = error.find('./desc') + if desc is not None: + errinf.message = desc.text + return errinf + + +class ErrorInfo(object): + """Represents an error response from the device or the devicecloud + + :ivar errno: The error number reported in the response + :ivar message: The error message reported in the response + """ + + def __init__(self, errno, message): + self.errno = int(errno) + self.message = message + + def __str__(self): + return "".format(errno=self.errno, message=self.message) + + +LsInfo = namedtuple('LsInfo', ['directories', 'files']) + + +class FileInfo(object): + """Represents a file from a device + + This stores the information about a file on the device returned from an ls command. + It also provides functionality to get the contents of the represented file and delete + it. However, writing to the file is not supported as it would invalidate the other + information stored in this object. + + + :param fssapi: A :class:`~.FileSystemServiceAPI` object to perform device file operations with + :type fssapi: :class:`~.FileSystemServiceAPI` + :param device_id: The Device ID of the device this file is on + :param path: The path to this file on the device + :param last_modified: The last time the file was modified + :type last_modified: int + :param size: The size of the file + :type size: int + :param hash: The files hash + :param hash_type: The method used to produce the hash + + :ivar device_id: The Device ID of the device this file is on + :ivar path: The path to this file on the device + :ivar last_modified: The last time the file was modified + :ivar size: The size of the file + :ivar hash: The files hash + :ivar hash_type: The method used to produce the hash + """ + + def __init__(self, fssapi, device_id, path, last_modified, size, hash, hash_type): + self._fssapi = fssapi + self.device_id = device_id + self.path = path + self.last_modified = last_modified + self.size = size + self.hash = hash + self.hash_type = hash_type + + def get_data(self): + """Get the contents of this file + + :return: The contents of this file + :rtype: six.binary_type + """ + + target = DeviceTarget(self.device_id) + return self._fssapi.get_file(target, self.path)[self.device_id] + + def delete(self): + """Delete this file from the device + + .. note:: + After deleting the file, this object will no longer contain valid information + and further calls to delete or get_data will return :class:`~.ErrorInfo` objects + """ + target = DeviceTarget(self.device_id) + return self._fssapi.delete_file(target, self.path)[self.device_id] + + def __str__(self): + return "".format( + device=self.device_id, + path=self.path + ) + + def __eq__(self, other): + return (self.device_id == other.device_id and + self.path == other.path and + self.last_modified == other.last_modified and + self.size == other.size and + self.hash == other.hash) + + +class DirectoryInfo(object): + """Represents a directory from a device + + This stores the information about a directory on the device returned from an ls command. + It also provides functionality to list the contents of the directory. + + :param fssapi: A :class:`~.FileSystemServiceAPI` object to perform device directory operations with + :type fssapi: :class:`~.FileSystemServiceAPI` + :param device_id: The Device ID of the device this file is on + :param path: The path to this file on the device + :param last_modified: The last time the file was modified + :type last_modified: int + + :ivar device_id: The Device ID of the device this file is on + :ivar path: The path to this file on the device + :ivar last_modified: the last time the file was modified + """ + + def __init__(self, fssapi, device_id, path, last_modified): + self._fssapi = fssapi + self.device_id = device_id + self.path = path + self.last_modified = last_modified + + def list_contents(self): + """List the contents of this directory + + :return: A LsInfo object that contains directories and files + :rtype: :class:`~.LsInfo` or :class:`~.ErrorInfo` + + Here is an example usage + :: + # let dirinfo be a DirectoryInfo object + ldata = dirinfo.list_contents() + if isinstance(ldata, ErrorInfo): + # Do some error handling + logger.warn("Error listing file info: (%s) %s", ldata.errno, ldata.message) + # It's of type LsInfo + else: + # Look at all the files + for finfo in ldata.files: + logger.info("Found file %s of size %s", finfo.path, finfo.size) + # Look at all the directories + for dinfo in ldata.directories: + logger.info("Found directory %s of last modified %s", dinfo.path, dinfo.last_modified) + """ + target = DeviceTarget(self.device_id) + return self._fssapi.list_files(target, self.path)[self.device_id] + + def __str__(self): + return "".format( + device=self.device_id, + path=self.path, + ) + + def __eq__(self, other): + return (self.device_id == other.device_id and + self.path == other.path and + self.last_modified == other.last_modified) + + +class FileSystemServiceAPI(SCIAPIBase): + """ Encapsulate the File System Service API """ + + def list_files(self, target, path, hash='any'): + """List all files and directories in the path on the target + + :param target: The device(s) to be targeted with this request + :type target: :class:`devicecloud.sci.TargetABC` or list of :class:`devicecloud.sci.TargetABC` instances + :param path: The path on the target to list files and directories from + :param hash: an optional attribute which indicates a hash over the file contents should be retrieved. Values + include none, any, md5, and crc32. any is used to indicate the device should choose its best available hash. + :return: A dictionary with keys of device ids and values of :class:`~.LsInfo` objects containing the files and + directories or an :class:`~.ErrorInfo` object if there was an error response + :raises: :class:`~.ResponseParseError` If the SCI response has unrecognized formatting + + Here is an example usage + :: + # dc is a DeviceCloud instance + fssapi = dc.get_fss_api() + + target = AllTarget() + ls_dir = '/root/home/user/important_files/' + + ls_data = fssapi.list_files(target, ls_dir) + + # Loop over all device results + for device_id, device_data in ls_data.iteritems(): + # Check if it succeeded or was an error + if isinstance(device_data, ErrorInfo): + # Do some error handling + logger.warn("Error listing file info on device %s. errno: %s message:%s", + device_id, device_data.errno, device_data.message) + + # It's of type LsInfo + else: + # Look at all the files + for finfo in device_data.files: + logger.info("Found file %s of size %s on device %s", + finfo.path, finfo.size, device_id) + # Look at all the directories + for dinfo in device_data.directories: + logger.info("Found directory %s of last modified %s on device %s", + dinfo.path, dinfo.last_modified, device_id) + + """ + commands_el = ET.Element('commands') + ls_el = ET.SubElement(commands_el, 'ls') + ls_el.set('path', path) + ls_el.set('hash', hash) + root = _parse_command_response( + self._sci_api.send_sci("file_system", target, ET.tostring(commands_el))) + + out_dict = {} + + # At this point the XML we have is of the form + # + # + # + # + # + # + # ... + # + # ... + # + # + # + # + # + # + # + # ... + # + # ... + # + # + # + # ... + # + # + + # Here we will get each of the XML trees rooted at the device nodes + for device in root.findall('./file_system/device'): + device_id = device.get('id') + error = device.find('.//error') + if error is not None: + out_dict[device_id] = _parse_error_tree(error) + else: + hash_type = device.find('./commands/ls').get('hash') + dirs = [] + files = [] + # Get each file listed for this device + for myfile in device.findall('./commands/ls/file'): + fi = FileInfo(self, + device_id, + myfile.get('path'), + int(myfile.get('last_modified')), + int(myfile.get('size')), + myfile.get('hash'), + hash_type) + files.append(fi) + # Get each directory listed for this device + for mydir in device.findall('./commands/ls/dir'): + di = DirectoryInfo(self, + device_id, + mydir.get('path'), + int(mydir.get('last_modified'))) + dirs.append(di) + out_dict[device_id] = LsInfo(directories=dirs, files=files) + return out_dict + + def get_file(self, target, path, offset=None, length=None): + """Get the contents of a file on the device + + :param target: The device(s) to be targeted with this request + :type target: :class:`devicecloud.sci.TargetABC` or list of :class:`devicecloud.sci.TargetABC` instances + :param path: The path on the target to the file to retrieve + :param offset: Start retrieving data from this byte position in the file, if None start from the beginning + :param length: How many bytes to retrieve, if None retrieve until the end of the file + :return: A dictionary with keys of device ids and values of the bytes of the file (or partial file if offset + and/or length are specified) or an :class:`~.ErrorInfo` object if there was an error response + :raises: :class:`~.ResponseParseError` If the SCI response has unrecognized formatting + """ + commands_el = ET.Element('commands') + get_file_el = ET.SubElement(commands_el, 'get_file') + get_file_el.set('path', path) + if offset is not None: + get_file_el.set('offset', str(offset)) + if length is not None: + get_file_el.set('length', str(length)) + + root = _parse_command_response( + self._sci_api.send_sci("file_system", target, ET.tostring(commands_el))) + out_dict = {} + for device in root.findall('./file_system/device'): + device_id = device.get('id') + error = device.find('.//error') + if error is not None: + out_dict[device_id] = _parse_error_tree(error) + else: + data = six.b(device.find('./commands/get_file/data').text) + out_dict[device_id] = base64.b64decode(data) + return out_dict + + def put_file(self, target, path, file_data=None, server_file=None, offset=None, truncate=False): + """Put data into a file on the device + + :param target: The device(s) to be targeted with this request + :type target: :class:`devicecloud.sci.TargetABC` or list of :class:`devicecloud.sci.TargetABC` instances + :param path: The path on the target to the file to write to. If the file already exists it will be overwritten. + :param file_data: A `six.binary_type` containing the data to put into the file + :param server_file: The path to a file on the devicecloud server containing the data to put into the file on the + device + :param offset: Start writing bytes to the file at this position, if None start at the beginning + :param truncate: Boolean, if True after bytes are done being written end the file their even if previous data + exists beyond it. If False, leave any existing data in place. + :return: A dictionary with keys being device ids and value being None if successful or an :class:`~.ErrorInfo` + if the operation failed on that device + :raises: :class:`~.FileSystemServiceException` if either both file_data and server_file are specified or + neither are specified + :raises: :class:`~.ResponseParseError` If the SCI response has unrecognized formatting + """ + if file_data is not None and server_file is not None: + raise FileSystemServiceException("Can only specify one of server_file or file_data") + + commands_el = ET.Element('commands') + put_file_el = ET.SubElement(commands_el, 'put_file') + put_file_el.set('path', path) + put_file_el.set('truncate', 'true' if truncate else 'false') + + if offset is not None: + put_file_el.set('offset', str(offset)) + + if file_data is not None: + if not isinstance(file_data, six.binary_type): + raise TypeError("file_data must be of type {}".format(six.binary_type)) + data_el = ET.SubElement(put_file_el, 'data') + data_el.text = base64.b64encode(file_data).decode('ascii') + elif server_file is not None: + file_el = ET.SubElement(put_file_el, 'file') + file_el.text = server_file + else: + raise FileSystemServiceException("You must specify either file_data or server_file to put data into a file") + + root = _parse_command_response(self._sci_api.send_sci("file_system", target, ET.tostring(commands_el))) + out_dict = {} + for device in root.findall('./file_system/device'): + device_id = device.get('id') + error = device.find('.//error') + if error is not None: + out_dict[device_id] = _parse_error_tree(error) + else: + out_dict[device_id] = None + + return out_dict + + def delete_file(self, target, path): + """Delete a file from a device + + :param target: The device(s) to be targeted with this request + :type target: :class:`devicecloud.sci.TargetABC` or list of :class:`devicecloud.sci.TargetABC` instances + :param path: The path on the target to the file to delete. + :return: A dictionary with keys being device ids and value being None if successful or an :class:`~.ErrorInfo` + if the operation failed on that device + :raises: :class:`~.ResponseParseError` If the SCI response has unrecognized formatting + """ + commands_el = ET.Element('commands') + rm_el = ET.SubElement(commands_el, 'rm') + rm_el.set('path', path) + root = _parse_command_response(self._sci_api.send_sci("file_system", target, ET.tostring(commands_el))) + + out_dict = {} + for device in root.findall('./file_system/device'): + device_id = device.get('id') + error = device.find('.//error') + if error is not None: + out_dict[device_id] = _parse_error_tree(error) + else: + out_dict[device_id] = None + return out_dict diff --git a/devicecloud/sci.py b/devicecloud/sci.py index 7f8a970..a216754 100644 --- a/devicecloud/sci.py +++ b/devicecloud/sci.py @@ -142,8 +142,8 @@ def send_sci(self, operation, target, payload, reply=None, synchronous=None, syn TODO: document other params """ - if not isinstance(payload, six.string_types): - raise TypeError("payload is required to be string") + if not isinstance(payload, six.string_types) and not isinstance(payload, six.binary_type): + raise TypeError("payload is required to be a string or bytes") # validate targets and bulid targets xml section try: diff --git a/devicecloud/test/unit/test_file_system_service.py b/devicecloud/test/unit/test_file_system_service.py new file mode 100644 index 0000000..a7128ef --- /dev/null +++ b/devicecloud/test/unit/test_file_system_service.py @@ -0,0 +1,451 @@ +import base64 +import unittest +from xml.etree import ElementTree as ET + +from devicecloud.file_system_service import FileInfo, DirectoryInfo, FileSystemServiceException, \ + _parse_command_response, ResponseParseError, \ + ErrorInfo, LsInfo, _parse_error_tree +from devicecloud.sci import AllTarget +from devicecloud.test.unit.test_utilities import HttpTestBase +import mock +import six + + +class FileSystemResponse(object): + def __init__(self): + self.string_start = "" + self.end_string = "" + self.block = "" + + def add_device_block(self, dev_id, command_block): + block = "{command_block}" + self.block += block.format(dev_id=dev_id, command_block=command_block) + + def get_string(self): + return self.string_start + self.block + self.end_string + + @property + def text(self): + return self.string_start + self.block + self.end_string + + +LS_BLOCK = """\ + + + + + +""" + +ERROR_BLOCK = """\ +<{command}> +{errtext} + +""" + +GET_FILE_BLOCK = """\ + +{data} + +""" + +GENERIC_COMMAND_BLOCK = """<{command}>""" + +PUT_FILE_DATA_COMMAND = """\ +\ +\ +{data}\ +\ +\ +""" + +PUT_FILE_FILE_COMMAND = """\ +\ +\ +{server_file}\ +\ +\ +""" + +DELETE_FILE_COMMAND = """\ +\ +\ +\ +""" + + +class TestFileInfo(unittest.TestCase): + def setUp(self): + self.fss_api = mock.Mock() + self.dev_id = '00000000-00000000-18A905FF-FF2F1BBD' + + def test_eq_not_eq(self): + file1 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + file2 = FileInfo(self.fss_api, self.dev_id, '/a/path/file2.py', 1434377919, 181, + "DEA17715739E46079C1A6DDCB38344DF", 'md5') + self.assertNotEqual(file1, file2) + + def test_eq(self): + file1 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + file2 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + self.assertEqual(file1, file2) + self.assertFalse(file1 is file2) + + def test_get_data(self): + self.fss_api.get_file.side_effect = ({self.dev_id: "some file data"},) + file1 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + self.assertEqual("some file data", file1.get_data()) + self.assertEqual(1, self.fss_api.get_file.call_count) + call_name, call_args, call_kwargs = self.fss_api.get_file.mock_calls[0] + self.assertEqual(call_args[0]._device_id, self.dev_id) + self.assertEqual(call_args[1], '/a/path/file1.txt') + + def test_get_data_error(self): + error = ErrorInfo(1, "error message") + self.fss_api.get_file.side_effect = ({self.dev_id: error},) + file1 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + self.assertEqual(error, file1.get_data()) + self.assertEqual(1, self.fss_api.get_file.call_count) + call_name, call_args, call_kwargs = self.fss_api.get_file.mock_calls[0] + self.assertEqual(call_args[0]._device_id, self.dev_id) + self.assertEqual(call_args[1], '/a/path/file1.txt') + + def test_delete(self): + self.fss_api.delete_file.side_effect = ({self.dev_id: None},) + file1 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + self.assertIsNone(file1.delete()) + self.assertEqual(1, self.fss_api.delete_file.call_count) + call_name, call_args, call_kwargs = self.fss_api.delete_file.mock_calls[0] + self.assertEqual(call_args[0]._device_id, self.dev_id) + self.assertEqual(call_args[1], '/a/path/file1.txt') + + def test_delete_error(self): + error = ErrorInfo(1, "error message") + self.fss_api.delete_file.side_effect = ({self.dev_id: error},) + file1 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + ret_err = file1.delete() + self.assertEqual(error.errno, ret_err.errno) + self.assertEqual(error.message, ret_err.message) + self.assertEqual(1, self.fss_api.delete_file.call_count) + call_name, call_args, call_kwargs = self.fss_api.delete_file.mock_calls[0] + self.assertEqual(call_args[0]._device_id, self.dev_id) + self.assertEqual(call_args[1], '/a/path/file1.txt') + + +class TestDirectoryInfo(unittest.TestCase): + def setUp(self): + self.fss_api = mock.Mock() + self.dev_id = '00000000-00000000-18A905FF-FF2F1BBD' + + def test_eq_not_eq(self): + dir1 = DirectoryInfo(self.fss_api, self.dev_id, '/a/path/dir1', 1436276773) + dir2 = DirectoryInfo(self.fss_api, self.dev_id, '/a/path/dir2', 1434377919) + self.assertNotEqual(dir1, dir2) + + def test_eq(self): + dir1 = DirectoryInfo(self.fss_api, self.dev_id, '/a/path/dir1', 1436276773) + dir2 = DirectoryInfo(self.fss_api, self.dev_id, '/a/path/dir1', 1436276773) + self.assertEqual(dir1, dir2) + self.assertFalse(dir1 is dir2) + + def test_list_contents(self): + file1 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + file2 = FileInfo(self.fss_api, self.dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + self.fss_api.list_files.side_effect = ({self.dev_id: LsInfo([], [file1, file2])},) + dir1 = DirectoryInfo(self.fss_api, self.dev_id, '/a/path/dir1', 1436276773) + dirs, files = dir1.list_contents() + self.assertEqual(0, len(dirs)) + self.assertListEqual([file1, file2], files) + self.assertEqual(1, self.fss_api.list_files.call_count) + call_name, call_args, call_kwargs = self.fss_api.list_files.mock_calls[0] + self.assertEqual(call_args[0]._device_id, self.dev_id) + self.assertEqual(call_args[1], '/a/path/dir1') + + +class TestFileSystemServiceAPI(HttpTestBase): + def setUp(self): + HttpTestBase.setUp(self) + self.fss_api = self.dc.get_fss_api() + self.sci_api = mock.Mock() + self.fss_api._sci_api = self.sci_api + self.target = AllTarget() + self.dev1_id = '00000000-00000000-18A905FF-FF2F1BBD' + self.dev2_id = '00000000-00000000-18A905FF-FF2F1BBE' + + def prep_sci_response(self, response): + self.sci_api.send_sci.side_effect = (response,) + + def test_parse_command_response_bad_xml_response(self): + fsr = FileSystemResponse() + fsr.add_device_block('asdf', '<> some_garbage_data') + self.assertRaises(ResponseParseError, _parse_command_response, fsr) + + def test_parse_command_response_good_data(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, "") + root = _parse_command_response(fsr) + self.assertIsNotNone(root.find('.//some_command')) + + def test_parse_error_tree_text(self): + command = ET.fromstring(ERROR_BLOCK.format(command='command', errno=1, errtext="some text")) + error = command.find('./error') + errinfo = _parse_error_tree(error) + self.assertEqual(errinfo.errno, 1) + self.assertEqual(errinfo.message, 'some text') + + def test_parse_error_tree_desc_node(self): + command = ET.fromstring(ERROR_BLOCK.format(command='command', errno=1, errtext="some text")) + error = command.find('./error') + errinfo = _parse_error_tree(error) + self.assertEqual(errinfo.errno, 1) + self.assertEqual(errinfo.message, 'some text') + + def test_list_dir(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, LS_BLOCK) + fsr.add_device_block(self.dev2_id, LS_BLOCK) + self.prep_sci_response(fsr) + list_dict = self.fss_api.list_files(self.target, '/a/path/') + + file1 = FileInfo(self.fss_api, self.dev1_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + file2 = FileInfo(self.fss_api, self.dev1_id, '/a/path/file2.py', 1434377919, 181, + "DEA17715739E46079C1A6DDCB38344DF", 'md5') + files_dev1 = [file1, file2] + dir1 = DirectoryInfo(self.fss_api, self.dev1_id, '/a/path/dir', 1436203917) + dirs_dev1 = [dir1] + + file1 = FileInfo(self.fss_api, self.dev2_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + file2 = FileInfo(self.fss_api, self.dev2_id, '/a/path/file2.py', 1434377919, 181, + "DEA17715739E46079C1A6DDCB38344DF", 'md5') + files_dev2 = [file1, file2] + dir1 = DirectoryInfo(self.fss_api, self.dev2_id, '/a/path/dir', 1436203917) + dirs_dev2 = [dir1] + + expected_dict = {self.dev1_id: LsInfo(dirs_dev1, files_dev1), + self.dev2_id: LsInfo(dirs_dev2, files_dev2)} + + self.assertDictEqual(expected_dict, list_dict) + + def test_list_nonexistent_dir(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, + ERROR_BLOCK.format(command="ls", errno=1, errtext="No such file or directory")) + self.prep_sci_response(fsr) + out_dict = self.fss_api.list_files(self.target, '/a/nonexistent/path') + self.assertEqual(1, len(out_dict.keys())) + self.assertTrue(self.dev1_id in out_dict.keys()) + error = out_dict[self.dev1_id] + self.assertEqual(1, error.errno) + self.assertEqual("No such file or directory", error.message) + + def test_get_entire_file(self): + data_string = base64.b64encode(six.b('testing string')).decode('ascii') + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GET_FILE_BLOCK.format(data=data_string)) + fsr.add_device_block(self.dev2_id, GET_FILE_BLOCK.format(data=data_string)) + self.prep_sci_response(fsr) + get_file_data = self.fss_api.get_file(self.target, '/a/path/file1.txt') + expected_dict = { + self.dev1_id: six.b('testing string'), + self.dev2_id: six.b('testing string'), + } + self.assertDictEqual(expected_dict, get_file_data) + + def test_get_partial_file(self): + data_string = base64.b64encode(six.b('ting')).decode('ascii') + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GET_FILE_BLOCK.format(data=data_string)) + fsr.add_device_block(self.dev2_id, GET_FILE_BLOCK.format(data=data_string)) + self.prep_sci_response(fsr) + get_file_data = self.fss_api.get_file(self.target, '/a/path/file1.txt', offset=2, length=4) + expected_dict = { + self.dev1_id: six.b('ting'), + self.dev2_id: six.b('ting'), + } + self.assertDictEqual(expected_dict, get_file_data) + + def test_get_nonexistent_file(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, + ERROR_BLOCK.format(command='get_file', errno=1, errtext="No such file or directory")) + self.prep_sci_response(fsr) + out_dict = self.fss_api.get_file(self.target, '/a/nonexistent/path') + self.assertEqual(1, len(out_dict.keys())) + self.assertTrue(self.dev1_id in out_dict.keys()) + error = out_dict[self.dev1_id] + self.assertEqual(1, error.errno) + self.assertEqual("No such file or directory", error.message) + + def test_get_file_some_error(self): + data_string = base64.b64encode(six.b('testing string')).decode('ascii') + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, + ERROR_BLOCK.format(command='get_file', errno=1, errtext="No such file or directory")) + fsr.add_device_block(self.dev2_id, GET_FILE_BLOCK.format(data=data_string)) + self.prep_sci_response(fsr) + out_dict = self.fss_api.get_file(self.target, '/a/nonexistent/path') + self.assertEqual(2, len(out_dict.keys())) + self.assertTrue(self.dev1_id in out_dict.keys()) + self.assertTrue(self.dev2_id in out_dict.keys()) + + # Verify error info + error = out_dict[self.dev1_id] + self.assertEqual(1, error.errno) + self.assertEqual("No such file or directory", error.message) + + # Verify OK data + self.assertEqual(six.b('testing string'), out_dict[self.dev2_id]) + + def test_put_complete_file(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + fsr.add_device_block(self.dev2_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + out_dict = self.fss_api.put_file(self.target, file_path, file_data=six.b('testing string')) + + expected_dict = { + self.dev1_id: None, + self.dev2_id: None + } + self.assertDictEqual(expected_dict, out_dict) + + self.sci_api.send_sci.assert_called_once_with('file_system', self.target, six.b(PUT_FILE_DATA_COMMAND.format( + path=file_path, + data=base64.b64encode(six.b('testing string')).decode('ascii'), + offset="", + truncate='false'))) + + def test_put_partial_file(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + fsr.add_device_block(self.dev2_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + out_dict = self.fss_api.put_file(self.target, file_path, file_data=six.b('testing string'), offset=5) + expected_dict = { + self.dev1_id: None, + self.dev2_id: None + } + self.assertDictEqual(expected_dict, out_dict) + + self.sci_api.send_sci.assert_called_once_with('file_system', self.target, six.b(PUT_FILE_DATA_COMMAND.format( + path=file_path, + data=base64.b64encode(six.b('testing string')).decode('ascii'), + offset="offset=\"5\" ", + truncate='false'))) + + def test_put_partial_file_truncate(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + fsr.add_device_block(self.dev2_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + out_dict = self.fss_api.put_file(self.target, file_path, file_data=six.b('testing string'), offset=5, + truncate=True) + expected_dict = { + self.dev1_id: None, + self.dev2_id: None + } + self.assertDictEqual(expected_dict, out_dict) + + self.sci_api.send_sci.assert_called_once_with('file_system', self.target, six.b(PUT_FILE_DATA_COMMAND.format( + path=file_path, + data=base64.b64encode(six.b('testing string')).decode('ascii'), + offset="offset=\"5\" ", + truncate='true'))) + + def test_put_file_both_data_args(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + fsr.add_device_block(self.dev2_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + self.assertRaises(FileSystemServiceException, self.fss_api.put_file, self.target, file_path, + file_data=six.b('testing string'), server_file='/a/path/file2.txt') + + def test_put_file_both_neither_args(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + fsr.add_device_block(self.dev2_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + self.assertRaises(FileSystemServiceException, self.fss_api.put_file, self.target, file_path) + + def test_put_file_server_file(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + fsr.add_device_block(self.dev2_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + server_file = '/a/path/file2.txt' + out_dict = self.fss_api.put_file(self.target, file_path, server_file=server_file) + expected_dict = { + self.dev1_id: None, + self.dev2_id: None + } + self.assertDictEqual(expected_dict, out_dict) + + self.sci_api.send_sci.assert_called_once_with('file_system', self.target, six.b(PUT_FILE_FILE_COMMAND.format( + path=file_path, + server_file=server_file, + offset="", + truncate='false'))) + + def test_put_file_some_error(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='put_file')) + fsr.add_device_block(self.dev2_id, + ERROR_BLOCK.format(command='put_file', errno='1', errtext='something went wrong')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + server_file = '/a/path/file2.txt' + out_dict = self.fss_api.put_file(self.target, file_path, server_file=server_file) + self.assertIsNone(out_dict[self.dev1_id]) + error = out_dict[self.dev2_id] + self.assertEqual(1, error.errno) + self.assertEqual('something went wrong', error.message) + + def test_delete_file(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='rm')) + fsr.add_device_block(self.dev2_id, GENERIC_COMMAND_BLOCK.format(command='rm')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + out_dict = self.fss_api.delete_file(self.target, file_path) + expected_dict = { + self.dev1_id: None, + self.dev2_id: None + } + self.assertDictEqual(expected_dict, out_dict) + + self.sci_api.send_sci.assert_called_once_with('file_system', self.target, six.b(DELETE_FILE_COMMAND.format( + path=file_path))) + + def test_delete_file_some_error(self): + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, GENERIC_COMMAND_BLOCK.format(command='rm')) + fsr.add_device_block(self.dev2_id, ERROR_BLOCK.format(command='rm', errno='1', errtext='something went wrong')) + self.prep_sci_response(fsr) + file_path = '/a/path/file1.txt' + server_file = '/a/path/file2.txt' + out_dict = self.fss_api.delete_file(self.target, file_path) + self.assertIsNone(out_dict[self.dev1_id]) + error = out_dict[self.dev2_id] + self.assertEqual(1, error.errno) + self.assertEqual('something went wrong', error.message) + + +if __name__ == '__main__': + unittest.main() diff --git a/docs/filesystem.rst b/docs/filesystem.rst new file mode 100644 index 0000000..9622665 --- /dev/null +++ b/docs/filesystem.rst @@ -0,0 +1,14 @@ +File System Service API +================================== + +File System Service Overview +---------------------------- + +Provide access to the device cloud File System commands that use SCI to +get the data from your devices connected to the cloud. + +File System Service API Documentation +------------------------------------- + +.. automodule:: devicecloud.file_system_service + :members: diff --git a/docs/index.rst b/docs/index.rst index d65b302..44a97fa 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,6 +12,7 @@ Documention Map streams filedata sci + filesystem monitor ws cookbook From bacbe57ea1f4bb09ab0d0f62fb65749f6eb3d79d Mon Sep 17 00:00:00 2001 From: Zach Varberg Date: Tue, 28 Jul 2015 13:07:16 -0500 Subject: [PATCH 066/140] Add support for listing all items modified since a given time. This resolves the following: https://jira.digi.com/browse/PYTHONDC-34 Signed-off-by: Zach Varberg --- .../file_system_service_playground.py | 74 ++++++++--- devicecloud/file_system_service.py | 31 +++++ .../test/unit/test_file_system_service.py | 120 ++++++++++++++++-- 3 files changed, 196 insertions(+), 29 deletions(-) diff --git a/devicecloud/examples/file_system_service_playground.py b/devicecloud/examples/file_system_service_playground.py index 252b72a..1c07ba4 100644 --- a/devicecloud/examples/file_system_service_playground.py +++ b/devicecloud/examples/file_system_service_playground.py @@ -1,7 +1,49 @@ +import time import six from devicecloud.examples.example_helpers import get_authenticated_dc -from devicecloud.file_system_service import FileSystemServiceAPI from devicecloud.sci import DeviceTarget +from devicecloud.file_system_service import FileSystemServiceAPI, ErrorInfo + + +def put_test_file(fssapi, target, tmp_file_path): + tmp_str = six.b('testing string') + + print("\nWriting test file {}".format(tmp_file_path)) + print(fssapi.put_file(target, tmp_file_path, file_data=tmp_str)) + + +def list_contents(fssapi, target, dev_dir): + print("\nList of files in {}".format(dev_dir)) + out_dict = fssapi.list_files(target, dev_dir) + print("list_files returned: {}".format(str(out_dict))) + print_list_contents(out_dict) + + +def print_list_contents(out_dict): + print("\nPrint each item from list_files:") + for device_id, device_data in six.iteritems(out_dict): + print("Items from device {}".format(device_id)) + if isinstance(device_data, ErrorInfo): + print(" ErrorInfo: {}".format(device_data)) + else: + (dirs, files) = device_data + if len(dirs) + len(files) == 0: + print " None" + for d in dirs: + print(" Directory: {}".format(str(d))) + for f in files: + print(" File: {}".format(str(f))) + + +def delete_test_file(fssapi, target, tmp_file_path): + print("\nDeleting test file: {}".format(tmp_file_path)) + print(fssapi.delete_file(target, tmp_file_path)) + + +def get_modified_files(fssapi, target, dev_dir, last_modified_cutoff): + print("\nGetting all files modified since {}".format(last_modified_cutoff)) + out_dict = fssapi.get_modified_items(target, dev_dir, last_modified_cutoff) + print_list_contents(out_dict) def use_filesystem(dc, target, base_dir): @@ -9,10 +51,8 @@ def use_filesystem(dc, target, base_dir): fd = dc.get_filedata_api() tmp_file = '{}/test.txt'.format(base_dir) - tmp_str = six.b('testing string') - print("\nWriting temp file {}".format(tmp_file)) - print(fssapi.put_file(target, tmp_file, file_data=tmp_str)) + put_test_file(fssapi, target, tmp_file) tmp_server_file = 'test_file.txt' print("\nWriting temp file to server {}".format(tmp_server_file)) @@ -22,15 +62,7 @@ def use_filesystem(dc, target, base_dir): print("\nWriting temp file from server {}".format(tmp_server_device_file)) fssapi.put_file(target, tmp_server_device_file, server_file='/~/test_dir/{}'.format(tmp_server_file)) - print("\nList of files in {}".format(base_dir)) - out_dict = fssapi.list_files(target, base_dir) - print(out_dict) - - for device, (dirs, files) in out_dict.iteritems(): - for f in files: - if f.path.endswith('test.txt'): - print("\nUsing file info object to get data") - print(f.get_data()) + list_contents(fssapi, target, base_dir) print("\nUsing API to get file data") print(fssapi.get_file(target, tmp_file)) @@ -49,9 +81,8 @@ def use_filesystem(dc, target, base_dir): fssapi.put_file(target, tmp_file, file_data=six.b("why"), offset=4, truncate=True) print(fssapi.get_file(target, tmp_file)) - print("\nDeleting temp file") - print(fssapi.delete_file(target, tmp_file)) - print(fssapi.delete_file(target, tmp_server_device_file)) + delete_test_file(fssapi, target, tmp_file) + delete_test_file(fssapi, target, tmp_server_device_file) print("\nList of files in {}".format(base_dir)) out_dict = fssapi.list_files(target, base_dir) @@ -64,3 +95,14 @@ def use_filesystem(dc, target, base_dir): target = DeviceTarget(device_id) base_dir = '/a/directory/on/your/device' use_filesystem(dc, target, base_dir) + + fssapi = dc.get_fss_api() + tmp_file_path = "{}/{}".format(base_dir, 'test_file.txt') + put_test_file(fssapi, target, tmp_file_path) + cutoff_time = time.time() + get_modified_files(fssapi, target, base_dir, cutoff_time) + print("\nModifying file {}".format(tmp_file_path)) + fssapi.put_file(target, tmp_file_path, file_data=six.b("data"), offset=4) + time.sleep(5) + get_modified_files(fssapi, target, base_dir, cutoff_time) + delete_test_file(fssapi, target, tmp_file_path) diff --git a/devicecloud/file_system_service.py b/devicecloud/file_system_service.py index e2bf3f3..0631d84 100644 --- a/devicecloud/file_system_service.py +++ b/devicecloud/file_system_service.py @@ -428,3 +428,34 @@ def delete_file(self, target, path): else: out_dict[device_id] = None return out_dict + + def get_modified_items(self, target, path, last_modified_cutoff): + """Get all files and directories from a path on the device modified since a given time + + :param target: The device(s) to be targeted with this request + :type target: :class:`devicecloud.sci.TargetABC` or list of :class:`devicecloud.sci.TargetABC` instances + :param path: The path on the target to the directory to check for modified files. + :param last_modified_cutoff: The time (as Unix epoch time) to get files modified since + :type last_modified_cutoff: int + :return: A dictionary where the key is a device id and the value is either an :class:`~.ErrorInfo` if there + was a problem with the operation or a :class:`~.LsInfo` with the items modified since the + specified date + """ + file_list = self.list_files(target, path) + out_dict = {} + for device_id, device_data in six.iteritems(file_list): + if isinstance(device_data, ErrorInfo): + out_dict[device_id] = device_data + else: + files = [] + dirs = [] + for cur_file in device_data.files: + if cur_file.last_modified > last_modified_cutoff: + files.append(cur_file) + + for cur_dir in device_data.directories: + if cur_dir.last_modified > last_modified_cutoff: + dirs.append(cur_dir) + out_dict[device_id] = LsInfo(directories=dirs, files=files) + + return out_dict diff --git a/devicecloud/test/unit/test_file_system_service.py b/devicecloud/test/unit/test_file_system_service.py index a7128ef..d4de030 100644 --- a/devicecloud/test/unit/test_file_system_service.py +++ b/devicecloud/test/unit/test_file_system_service.py @@ -181,6 +181,21 @@ def setUp(self): self.dev1_id = '00000000-00000000-18A905FF-FF2F1BBD' self.dev2_id = '00000000-00000000-18A905FF-FF2F1BBE' + # Create some file, directory, and error info objects to use in tests + self.file1 = FileInfo(self.fss_api, self.dev1_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + self.file2 = FileInfo(self.fss_api, self.dev1_id, '/a/path/file2.py', 1434377919, 181, + "DEA17715739E46079C1A6DDCB38344DF", 'md5') + self.dir1 = DirectoryInfo(self.fss_api, self.dev1_id, '/a/path/dir', 1436203917) + self.errinfo = ErrorInfo(errno=1, message="an error message") + + def clone_file(self, orig_file): + return FileInfo(orig_file._fssapi, orig_file.device_id, orig_file.path, + orig_file.last_modified, orig_file.size, orig_file.hash, orig_file.hash_type) + + def clone_dir(self, orig_dir): + return DirectoryInfo(orig_dir._fssapi, orig_dir.device_id, orig_dir.path, orig_dir.last_modified) + def prep_sci_response(self, response): self.sci_api.send_sci.side_effect = (response,) @@ -216,20 +231,17 @@ def test_list_dir(self): self.prep_sci_response(fsr) list_dict = self.fss_api.list_files(self.target, '/a/path/') - file1 = FileInfo(self.fss_api, self.dev1_id, '/a/path/file1.txt', 1436276773, 7989, - "967FDA522517B9CE0C3E056EDEB485BB", 'md5') - file2 = FileInfo(self.fss_api, self.dev1_id, '/a/path/file2.py', 1434377919, 181, - "DEA17715739E46079C1A6DDCB38344DF", 'md5') - files_dev1 = [file1, file2] - dir1 = DirectoryInfo(self.fss_api, self.dev1_id, '/a/path/dir', 1436203917) - dirs_dev1 = [dir1] + files_dev1 = [self.file1, self.file2] + dirs_dev1 = [self.dir1] + + file1 = self.clone_file(self.file1) + file1.device_id = self.dev2_id + file2 = self.clone_file(self.file2) + file2.device_id = self.dev2_id - file1 = FileInfo(self.fss_api, self.dev2_id, '/a/path/file1.txt', 1436276773, 7989, - "967FDA522517B9CE0C3E056EDEB485BB", 'md5') - file2 = FileInfo(self.fss_api, self.dev2_id, '/a/path/file2.py', 1434377919, 181, - "DEA17715739E46079C1A6DDCB38344DF", 'md5') files_dev2 = [file1, file2] - dir1 = DirectoryInfo(self.fss_api, self.dev2_id, '/a/path/dir', 1436203917) + dir1 = self.clone_dir(self.dir1) + dir1.device_id = self.dev2_id dirs_dev2 = [dir1] expected_dict = {self.dev1_id: LsInfo(dirs_dev1, files_dev1), @@ -439,13 +451,95 @@ def test_delete_file_some_error(self): fsr.add_device_block(self.dev2_id, ERROR_BLOCK.format(command='rm', errno='1', errtext='something went wrong')) self.prep_sci_response(fsr) file_path = '/a/path/file1.txt' - server_file = '/a/path/file2.txt' out_dict = self.fss_api.delete_file(self.target, file_path) self.assertIsNone(out_dict[self.dev1_id]) error = out_dict[self.dev2_id] self.assertEqual(1, error.errno) self.assertEqual('something went wrong', error.message) + def test_get_modified_items(self): + + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[], files=[self.file1, self.file2]) + out_dict = {self.dev1_id: linfo, self.dev2_id: LsInfo(directories=[], files=[])} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.get_modified_items(self.target, '/a/path/', self.file1.last_modified - 1) + + expected_out_dict = { + self.dev1_id: LsInfo(files=[self.file1], directories=[]), + self.dev2_id: LsInfo([], []) + } + + self.assertDictEqual(expected_out_dict, out_dict) + + def test_get_modified_items_errinfo(self): + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[], files=[self.file1, self.file2]) + out_dict = {self.dev1_id: linfo, self.dev2_id: self.errinfo} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.get_modified_items(self.target, '/a/path/', self.file1.last_modified - 1) + + expected_out_dict = { + self.dev1_id: LsInfo(files=[self.file1], directories=[]), + self.dev2_id: self.errinfo + } + + self.assertDictEqual(expected_out_dict, out_dict) + + def test_get_modified_items_no_results(self): + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[], files=[self.file1, self.file2]) + out_dict = {self.dev1_id: linfo, self.dev2_id: linfo} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.get_modified_items(self.target, '/a/path/', self.file1.last_modified + self.file2.last_modified) + + expected_out_dict = { + self.dev1_id: LsInfo([], []), + self.dev2_id: LsInfo([], []) + } + + self.assertDictEqual(expected_out_dict, out_dict) + + def test_get_modified_items_mult_dev_results(self): + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[], files=[self.file1, self.file2]) + out_dict = {self.dev1_id: linfo, self.dev2_id: linfo} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.get_modified_items(self.target, '/a/path/', self.file1.last_modified - 1) + + expected_out_dict = { + self.dev1_id: LsInfo(files=[self.file1], directories=[]), + self.dev2_id: LsInfo(files=[self.file1], directories=[]) + } + + self.assertDictEqual(expected_out_dict, out_dict) + + def test_get_modified_directory_results(self): + dir2 = DirectoryInfo(self.fss_api, self.dev1_id, '/a/path/dir2', self.dir1.last_modified - 2) + + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[self.dir1, dir2], files=[]) + out_dict = {self.dev1_id: linfo, self.dev2_id: LsInfo([], [])} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.get_modified_items(self.target, '/a/path/', self.dir1.last_modified - 1) + + expected_out_dict = { + self.dev1_id: LsInfo(files=[], directories=[self.dir1]), + self.dev2_id: LsInfo([], []) + } + + self.assertDictEqual(expected_out_dict, out_dict) + if __name__ == '__main__': unittest.main() From 61ebea076a67cac4aa472759aef01062e650b65d Mon Sep 17 00:00:00 2001 From: Zach Varberg Date: Thu, 16 Jul 2015 11:50:10 -0500 Subject: [PATCH 067/140] Add support to the FileSysteServiceAPI to check if a path exists on devices This resolves the following: https://jira.digi.com/browse/PYTHONDC-27 Signed-off-by: Zach Varberg --- devicecloud/file_system_service.py | 29 +++++++++ .../test/unit/test_file_system_service.py | 65 +++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/devicecloud/file_system_service.py b/devicecloud/file_system_service.py index 0631d84..273be85 100644 --- a/devicecloud/file_system_service.py +++ b/devicecloud/file_system_service.py @@ -459,3 +459,32 @@ def get_modified_items(self, target, path, last_modified_cutoff): out_dict[device_id] = LsInfo(directories=dirs, files=files) return out_dict + + def exists(self, target, path, path_sep="/"): + """Check if path refers to an existing path on the device + + :param target: The device(s) to be targeted with this request + :type target: :class:`devicecloud.sci.TargetABC` or list of :class:`devicecloud.sci.TargetABC` instances + :param path: The path on the target to check for existence. + :param path_sep: The path separator of the device + :return: A dictionary where the key is a device id and the value is either an :class:`~.ErrorInfo` if there + was a problem with the operation or a boolean with the existence status of the path on that device + """ + if path.endswith(path_sep): + path = path[:-len(path_sep)] + par_dir, filename = path.rsplit(path_sep, 1) + file_list = self.list_files(target, par_dir) + out_dict = {} + for device_id, device_data in six.iteritems(file_list): + if isinstance(device_data, ErrorInfo): + out_dict[device_id] = device_data + else: + out_dict[device_id] = False + for cur_file in device_data.files: + if cur_file.path == path: + out_dict[device_id] = True + for cur_dir in device_data.directories: + if cur_dir.path == path: + out_dict[device_id] = True + + return out_dict diff --git a/devicecloud/test/unit/test_file_system_service.py b/devicecloud/test/unit/test_file_system_service.py index d4de030..b1cb911 100644 --- a/devicecloud/test/unit/test_file_system_service.py +++ b/devicecloud/test/unit/test_file_system_service.py @@ -540,6 +540,71 @@ def mock_list_files(*args, **kwargs): self.assertDictEqual(expected_out_dict, out_dict) + def test_exists_file(self): + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[], files=[self.file1]) + out_dict = {self.dev1_id: linfo, self.dev2_id: LsInfo([], [])} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.exists(self.target, self.file1.path) + + expected_out_dict = { + self.dev1_id: True, + self.dev2_id: False + } + + self.assertDictEqual(expected_out_dict, out_dict) + + def test_exists_dir(self): + + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[self.dir1], files=[]) + out_dict = {self.dev1_id: linfo, self.dev2_id: LsInfo([], [])} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.exists(self.target, self.dir1.path) + + expected_out_dict = { + self.dev1_id: True, + self.dev2_id: False + } + + self.assertDictEqual(expected_out_dict, out_dict) + + def test_exists_dir_trailing_slash(self): + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[self.dir1], files=[]) + out_dict = {self.dev1_id: linfo, self.dev2_id: LsInfo([], [])} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.exists(self.target, self.dir1.path + '/') + + expected_out_dict = { + self.dev1_id: True, + self.dev2_id: False + } + + self.assertDictEqual(expected_out_dict, out_dict) + + def test_exists_errinfo(self): + + def mock_list_files(*args, **kwargs): + linfo = LsInfo(directories=[], files=[self.file1]) + out_dict = {self.dev1_id: linfo, self.dev2_id: self.errinfo} + return out_dict + + with mock.patch.object(self.fss_api, 'list_files', side_effect=mock_list_files) as m: + out_dict = self.fss_api.exists(self.target, self.file1.path) + + expected_out_dict = { + self.dev1_id: True, + self.dev2_id: self.errinfo + } + + self.assertDictEqual(expected_out_dict, out_dict) if __name__ == '__main__': unittest.main() From fdf3ad433a8c39405f59ea3f41bb9ef241be3553 Mon Sep 17 00:00:00 2001 From: Zach Varberg Date: Tue, 28 Jul 2015 13:32:12 -0500 Subject: [PATCH 068/140] Support sending multiple file system commands with a single web service request This refactors the way commands are built for all file system related commands and allows a simpler and more useful way to build complex commands that include multiple file system commands within a single request to reduce the number of require web service requests. Signed-off-by: Zach Varberg --- .../file_system_service_playground.py | 8 +- devicecloud/file_system_service.py | 448 +++++++++++++++--- .../test/unit/test_file_system_service.py | 240 +++++++++- docs/conf.py | 5 + 4 files changed, 631 insertions(+), 70 deletions(-) diff --git a/devicecloud/examples/file_system_service_playground.py b/devicecloud/examples/file_system_service_playground.py index 1c07ba4..51aa1b3 100644 --- a/devicecloud/examples/file_system_service_playground.py +++ b/devicecloud/examples/file_system_service_playground.py @@ -1,8 +1,8 @@ import time import six from devicecloud.examples.example_helpers import get_authenticated_dc +from devicecloud.file_system_service import ErrorInfo, FileSystemServiceCommandBlock, LsCommand, PutCommand from devicecloud.sci import DeviceTarget -from devicecloud.file_system_service import FileSystemServiceAPI, ErrorInfo def put_test_file(fssapi, target, tmp_file_path): @@ -106,3 +106,9 @@ def use_filesystem(dc, target, base_dir): time.sleep(5) get_modified_files(fssapi, target, base_dir, cutoff_time) delete_test_file(fssapi, target, tmp_file_path) + + command_block = FileSystemServiceCommandBlock() + command_block.add_command(LsCommand(base_dir)) + command_block.add_command(LsCommand('/another/directory/on/your/device')) + + print(fssapi.send_command_block(target, command_block)) diff --git a/devicecloud/file_system_service.py b/devicecloud/file_system_service.py index 273be85..fd11bc9 100644 --- a/devicecloud/file_system_service.py +++ b/devicecloud/file_system_service.py @@ -208,9 +208,367 @@ def __eq__(self, other): self.last_modified == other.last_modified) +class FileSystemServiceCommandBlock(object): + + def __init__(self): + self.commands_node = ET.Element('commands') + + def add_command(self, command): + """Add a :class:`~.FileSystemServiceCommandABC` to this command block + + :param command: The command to include in this command block + :type command: :class:`~.FileSystemServiceCommandABC` + """ + self.commands_node.append(command.get_etree()) + + def get_command_string(self): + """Return this command block represented as a string + + :return: The string representation of this command block + """ + return ET.tostring(self.commands_node) + + def get_etree(self): + """Return the :class:`xml.etree.ElementTree.Element` representing the XML of this command block + + :return: The command block + :rtype: :class:`xml.etree.ElementTree.Element` + """ + return self.commands_node + + +class FileSystemServiceCommandABC(object): + """Abstract base class for FileSystemServiceCommands + + :cvar command_name: The file_system command name of this command + """ + + command_name = 'abstract' + + def get_etree(self): + """Return an :class:`xml.etree.ElementTree.Element` that is the root of the command XML for this command""" + raise NotImplementedError() + + @classmethod + def parse_response(cls, response, **kwargs): + """Parse the server response for this command type""" + raise NotImplementedError + + +class LsCommand(FileSystemServiceCommandABC): + """Class representing the ls command to list files on a device + + :param path: the path to the device directory to list + :param hash: the method to generate the file hashes + + :cvar command_name: The file_system command name of this command ("ls") + """ + + command_name = "ls" + + def __init__(self, path, hash='any'): + self.etree = ET.Element(self.command_name) + self.etree.set('path', path) + self.etree.set('hash', hash) + + def get_etree(self): + """Get the XML representation of this command + + :return: the :class:`xml.etree.ElementTree.Element` that is the root of the XML of the ls command + """ + return self.etree + + @classmethod + def parse_response(cls, response, device_id=None, fssapi=None, **kwargs): + """Parse the server response for this ls command + + This will parse xml of the following form + :: + + + ... + + ... + + + or with an error + :: + + + + + :param response: The XML root of the response for an ls command + :type response: :class:`xml.etree.ElementTree.Element` + :param device_id: The device id of the device this ls response came from + :param fssapi: A reference to a :class:`~FileSystemServiceAPI` for use with the + :class:`~FileInfo` and :class:`~DirectoryInfo` objects for future commands + :return: An :class:`~LsInfo` object containing the list of directories and files on + the device or an :class:`~ErrorInfo` if the xml contained an error + """ + if response.tag != cls.command_name: + raise ResponseParseError( + "Received response of type {}, LsCommand can only parse responses of type {}".format(response.tag, + cls.command_name)) + + if fssapi is None: + raise FileSystemServiceException("fssapi is required to parse an LsCommand response") + if device_id is None: + raise FileSystemServiceException("device_id is required to parse an LsCommand response") + + error = response.find('./error') + if error is not None: + return _parse_error_tree(error) + + hash_type = response.get('hash') + dirs = [] + files = [] + + # Get each file listed in this response + for myfile in response.findall('./file'): + fi = FileInfo(fssapi, + device_id, + myfile.get('path'), + int(myfile.get('last_modified')), + int(myfile.get('size')), + myfile.get('hash'), + hash_type) + files.append(fi) + # Get each directory listed for this device + for mydir in response.findall('./dir'): + di = DirectoryInfo(fssapi, + device_id, + mydir.get('path'), + int(mydir.get('last_modified'))) + dirs.append(di) + return LsInfo(directories=dirs, files=files) + + +class GetCommand(FileSystemServiceCommandABC): + """Class representing the get file command to get the contents of a file on a device + + :param path: the path to the file on the device to get the contents of + :param offset: the point in the file to start reading data from (if None, start at the beginning) + :param length: the length of data to retrieve (if None, read until the end) + + :cvar command_name: The file_system command name of this command ("get_file") + """ + command_name = 'get_file' + + def __init__(self, path, offset=None, length=None): + get_file_el = ET.Element(self.command_name) + get_file_el.set('path', path) + if offset is not None: + get_file_el.set('offset', str(offset)) + if length is not None: + get_file_el.set('length', str(length)) + + self.etree = get_file_el + + def get_etree(self): + """Get the XML representation of this get file command + + :return: the :class:`xml.etree.ElementTree.Element` that is the root of the XML of the get_file command + """ + return self.etree + + @classmethod + def parse_response(cls, response, **kwargs): + """Parse the server response for this get file command + + This will parse xml of the following form + :: + + + asdfasdfasdfasdfasf + + + + or with an error + :: + + + + + :param response: The XML root of the response for a get file command + :type response: :class:`xml.etree.ElementTree.Element` + :return: a six.binary_type string of the data of a file or an :class:`~ErrorInfo` if the xml contained an error + """ + if response.tag != cls.command_name: + raise ResponseParseError( + "Received response of type {}, GetCommand can only parse responses of type {}".format(response.tag, + cls.command_name)) + + error = response.find('./error') + if error is not None: + return _parse_error_tree(error) + + data = six.b(response.find('./data').text) + return base64.b64decode(data) + + +class PutCommand(FileSystemServiceCommandABC): + """Class representing the put file command to write contents to a file on a device + + :param path: The path on the target to the file to write to. If the file already exists it will be overwritten. + :param file_data: A `six.binary_type` containing the data to put into the file + :param server_file: The path to a file on the devicecloud server containing the data to put into the file on the + device + :param offset: Start writing bytes to the file at this position, if None start at the beginning + :param truncate: Boolean, if True after bytes are done being written end the file their even if previous data + exists beyond it. If False, leave any existing data in place. + + :cvar command_name: The file_system command name of this command ("put_file") + """ + command_name = 'put_file' + + def __init__(self, path, file_data=None, server_file=None, offset=None, truncate=False): + if file_data is not None and server_file is not None: + raise FileSystemServiceException("Can only specify one of server_file or file_data") + + put_file_el = ET.Element(self.command_name) + put_file_el.set('path', path) + put_file_el.set('truncate', 'true' if truncate else 'false') + + if offset is not None: + put_file_el.set('offset', str(offset)) + + if file_data is not None: + if not isinstance(file_data, six.binary_type): + raise TypeError("file_data must be of type {}".format(six.binary_type)) + data_el = ET.SubElement(put_file_el, 'data') + data_el.text = base64.b64encode(file_data).decode('ascii') + elif server_file is not None: + file_el = ET.SubElement(put_file_el, 'file') + file_el.text = server_file + else: + raise FileSystemServiceException("You must specify either file_data or server_file to put data into a file") + + self.etree = put_file_el + + def get_etree(self): + """Get the XML representation of this put file command + + :return: the :class:`xml.etree.ElementTree.Element` that is the root of the XML of the put file command + """ + return self.etree + + @classmethod + def parse_response(cls, response, **kwargs): + """Parse the server response for this put file command + + This will parse xml of the following form + :: + + + or with an error + :: + + + + + :param response: The XML root of the response for a put file command + :type response: :class:`xml.etree.ElementTree.Element` + :return: None if everything was ok or an :class:`~ErrorInfo` if the xml contained an error + """ + if response.tag != cls.command_name: + raise ResponseParseError( + "Received response of type {}, PutCommand can only parse responses of type {}".format(response.tag, + cls.command_name)) + error = response.find('./error') + if error is not None: + return _parse_error_tree(error) + + return None + + +class DeleteCommand(FileSystemServiceCommandABC): + """Class representing the delete file command on a device + + :param path: The path on the target to the file to delete. + + :cvar command_name: The file_system command name of this command ("rm") + """ + command_name = 'rm' + + def __init__(self, path): + rm_el = ET.Element(self.command_name) + rm_el.set('path', path) + self.etree = rm_el + + def get_etree(self): + """Get the XML representation of this delete file command + + :return: the :class:`xml.etree.ElementTree.Element` that is the root of the XML of the delete file command + """ + return self.etree + + @classmethod + def parse_response(cls, response, **kwargs): + """Parse the server response for this put file command + + This will parse xml of the following form + :: + + + or with an error + :: + + + + + :param response: The XML root of the response for a delete file command + :type response: :class:`xml.etree.ElementTree.Element` + :return: None if everything was ok or an :class:`~ErrorInfo` if the xml contained an error + """ + if response.tag != cls.command_name: + raise ResponseParseError( + "Received response of type {}, DeleteCommand can only parse responses of type {}".format(response.tag, + cls.command_name)) + error = response.find('./error') + if error is not None: + return _parse_error_tree(error) + return None + + +FILE_SYSTEM_COMMANDS = [LsCommand, GetCommand, PutCommand, DeleteCommand] + + class FileSystemServiceAPI(SCIAPIBase): """ Encapsulate the File System Service API """ + def send_command_block(self, target, command_block): + """Send an arbitrary file system command block + + The primary use for this method is to send multiple file system commands with a single + web service request. This can help to avoid throttling. + + :param target: The device(s) to be targeted with this request + :type target: :class:`devicecloud.sci.TargetABC` or list of :class:`devicecloud.sci.TargetABC` instances + :param command_block: The block of commands to execute on the target + :type command_block: :class:`~FileSystemServiceCommandBlock` + :return: The response will be a dictionary where the keys are device_ids and the values are + the parsed responses of each command sent in the order listed in the command response for + that device. In practice it seems to be the same order as the commands were sent in, however, + the device cloud documentation does not explicitly state anywhere that is the case so I cannot + guarantee it. This does mean that if you send different types of commands the response list + will be different types. Please see the commands parse_response functions for what those types + will be. (:meth:`LsCommand.parse_response`, :class:`GetCommand.parse_response`, + :class:`PutCommand.parse_response`, :class:`DeleteCommand.parse_response`) + """ + root = _parse_command_response( + self._sci_api.send_sci("file_system", target, command_block.get_command_string())) + + out_dict = {} + for device in root.findall('./file_system/device'): + device_id = device.get('id') + results = [] + for command in device.find('./commands'): + for command_class in FILE_SYSTEM_COMMANDS: + if command_class.command_name == command.tag.lower(): + results.append(command_class.parse_response(command, fssapi=self, device_id=device_id)) + out_dict[device_id] = results + return out_dict + def list_files(self, target, path, hash='any'): """List all files and directories in the path on the target @@ -253,12 +611,10 @@ def list_files(self, target, path, hash='any'): dinfo.path, dinfo.last_modified, device_id) """ - commands_el = ET.Element('commands') - ls_el = ET.SubElement(commands_el, 'ls') - ls_el.set('path', path) - ls_el.set('hash', hash) + command_block = FileSystemServiceCommandBlock() + command_block.add_command(LsCommand(path, hash=hash)) root = _parse_command_response( - self._sci_api.send_sci("file_system", target, ET.tostring(commands_el))) + self._sci_api.send_sci("file_system", target, command_block.get_command_string())) out_dict = {} @@ -292,31 +648,12 @@ def list_files(self, target, path, hash='any'): # Here we will get each of the XML trees rooted at the device nodes for device in root.findall('./file_system/device'): device_id = device.get('id') - error = device.find('.//error') + error = device.find('./error') if error is not None: out_dict[device_id] = _parse_error_tree(error) else: - hash_type = device.find('./commands/ls').get('hash') - dirs = [] - files = [] - # Get each file listed for this device - for myfile in device.findall('./commands/ls/file'): - fi = FileInfo(self, - device_id, - myfile.get('path'), - int(myfile.get('last_modified')), - int(myfile.get('size')), - myfile.get('hash'), - hash_type) - files.append(fi) - # Get each directory listed for this device - for mydir in device.findall('./commands/ls/dir'): - di = DirectoryInfo(self, - device_id, - mydir.get('path'), - int(mydir.get('last_modified'))) - dirs.append(di) - out_dict[device_id] = LsInfo(directories=dirs, files=files) + linfo = LsCommand.parse_response(device.find('./commands/ls'), device_id=device_id, fssapi=self) + out_dict[device_id] = linfo return out_dict def get_file(self, target, path, offset=None, length=None): @@ -331,25 +668,19 @@ def get_file(self, target, path, offset=None, length=None): and/or length are specified) or an :class:`~.ErrorInfo` object if there was an error response :raises: :class:`~.ResponseParseError` If the SCI response has unrecognized formatting """ - commands_el = ET.Element('commands') - get_file_el = ET.SubElement(commands_el, 'get_file') - get_file_el.set('path', path) - if offset is not None: - get_file_el.set('offset', str(offset)) - if length is not None: - get_file_el.set('length', str(length)) - + command_block = FileSystemServiceCommandBlock() + command_block.add_command(GetCommand(path, offset, length)) root = _parse_command_response( - self._sci_api.send_sci("file_system", target, ET.tostring(commands_el))) + self._sci_api.send_sci("file_system", target, command_block.get_command_string())) out_dict = {} for device in root.findall('./file_system/device'): device_id = device.get('id') - error = device.find('.//error') + error = device.find('./error') if error is not None: out_dict[device_id] = _parse_error_tree(error) else: - data = six.b(device.find('./commands/get_file/data').text) - out_dict[device_id] = base64.b64decode(data) + data = GetCommand.parse_response(device.find('./commands/get_file')) + out_dict[device_id] = data return out_dict def put_file(self, target, path, file_data=None, server_file=None, offset=None, truncate=False): @@ -370,37 +701,19 @@ def put_file(self, target, path, file_data=None, server_file=None, offset=None, neither are specified :raises: :class:`~.ResponseParseError` If the SCI response has unrecognized formatting """ - if file_data is not None and server_file is not None: - raise FileSystemServiceException("Can only specify one of server_file or file_data") - commands_el = ET.Element('commands') - put_file_el = ET.SubElement(commands_el, 'put_file') - put_file_el.set('path', path) - put_file_el.set('truncate', 'true' if truncate else 'false') - - if offset is not None: - put_file_el.set('offset', str(offset)) - - if file_data is not None: - if not isinstance(file_data, six.binary_type): - raise TypeError("file_data must be of type {}".format(six.binary_type)) - data_el = ET.SubElement(put_file_el, 'data') - data_el.text = base64.b64encode(file_data).decode('ascii') - elif server_file is not None: - file_el = ET.SubElement(put_file_el, 'file') - file_el.text = server_file - else: - raise FileSystemServiceException("You must specify either file_data or server_file to put data into a file") + command_block = FileSystemServiceCommandBlock() + command_block.add_command(PutCommand(path, file_data, server_file, offset, truncate)) - root = _parse_command_response(self._sci_api.send_sci("file_system", target, ET.tostring(commands_el))) + root = _parse_command_response(self._sci_api.send_sci("file_system", target, command_block.get_command_string())) out_dict = {} for device in root.findall('./file_system/device'): device_id = device.get('id') - error = device.find('.//error') + error = device.find('./error') if error is not None: out_dict[device_id] = _parse_error_tree(error) else: - out_dict[device_id] = None + out_dict[device_id] = PutCommand.parse_response(device.find('./commands/put_file')) return out_dict @@ -414,19 +727,18 @@ def delete_file(self, target, path): if the operation failed on that device :raises: :class:`~.ResponseParseError` If the SCI response has unrecognized formatting """ - commands_el = ET.Element('commands') - rm_el = ET.SubElement(commands_el, 'rm') - rm_el.set('path', path) - root = _parse_command_response(self._sci_api.send_sci("file_system", target, ET.tostring(commands_el))) + command_block = FileSystemServiceCommandBlock() + command_block.add_command(DeleteCommand(path)) + root = _parse_command_response(self._sci_api.send_sci("file_system", target, command_block.get_command_string())) out_dict = {} for device in root.findall('./file_system/device'): device_id = device.get('id') - error = device.find('.//error') + error = device.find('./error') if error is not None: out_dict[device_id] = _parse_error_tree(error) else: - out_dict[device_id] = None + out_dict[device_id] = DeleteCommand.parse_response(device.find('./commands/rm')) return out_dict def get_modified_items(self, target, path, last_modified_cutoff): diff --git a/devicecloud/test/unit/test_file_system_service.py b/devicecloud/test/unit/test_file_system_service.py index b1cb911..68808ab 100644 --- a/devicecloud/test/unit/test_file_system_service.py +++ b/devicecloud/test/unit/test_file_system_service.py @@ -4,7 +4,8 @@ from devicecloud.file_system_service import FileInfo, DirectoryInfo, FileSystemServiceException, \ _parse_command_response, ResponseParseError, \ - ErrorInfo, LsInfo, _parse_error_tree + ErrorInfo, LsInfo, _parse_error_tree, LsCommand, GetCommand, PutCommand, DeleteCommand, \ + FileSystemServiceCommandBlock from devicecloud.sci import AllTarget from devicecloud.test.unit.test_utilities import HttpTestBase import mock @@ -171,6 +172,180 @@ def test_list_contents(self): self.assertEqual(call_args[1], '/a/path/dir1') +class TestLsCommand(unittest.TestCase): + def test_init(self): + lscommand = LsCommand(path='/a/path/here') + et = lscommand.get_etree() + self.assertEqual(et.tag, 'ls') + self.assertEqual('/a/path/here', et.get('path')) + self.assertEqual('any', et.get('hash')) + self.assertEqual(0, len(list(et))) + self.assertEqual(None, et.text) + + def test_parse(self): + fss_api = mock.Mock() + dev_id = 'my_dev_id' + file1 = FileInfo(fss_api, dev_id, '/a/path/file1.txt', 1436276773, 7989, + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + file2 = FileInfo(fss_api, dev_id, '/a/path/file2.py', 1434377919, 181, + "DEA17715739E46079C1A6DDCB38344DF", 'md5') + dir1 = DirectoryInfo(fss_api, dev_id, '/a/path/dir', 1436203917) + linfo = LsCommand.parse_response(ET.fromstring(LS_BLOCK), device_id=dev_id, fssapi=fss_api) + self.assertEqual(linfo, LsInfo(directories=[dir1], files=[file1, file2])) + + def test_parse_error(self): + fss_api = mock.Mock() + dev_id = 'my_dev_id' + errinfo = LsCommand.parse_response( + ET.fromstring(ERROR_BLOCK.format(command='ls', errno=1, errtext="error text")), + device_id=dev_id, fssapi=fss_api) + + self.assertEqual(errinfo.errno, 1) + self.assertEqual(errinfo.message, "error text") + + def test_parse_wrong_response(self): + self.assertRaises(ResponseParseError, LsCommand.parse_response, ET.fromstring('')) + + +class TestGetCommand(unittest.TestCase): + def test_init_defaults(self): + getcommand = GetCommand(path='/a/path/here') + et = getcommand.get_etree() + self.assertEqual(et.tag, 'get_file') + self.assertEqual('/a/path/here', et.get('path')) + self.assertEqual(None, et.get('offset', None)) + self.assertEqual(None, et.get('length', None)) + self.assertEqual(0, len(list(et))) + self.assertEqual(None, et.text) + + def test_init_values(self): + getcommand = GetCommand(path='/a/path/here', offset=5, length=10) + et = getcommand.get_etree() + self.assertEqual(et.tag, 'get_file') + self.assertEqual('/a/path/here', et.get('path')) + self.assertEqual('5', et.get('offset', None)) + self.assertEqual('10', et.get('length', None)) + self.assertEqual(0, len(list(et))) + self.assertEqual(None, et.text) + + def test_parse(self): + data_str = base64.b64encode(six.b("File Data")).decode('ascii') + data = GetCommand.parse_response(ET.fromstring(GET_FILE_BLOCK.format(data=data_str))) + self.assertEqual(six.b("File Data"), data) + + def test_parse_error(self): + errinfo = GetCommand.parse_response( + ET.fromstring(ERROR_BLOCK.format(command='get_file', errno=1, errtext="error text"))) + + self.assertEqual(errinfo.errno, 1) + self.assertEqual(errinfo.message, "error text") + + def test_parse_wrong_response(self): + self.assertRaises(ResponseParseError, GetCommand.parse_response, ET.fromstring('')) + + +class TestPutCommand(unittest.TestCase): + def test_init_defaults(self): + putcommand = PutCommand(path='/a/path/here', file_data=six.b("some file data")) + et = putcommand.get_etree() + self.assertEqual(et.tag, 'put_file') + self.assertEqual('/a/path/here', et.get('path')) + self.assertEqual(None, et.get('offset', None)) + self.assertEqual('false', et.get('truncate')) + self.assertEqual(1, len(list(et))) + data = et.find('./data') + self.assertEqual(base64.b64encode(six.b("some file data")), six.b(data.text)) + + def test_init_server_file(self): + putcommand = PutCommand(path='/a/path/here', server_file='/a/file/on/server') + et = putcommand.get_etree() + self.assertEqual(et.tag, 'put_file') + self.assertEqual('/a/path/here', et.get('path')) + self.assertEqual(None, et.get('offset', None)) + self.assertEqual('false', et.get('truncate')) + self.assertEqual(1, len(list(et))) + server_file = et.find('./file') + self.assertEqual('/a/file/on/server', server_file.text) + + def test_init_values(self): + putcommand = PutCommand(path='/a/path/here', file_data=six.b("some file data"), offset=5, truncate=True) + et = putcommand.get_etree() + self.assertEqual(et.tag, 'put_file') + self.assertEqual('/a/path/here', et.get('path')) + self.assertEqual('5', et.get('offset', None)) + self.assertEqual('true', et.get('truncate')) + self.assertEqual(1, len(list(et))) + data = et.find('./data') + self.assertEqual(base64.b64encode(six.b("some file data")), six.b(data.text)) + + def test_put_command_no_data(self): + self.assertRaises(FileSystemServiceException, PutCommand, path='/a/path/here') + + def test_put_command_both_data(self): + self.assertRaises(FileSystemServiceException, PutCommand, path='/a/path/here', + file_data=six.b("some file data"), server_file='/a/file/on/server') + + def test_parse(self): + self.assertIsNone(PutCommand.parse_response(ET.fromstring(''))) + + def test_parse_error(self): + errinfo = PutCommand.parse_response( + ET.fromstring(ERROR_BLOCK.format(command='put_file', errno=1, errtext="error text"))) + + self.assertEqual(errinfo.errno, 1) + self.assertEqual(errinfo.message, "error text") + + def test_parse_wrong_response(self): + self.assertRaises(ResponseParseError, PutCommand.parse_response, ET.fromstring('')) + + +class TestDeleteCommand(unittest.TestCase): + def test_init(self): + delcommand = DeleteCommand(path='/a/path/here') + et = delcommand.get_etree() + self.assertEqual(et.tag, 'rm') + self.assertEqual('/a/path/here', et.get('path')) + self.assertEqual(0, len(list(et))) + self.assertEqual(None, et.text) + + def test_parse(self): + self.assertIsNone(DeleteCommand.parse_response(ET.fromstring(''))) + + def test_parse_error(self): + errinfo = DeleteCommand.parse_response( + ET.fromstring(ERROR_BLOCK.format(command='rm', errno=1, errtext="error text"))) + + self.assertEqual(errinfo.errno, 1) + self.assertEqual(errinfo.message, "error text") + + def test_parse_wrong_response(self): + self.assertRaises(ResponseParseError, DeleteCommand.parse_response, ET.fromstring('')) + + +class TestCommandBlock(unittest.TestCase): + def test_init(self): + command_block = FileSystemServiceCommandBlock() + et = command_block.get_etree() + self.assertEqual(et.tag, 'commands') + self.assertEqual(0, len(list(et))) + self.assertEqual(0, len(et.keys())) + + def test_add_command(self): + command_block = FileSystemServiceCommandBlock() + command_block.add_command(DeleteCommand(path='/a/path')) + et = command_block.get_etree() + self.assertEqual(et.tag, 'commands') + self.assertEqual(1, len(list(et))) + self.assertEqual(0, len(et.keys())) + self.assertIsNotNone(et.find('./{}'.format(DeleteCommand.command_name))) + + def test_get_command_string(self): + command_block = FileSystemServiceCommandBlock() + self.assertEqual(six.b(''), command_block.get_command_string()) + command_block.add_command(DeleteCommand(path='/a/path')) + self.assertEqual(six.b(''), command_block.get_command_string()) + + class TestFileSystemServiceAPI(HttpTestBase): def setUp(self): HttpTestBase.setUp(self) @@ -606,5 +781,68 @@ def mock_list_files(*args, **kwargs): self.assertDictEqual(expected_out_dict, out_dict) + def test_send_command_block_all_ok(self): + command_block = FileSystemServiceCommandBlock() + command_block.add_command(DeleteCommand(path='/a/path/1.txt')) + command_block.add_command(DeleteCommand(path='/a/path/2.txt')) + + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, (GENERIC_COMMAND_BLOCK.format(command='rm') + GENERIC_COMMAND_BLOCK.format(command='rm'))) + fsr.add_device_block(self.dev2_id, (GENERIC_COMMAND_BLOCK.format(command='rm') + GENERIC_COMMAND_BLOCK.format(command='rm'))) + self.prep_sci_response(fsr) + + out_dict = self.fss_api.send_command_block(self.target, command_block) + + # Should just be Nones because they are delete commands with no errors + expected_dict = { + self.dev1_id: [None, None], + self.dev2_id: [None, None], + } + + self.assertDictEqual(expected_dict, out_dict) + + def test_send_command_block_mixed_commands(self): + command_block = FileSystemServiceCommandBlock() + command_block.add_command(DeleteCommand(path='/a/path/1.txt')) + command_block.add_command(GetCommand(path='/a/path/2.txt')) + + data_string = base64.b64encode(six.b('testing string')).decode('ascii') + + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, (GENERIC_COMMAND_BLOCK.format(command='rm') + GET_FILE_BLOCK.format(data=data_string))) + fsr.add_device_block(self.dev2_id, (GENERIC_COMMAND_BLOCK.format(command='rm') + GET_FILE_BLOCK.format(data=data_string))) + self.prep_sci_response(fsr) + + out_dict = self.fss_api.send_command_block(self.target, command_block) + + expected_dict = { + self.dev1_id: [None, six.b('testing string')], + self.dev2_id: [None, six.b('testing string')], + } + + self.assertDictEqual(expected_dict, out_dict) + + def test_send_command_block_errors(self): + command_block = FileSystemServiceCommandBlock() + command_block.add_command(DeleteCommand(path='/a/path/1.txt')) + command_block.add_command(GetCommand(path='/a/path/2.txt')) + + fsr = FileSystemResponse() + fsr.add_device_block(self.dev1_id, (GENERIC_COMMAND_BLOCK.format(command='rm') + ERROR_BLOCK.format(command='get_file', errno=1, errtext="an error message"))) + fsr.add_device_block(self.dev2_id, (GENERIC_COMMAND_BLOCK.format(command='rm') + ERROR_BLOCK.format(command='get_file', errno=1, errtext="an error message"))) + self.prep_sci_response(fsr) + + out_dict = self.fss_api.send_command_block(self.target, command_block) + + # Check empty DeleteCommand responses + self.assertIsNone(out_dict[self.dev1_id][0]) + self.assertIsNone(out_dict[self.dev2_id][0]) + + # Check error info values + self.assertEqual(1, out_dict[self.dev1_id][1].errno) + self.assertEqual(1, out_dict[self.dev2_id][1].errno) + self.assertEqual("an error message", out_dict[self.dev1_id][1].message) + self.assertEqual("an error message", out_dict[self.dev2_id][1].message) + if __name__ == '__main__': unittest.main() diff --git a/docs/conf.py b/docs/conf.py index a08b48c..0669534 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -32,8 +32,13 @@ # ones. extensions = [ 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx' ] +intersphinx_mapping = { + 'python': ('http://docs.python.org/3', None), +} + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] From 0b9321130a272bcae16e405c177a5e555617e248 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 30 Sep 2015 23:21:34 -0500 Subject: [PATCH 069/140] docs: use build status badge directly from travis The shields.io badge looks the same but tends to get out of sync and cannot be tied to the master branch (which is the build status we really care about). Signed-off-by: Paul Osborne --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6a9d7bf..2f146a1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Python Device Cloud Library =========================== -[![Build Status](https://img.shields.io/travis/digidotcom/python-devicecloud.svg)](https://travis-ci.org/digidotcom/python-devicecloud) +[![Build Status](https://travis-ci.org/digidotcom/python-devicecloud.svg?branch=master)](https://travis-ci.org/digidotcom/python-devicecloud) [![Coverage Status](https://img.shields.io/coveralls/digidotcom/python-devicecloud.svg)](https://coveralls.io/r/digidotcom/python-devicecloud) [![Code Climate](https://img.shields.io/codeclimate/github/digidotcom/python-devicecloud.svg)](https://codeclimate.com/github/digidotcom/python-devicecloud) [![Latest Version](https://img.shields.io/pypi/v/devicecloud.svg)](https://pypi.python.org/pypi/devicecloud/) From ee4b2a07be591b8b788962a3971316171c0c1c88 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 30 Sep 2015 23:44:20 -0500 Subject: [PATCH 070/140] travis: explicitly depend on coverage 3.7.1 This dependency is included in python-coveralls but not installed for some reason. Signed-off-by: Paul Osborne --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2187929..93045a2 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ commands=nosetests -m '^(int|unit)?[Tt]est' [testenv:coverage] deps= {[testenv]deps} - coverage + coverage==3.7.1 python-coveralls commands = coverage run --branch --omit={envdir}/* {envbindir}/nosetests From 743689ec6e2aa8c763bd97698aab9d97bb2a6823 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 30 Sep 2015 23:53:40 -0500 Subject: [PATCH 071/140] travis: switch to using container based infrastructure This should makes builds start and complete a bit faster than was previously the case. Signed-off-by: Paul Osborne --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6d1d6be..4d73613 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,3 +19,7 @@ install: # command to run tests, e.g. python setup.py test script: - tox -e $TOX_ENV + +# allow travis to use new, faster container based +# infrastructure to perform the testing +sudo: false From 39f4da87f6336ca3a8d3f4bcef69d13d4e4c839e Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Thu, 1 Oct 2015 00:04:37 -0500 Subject: [PATCH 072/140] release: add changelog for v0.4 Signed-off-by: Paul Osborne --- CHANGELOG.md | 16 ++++++++++++++++ devicecloud/version.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50eb84a..4ad2a43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ ## Python Devicecloud Library Changelog +### 0.4 / 2015-10-01 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.3...0.4) + +Enhancements: + +* monitors: basic support for creating HTTP monitors was added +* streams: support for the JSON data type added +* sci: support added for filesystem service added. This allows you to + access files and directories on any device supporting this service. + +Bug Fixes: + +* streams: fix data translations from device cloud <-> python types + when reading and writing data points. See + https://github.com/digidotcom/python-devicecloud/commit/5ac6c15cddf010709361c16b69e622aca93d6b28 + ### 0.3 / 2015-06-15 [Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.2...0.3) diff --git a/devicecloud/version.py b/devicecloud/version.py index f9695f4..4db066e 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -4,4 +4,4 @@ # # Copyright (c) 2015 Digi International, Inc. -__version__ = "0.3" +__version__ = "0.4" From 02191fc54686f69042f7c8d64dd34796ab9e7cb5 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Thu, 1 Oct 2015 00:08:38 -0500 Subject: [PATCH 073/140] docs: miscellaneous minor documentation fixes Signed-off-by: Paul Osborne --- devicecloud/file_system_service.py | 46 +++++++++++++++--------------- docs/conf.py | 6 ++-- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/devicecloud/file_system_service.py b/devicecloud/file_system_service.py index fd11bc9..4be4217 100644 --- a/devicecloud/file_system_service.py +++ b/devicecloud/file_system_service.py @@ -177,8 +177,8 @@ def list_contents(self): :return: A LsInfo object that contains directories and files :rtype: :class:`~.LsInfo` or :class:`~.ErrorInfo` - Here is an example usage - :: + Here is an example usage:: + # let dirinfo be a DirectoryInfo object ldata = dirinfo.list_contents() if isinstance(ldata, ErrorInfo): @@ -282,8 +282,8 @@ def get_etree(self): def parse_response(cls, response, device_id=None, fssapi=None, **kwargs): """Parse the server response for this ls command - This will parse xml of the following form - :: + This will parse xml of the following form:: + ... @@ -291,8 +291,8 @@ def parse_response(cls, response, device_id=None, fssapi=None, **kwargs): ... - or with an error - :: + or with an error:: + @@ -375,16 +375,16 @@ def get_etree(self): def parse_response(cls, response, **kwargs): """Parse the server response for this get file command - This will parse xml of the following form - :: + This will parse xml of the following form:: + asdfasdfasdfasdfasf - or with an error - :: + or with an error:: + @@ -456,12 +456,12 @@ def get_etree(self): def parse_response(cls, response, **kwargs): """Parse the server response for this put file command - This will parse xml of the following form - :: + This will parse xml of the following form:: + - or with an error - :: + or with an error:: + @@ -506,15 +506,15 @@ def get_etree(self): def parse_response(cls, response, **kwargs): """Parse the server response for this put file command - This will parse xml of the following form - :: + This will parse xml of the following form:: + - or with an error - :: - - - + or with an error:: + + + + :param response: The XML root of the response for a delete file command :type response: :class:`xml.etree.ElementTree.Element` @@ -581,8 +581,8 @@ def list_files(self, target, path, hash='any'): directories or an :class:`~.ErrorInfo` object if there was an error response :raises: :class:`~.ResponseParseError` If the SCI response has unrecognized formatting - Here is an example usage - :: + Here is an example usage:: + # dc is a DeviceCloud instance fssapi = dc.get_fss_api() diff --git a/docs/conf.py b/docs/conf.py index 0669534..18d5e62 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,6 +22,8 @@ this_dir = os.path.dirname(__file__) sys.path.insert(0, os.path.abspath(os.path.join(this_dir, ".."))) +import devicecloud + # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. @@ -60,9 +62,9 @@ # built documents. # # The short X.Y version. -version = '0.1' +version = devicecloud.__version__ # The full version, including alpha/beta/rc tags. -release = '0.1' +release = devicecloud.__version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 30bab3c0f5ee697f73f2f4b39eea5e9daecc13f0 Mon Sep 17 00:00:00 2001 From: Francois Deppierraz Date: Fri, 13 Nov 2015 17:12:34 +0100 Subject: [PATCH 074/140] Fix XML request formatting for group targets Without this patch, the following error message was received from the Device Cloud endpoint because of incorrect XML formatting. Traceback (most recent call last): File "./digicloud.py", line 137, in stats(get_targets()) File "./digicloud.py", line 43, in stats req = dc.sci.send_sci(operation='send_message', target=target, payload=STATS_PAYLOAD) File "/usr/local/lib/python2.7/dist-packages/devicecloud/sci.py", line 220, in send_sci return self._conn.post("/ws/sci", full_request) File "/usr/local/lib/python2.7/dist-packages/devicecloud/__init__.py", line 287, in post return self._make_request("POST", url, data=data, **kwargs) File "/usr/local/lib/python2.7/dist-packages/devicecloud/__init__.py", line 178, in _make_request raise DeviceCloudHttpException(response, err) devicecloud.DeviceCloudHttpException: HTTP Status 400: Failure to parse SCI request Error on line 1 of document : The element type "group" must be terminated by the matching end-tag "</group>". Nested exception: The element type "group" must be terminated by the matching end-tag "</group>". --- devicecloud/sci.py | 2 +- devicecloud/test/unit/test_sci.py | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/devicecloud/sci.py b/devicecloud/sci.py index a216754..7b5bc62 100644 --- a/devicecloud/sci.py +++ b/devicecloud/sci.py @@ -63,7 +63,7 @@ def __init__(self, group): self._group = group def to_xml(self): - return ''.format(self._group) + return ''.format(self._group) class AsyncRequestProxy(object): diff --git a/devicecloud/test/unit/test_sci.py b/devicecloud/test/unit/test_sci.py index d275caa..1c05c08 100644 --- a/devicecloud/test/unit/test_sci.py +++ b/devicecloud/test/unit/test_sci.py @@ -11,7 +11,7 @@ import six from devicecloud import DeviceCloud -from devicecloud.sci import DeviceTarget, AsyncRequestProxy, ServerCommandInterfaceAPI +from devicecloud.sci import DeviceTarget, GroupTarget, AsyncRequestProxy, ServerCommandInterfaceAPI from devicecloud.test.unit.test_utilities import HttpTestBase @@ -77,6 +77,21 @@ def test_sci_successful_error(self): '' '')) + def test_sci_successful_group_target(self): + self._prepare_sci_response(EXAMPLE_SCI_DEVICE_NOT_CONNECTED) + self.dc.get_sci_api().send_sci( + operation="send_message", + target=GroupTarget("TestGroup"), + payload="") + self.assertEqual(httpretty.last_request().body, + six.b('' + '' + '' + '' + '' + '' + '' + '')) class TestGetAsync(HttpTestBase): def test_sci_get_async(self): From 3285f8a7653cc1827cd07ff5cd3bb051560c0ae9 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 13 Nov 2015 11:16:57 -0600 Subject: [PATCH 075/140] release: bump version to 0.4.1 (bug fix release) Signed-off-by: Paul Osborne --- CHANGELOG.md | 7 +++++++ devicecloud/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ad2a43..7fe9a73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ ## Python Devicecloud Library Changelog +### 0.4.1 / 2015-11-13 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.4...0.4.1) + +Bug Fixes: + +* sci: Targetting groups now works + ### 0.4 / 2015-10-01 [Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.3...0.4) diff --git a/devicecloud/version.py b/devicecloud/version.py index 4db066e..fb4f796 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -4,4 +4,4 @@ # # Copyright (c) 2015 Digi International, Inc. -__version__ = "0.4" +__version__ = "0.4.1" From 183f15aa85cd83c0863c0d3feac0a6cfdcb83cfa Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Fri, 20 Nov 2015 13:31:32 -0600 Subject: [PATCH 076/140] Remove etherios from code base. --- HACKING.md | 3 +-- README.md | 8 +++----- devicecloud/__init__.py | 2 +- devicecloud/devicecore.py | 4 ++-- devicecloud/examples/example_helpers.py | 2 +- devicecloud/test/integration/__init__.py | 3 +-- .../test/integration/inttest_monitor_tcp.py | 2 +- .../test/integration/inttest_streams.py | 3 +-- .../test/integration/inttest_utilities.py | 3 +-- devicecloud/test/unit/__init__.py | 3 +-- devicecloud/test/unit/test_devicecore.py | 20 +++++++++---------- devicecloud/test/unit/test_monitor_tcp.py | 2 +- devicecloud/test/unit/test_utilities.py | 2 +- devicecloud/ws.py | 3 +-- docs/conf.py | 2 +- docs/index.rst | 6 +++--- 16 files changed, 30 insertions(+), 38 deletions(-) diff --git a/HACKING.md b/HACKING.md index bd988d9..74e9957 100644 --- a/HACKING.md +++ b/HACKING.md @@ -97,5 +97,4 @@ Each source file should be prefixed with the following header: # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # - # Copyright (c) 2015 Etherios, Inc. All rights reserved. - # Etherios, Inc. is a Division of Digi International. + # Copyright (c) 2015 Digi International, Inc. All rights reserved. diff --git a/README.md b/README.md index 2f146a1..7315a94 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Overview -------- Python-devicecloud is a library providing simple, intuitive access to -the [Digi Device Cloud](http://www.digi.com/cloud/digi-device-cloud) +the [Digi Device Cloud](http://www.digi.com/products/cloud/digi-device-cloud) for clients written in Python. The library wraps the Device Cloud REST API and hides the details of @@ -28,7 +28,7 @@ The primary target audience for this library is individuals interfacing with the device cloud from the server side or developers writing tools to aid device development. For efficient connectivity from devices, we suggest that you first look at using the [Device Cloud -Connector](http://www.etherios.com/products/devicecloud/connector). +Connector](http://www.digi.com/support/productdetail?pid=5575). That being said, this library could also be used on devices if deemed suitable. @@ -159,9 +159,7 @@ This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, you can obtain one at http://mozilla.org/MPL/2.0/. -Digi, Digi International, the Digi logo, the Digi website, Etherios, -the Etherios logo, the Etherios website, Device Cloud by Etherios, and -Etherios Cloud Connector are trademarks or registered trademarks of +Digi, Digi International, the Digi logo, the Digi website, Digi Device Cloud, and Digi Cloud Connector are trademarks or registered trademarks of Digi International, Inc. in the United States and other countries worldwide. All other trademarks are the property of their respective owners. diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index 0065d1c..81cd393 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -355,7 +355,7 @@ def __init__(self, username, password, base_url=None, throttle_delay_max=DEFAULT_THROTTLE_DELAY_MAX, throttle_delay_backoff_coefficient=DEFAULT_THROTTLE_DELAY_BACKOFF_COEFFICIENT): if base_url is None: - base_url = "https://login.etherios.com" + base_url = "https://devicecloud.digi.com" self._conn = DeviceCloudConnection( auth=HTTPBasicAuth(username, password), base_url=base_url, diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index f1366a1..8a66b61 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -325,8 +325,8 @@ def __init__(self, group_id, name, description, path, parent_id): def from_json(cls, json_data): """Build and return a new Group object from json data (used internally)""" # Example Data: - # { "grpId": "11817", "grpName": "7603_Etherios", "grpDescription": "7603_Etherios root group", - # "grpPath": "\/7603_Etherios\/", "grpParentId": "1"} + # { "grpId": "11817", "grpName": "7603_Digi", "grpDescription": "7603_Digi root group", + # "grpPath": "\/7603_Digi\/", "grpParentId": "1"} return cls( group_id=json_data["grpId"], name=json_data["grpName"], diff --git a/devicecloud/examples/example_helpers.py b/devicecloud/examples/example_helpers.py index ea1b7af..d548bf9 100644 --- a/devicecloud/examples/example_helpers.py +++ b/devicecloud/examples/example_helpers.py @@ -11,7 +11,7 @@ def get_authenticated_dc(): while True: - base_url = os.environ.get('DC_BASE_URL', 'https://login.etherios.com') + base_url = os.environ.get('DC_BASE_URL', 'https://devicecloud.digi.com') username = os.environ.get('DC_USERNAME', None) if not username: diff --git a/devicecloud/test/integration/__init__.py b/devicecloud/test/integration/__init__.py index 856342e..ce587c6 100644 --- a/devicecloud/test/integration/__init__.py +++ b/devicecloud/test/integration/__init__.py @@ -2,5 +2,4 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. All rights reserved. diff --git a/devicecloud/test/integration/inttest_monitor_tcp.py b/devicecloud/test/integration/inttest_monitor_tcp.py index e06eecd..94fd890 100644 --- a/devicecloud/test/integration/inttest_monitor_tcp.py +++ b/devicecloud/test/integration/inttest_monitor_tcp.py @@ -42,7 +42,7 @@ def receive_notification(notification): fd = msg.get('FileData', None) if fd: if (fd['id']['fdName'] == 'test_file.txt' and - fd['id']['fdPath'] == '/db/7603_Etherios/inttest/monitor_tcp/'): + fd['id']['fdPath'] == '/db/7603_Digi/inttest/monitor_tcp/'): fd_push_seen = True dp = msg.get('DataPoint') if dp: diff --git a/devicecloud/test/integration/inttest_streams.py b/devicecloud/test/integration/inttest_streams.py index 4e9694f..ae51fb3 100644 --- a/devicecloud/test/integration/inttest_streams.py +++ b/devicecloud/test/integration/inttest_streams.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. All rights reserved. """Integration tests for streams functionality diff --git a/devicecloud/test/integration/inttest_utilities.py b/devicecloud/test/integration/inttest_utilities.py index f92cae5..64b4be3 100644 --- a/devicecloud/test/integration/inttest_utilities.py +++ b/devicecloud/test/integration/inttest_utilities.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. All rights reserved. from getpass import getpass import unittest diff --git a/devicecloud/test/unit/__init__.py b/devicecloud/test/unit/__init__.py index 856342e..ce587c6 100644 --- a/devicecloud/test/unit/__init__.py +++ b/devicecloud/test/unit/__init__.py @@ -2,5 +2,4 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. All rights reserved. diff --git a/devicecloud/test/unit/test_devicecore.py b/devicecloud/test/unit/test_devicecore.py index f1b1f33..4d80430 100644 --- a/devicecloud/test/unit/test_devicecore.py +++ b/devicecloud/test/unit/test_devicecore.py @@ -120,8 +120,8 @@ "requestedSize": "1000", "remainingSize": "0", "items": [ - { "grpId": "11817", "grpName": "7603_Etherios", "grpDescription": "7603_Etherios root group", "grpPath": "\/7603_Etherios\/", "grpParentId": "1"}, - { "grpId": "13542", "grpName": "Demo", "grpPath": "\/7603_Etherios\/Demo\/", "grpParentId": "11817"} + { "grpId": "11817", "grpName": "7603_Digi", "grpDescription": "7603_Digi root group", "grpPath": "\/7603_Digi\/", "grpParentId": "1"}, + { "grpId": "13542", "grpName": "Demo", "grpPath": "\/7603_Digi\/Demo\/", "grpParentId": "11817"} ] } """ @@ -134,10 +134,10 @@ "requestedSize": "1000", "remainingSize": "0", "items": [ - { "grpId": "11817", "grpName": "7603_Etherios", "grpDescription": "7603_Etherios root group", "grpPath": "\/7603_Etherios\/", "grpParentId": "1"}, - { "grpId": "13542", "grpName": "Demo", "grpPath": "\/7603_Etherios\/Demo\/", "grpParentId": "11817"}, - { "grpId": "13544", "grpName": "SubDir2", "grpPath": "\/7603_Etherios\/Demo\/SubDir2\/", "grpParentId": "13542"}, - { "grpId": "13545", "grpName": "Another Second Level", "grpDescription": "Another Second Level", "grpPath": "\/7603_Etherios\/Another Second Level\/", "grpParentId": "11817"} + { "grpId": "11817", "grpName": "7603_Digi", "grpDescription": "7603_Digi root group", "grpPath": "\/7603_Digi\/", "grpParentId": "1"}, + { "grpId": "13542", "grpName": "Demo", "grpPath": "\/7603_Digi\/Demo\/", "grpParentId": "11817"}, + { "grpId": "13544", "grpName": "SubDir2", "grpPath": "\/7603_Digi\/Demo\/SubDir2\/", "grpParentId": "13542"}, + { "grpId": "13545", "grpName": "Another Second Level", "grpDescription": "Another Second Level", "grpPath": "\/7603_Digi\/Another Second Level\/", "grpParentId": "11817"} ] } """ @@ -181,9 +181,9 @@ def test_get_groups(self): grp = six.next(it) self.assertEqual(grp.is_root(), True) self.assertEqual(grp.get_id(), "11817") - self.assertEqual(grp.get_name(), "7603_Etherios") - self.assertEqual(grp.get_description(), "7603_Etherios root group") - self.assertEqual(grp.get_path(), "/7603_Etherios/") + self.assertEqual(grp.get_name(), "7603_Digi") + self.assertEqual(grp.get_description(), "7603_Digi root group") + self.assertEqual(grp.get_path(), "/7603_Digi/") self.assertEqual(grp.get_parent_id(), "1") grp = six.next(it) @@ -191,7 +191,7 @@ def test_get_groups(self): self.assertEqual(grp.get_id(), "13542") self.assertEqual(grp.get_name(), "Demo") self.assertEqual(grp.get_description(), "") - self.assertEqual(grp.get_path(), "/7603_Etherios/Demo/") + self.assertEqual(grp.get_path(), "/7603_Digi/Demo/") self.assertEqual(grp.get_parent_id(), "11817") def test_repr_and_tree_print(self): diff --git a/devicecloud/test/unit/test_monitor_tcp.py b/devicecloud/test/unit/test_monitor_tcp.py index 2c85f78..1eb3acc 100644 --- a/devicecloud/test/unit/test_monitor_tcp.py +++ b/devicecloud/test/unit/test_monitor_tcp.py @@ -20,7 +20,7 @@ def setUp(self): self.client_manager = TCPClientManager(self.dc.get_connection()) def test_hostname(self): - self.assertEqual(self.client_manager.hostname, "login.etherios.com") + self.assertEqual(self.client_manager.hostname, "devicecloud.digi.com") def test_username(self): self.assertEqual(self.client_manager.username, "user") diff --git a/devicecloud/test/unit/test_utilities.py b/devicecloud/test/unit/test_utilities.py index 73b4818..2ed5fcf 100644 --- a/devicecloud/test/unit/test_utilities.py +++ b/devicecloud/test/unit/test_utilities.py @@ -38,7 +38,7 @@ def prepare_response(self, method, path, data=None, status=200, match_querystrin if data is not None: kwargs['body'] = data httpretty.register_uri(method, - "https://login.etherios.com{}".format(path), + "https://devicecloud.digi.com{}".format(path), match_querystring=match_querystring, status=status, **kwargs) diff --git a/devicecloud/ws.py b/devicecloud/ws.py index ad5f68e..afac53e 100644 --- a/devicecloud/ws.py +++ b/devicecloud/ws.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Etherios, Inc. All rights reserved. -# Etherios, Inc. is a Division of Digi International. +# Copyright (c) 2015 Digi International, Inc. All rights reserved. import functools import inspect diff --git a/docs/conf.py b/docs/conf.py index 18d5e62..1a9ef60 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -55,7 +55,7 @@ # General information about the project. project = u'python-devicecloud' -copyright = u'2014, Etherios, Inc.' +copyright = u'2015, Digi International, Inc.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/docs/index.rst b/docs/index.rst index 44a97fa..5cab9a1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,8 +21,8 @@ Introduction ============ Python-devicecloud is a library providing simple, intuitive access to -the `Device Cloud by Etherios -`_ for clients written +the `Digi Device Cloud +`_ for clients written in Python. The library wraps the Device Cloud REST API and hides the details of @@ -36,7 +36,7 @@ The primary target audience for this library is individuals interfacing with the device cloud from the server side or developers writing tools to aid device development. For efficient connectivity from devices, we suggest that you first look at using the `Device -Cloud Connector `_. +Cloud Connector `_. That being said, this library could also be used on devices if deemed suitable. From 74c2542ca0730717c0f8c4b977d5817c926b9610 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Fri, 20 Nov 2015 14:29:14 -0600 Subject: [PATCH 077/140] Fix value lengths based upon change from Etherios to Digi --- devicecloud/test/unit/test_devicecore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devicecloud/test/unit/test_devicecore.py b/devicecloud/test/unit/test_devicecore.py index 4d80430..0c63a85 100644 --- a/devicecloud/test/unit/test_devicecore.py +++ b/devicecloud/test/unit/test_devicecore.py @@ -200,9 +200,9 @@ def test_repr_and_tree_print(self): root = self.dc.devicecore.get_group_tree_root() root.print_subtree(fobj) # the order of the traversal can vary, so just assert on the length if six.PY2: - self.assertEqual(len(fobj.getvalue()), 513) + self.assertEqual(len(fobj.getvalue()), 489) elif six.PY3: - self.assertEqual(len(fobj.getvalue()), 495) # no u'' on repr for strings + self.assertEqual(len(fobj.getvalue()), 471) # no u'' on repr for strings def test_get_groups_condition(self): self.prepare_response("GET", "/ws/Group", EXAMPLE_GET_GROUPS) From 1734d81a1d700af1ededec0b199ce5e68865c167 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 20 Nov 2015 14:46:24 -0600 Subject: [PATCH 078/140] release: prepare release 0.4.2 with base url update Signed-off-by: Paul Osborne --- CHANGELOG.md | 10 ++++++++++ devicecloud/version.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fe9a73..c5ae3c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ ## Python Devicecloud Library Changelog +### 0.4.1 / 2015-11-20 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.4.1...0.4.2) + +Bug Fixes: + +* core: All Etherios references have been replaced with Digi. Most + importantly, the default URL for the devicecloud is now + devicecloud.digi.com. The old URL may not redirect properly at some + point in the future. + ### 0.4.1 / 2015-11-13 [Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.4...0.4.1) diff --git a/devicecloud/version.py b/devicecloud/version.py index fb4f796..36e50c1 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -4,4 +4,4 @@ # # Copyright (c) 2015 Digi International, Inc. -__version__ = "0.4.1" +__version__ = "0.4.2" From c23409082c6248273d1d15da30f2e9a9dae2f481 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 20 Nov 2015 15:22:33 -0600 Subject: [PATCH 079/140] changelog: fix typo in 0.4.2 release heading Signed-off-by: Paul Osborne --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5ae3c7..23fa9a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## Python Devicecloud Library Changelog -### 0.4.1 / 2015-11-20 +### 0.4.2 / 2015-11-20 [Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.4.1...0.4.2) Bug Fixes: From a41de0b10a9416a290bbaf0b36c071cc0492439d Mon Sep 17 00:00:00 2001 From: Kurt Erickson Date: Thu, 4 Feb 2016 17:46:31 -0600 Subject: [PATCH 080/140] Fix check of send_sci sync_timeout parameter --- devicecloud/sci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devicecloud/sci.py b/devicecloud/sci.py index 7b5bc62..cbbbe22 100644 --- a/devicecloud/sci.py +++ b/devicecloud/sci.py @@ -173,7 +173,7 @@ def send_sci(self, operation, target, payload, reply=None, synchronous=None, syn # sync_timeout argument # TODO: What units is syncTimeout in? seconds? - if not sync_timeout is None or isinstance(sync_timeout, six.integer_types): + if sync_timeout is not None and not isinstance(sync_timeout, six.integer_types): raise TypeError("sync_timeout expected to either be None or a number") if sync_timeout is not None: sync_timeout_xml = ' syncTimeout="{}"'.format(sync_timeout) From 8c13f264ccc43737602d1267b51df5670cd83b8c Mon Sep 17 00:00:00 2001 From: Kurt Erickson Date: Fri, 5 Feb 2016 15:16:07 -0600 Subject: [PATCH 081/140] Fixed broken reply parameter for send_sci method Add unit test for send_sci parameter testing --- devicecloud/sci.py | 2 +- devicecloud/test/unit/test_sci.py | 87 +++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/devicecloud/sci.py b/devicecloud/sci.py index cbbbe22..a4069ac 100644 --- a/devicecloud/sci.py +++ b/devicecloud/sci.py @@ -12,7 +12,7 @@ SCI_TEMPLATE = """\ - <{operation}{synchronous}{cache}{sync_timeout}{allow_offline}{wait_for_reconnect}> + <{operation}{reply}{synchronous}{cache}{sync_timeout}{allow_offline}{wait_for_reconnect}> {targets} diff --git a/devicecloud/test/unit/test_sci.py b/devicecloud/test/unit/test_sci.py index 1c05c08..46096df 100644 --- a/devicecloud/test/unit/test_sci.py +++ b/devicecloud/test/unit/test_sci.py @@ -9,6 +9,8 @@ import httpretty import mock import six +import re +import xml.etree.ElementTree as ET from devicecloud import DeviceCloud from devicecloud.sci import DeviceTarget, GroupTarget, AsyncRequestProxy, ServerCommandInterfaceAPI @@ -48,6 +50,22 @@ """ +EXAMPLE_SCI_REQUEST_RESPONSE = """\ + + + + + + + running + + + + + + +""" + EXAMPLE_ASYNC_SCI_RESPONSE = """\ @@ -93,6 +111,75 @@ def test_sci_successful_group_target(self): '' '')) + def test_sci_no_parameters(self): + self._prepare_sci_response(EXAMPLE_SCI_REQUEST_RESPONSE) + self.dc.get_sci_api().send_sci( + operation="send_message", + target=DeviceTarget('00000000-00000000-00409dff-ffaabbcc'), + payload=EXAMPLE_SCI_REQUEST_PAYLOAD) + request = httpretty.last_request().body + # Strip white space from lines and concatenate request + request = ''.join([line.strip() for line in request.splitlines()]) + self.assertEqual(request, + six.b('' + '' + '' + '' + '' + '' + '' + '' + '' + '' + '' + '')) + + def test_sci_with_parameters(self): + self._prepare_sci_response(EXAMPLE_SCI_REQUEST_RESPONSE) + self.dc.get_sci_api().send_sci( + operation="send_message", + target=DeviceTarget('00000000-00000000-00409dff-ffaabbcc'), + payload=EXAMPLE_SCI_REQUEST_PAYLOAD, + reply="all", + synchronous=True, + sync_timeout=42, + cache=False, + allow_offline=True, + wait_for_reconnect=True, + ) + request = httpretty.last_request().body + # Verify attributes exist in + expected_attrib = { + "reply": "all", + "synchronous": "true", + "syncTimeout": "42", + "cache": "false", + "allowOffline": "true", + "waitForReconnect": "true", + } + request_e = ET.fromstring(request) + send_message_e = request_e.find('./send_message') + self.assertEqual(expected_attrib, send_message_e.attrib) + # Strip white space from lines and concatenate request + request = ''.join([line.strip() for line in request.splitlines()]) + # Replace from request with one without parameters so the final check can be done + match = re.search('', request) + request = request[:match.start()] + '' + request[match.end():] + self.assertEqual(request, + six.b('' + '' + '' + '' + '' + '' + '' + '' + '' + '' + '' + '')) + + class TestGetAsync(HttpTestBase): def test_sci_get_async(self): self.prepare_response("GET", "/ws/sci/123", EXAMPLE_ASYNC_SCI_DEVICE_NOT_CONNECTED, 200) From 86b21424bcb68d830dbab07a6b09ed7a947f0b97 Mon Sep 17 00:00:00 2001 From: Dan Harrison Date: Tue, 8 Mar 2016 15:45:09 -0600 Subject: [PATCH 082/140] test/unit/test_sci.py: fixed unit tests on Python 3 --- devicecloud/test/unit/test_sci.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/devicecloud/test/unit/test_sci.py b/devicecloud/test/unit/test_sci.py index 46096df..6acd9d3 100644 --- a/devicecloud/test/unit/test_sci.py +++ b/devicecloud/test/unit/test_sci.py @@ -117,11 +117,11 @@ def test_sci_no_parameters(self): operation="send_message", target=DeviceTarget('00000000-00000000-00409dff-ffaabbcc'), payload=EXAMPLE_SCI_REQUEST_PAYLOAD) - request = httpretty.last_request().body + request = httpretty.last_request().body.decode('utf8') # Strip white space from lines and concatenate request request = ''.join([line.strip() for line in request.splitlines()]) self.assertEqual(request, - six.b('' + six.u('' '' '' '' @@ -147,7 +147,7 @@ def test_sci_with_parameters(self): allow_offline=True, wait_for_reconnect=True, ) - request = httpretty.last_request().body + request = httpretty.last_request().body.decode('utf8') # Verify attributes exist in expected_attrib = { "reply": "all", @@ -166,7 +166,7 @@ def test_sci_with_parameters(self): match = re.search('', request) request = request[:match.start()] + '' + request[match.end():] self.assertEqual(request, - six.b('' + six.u('' '' '' '' From 5355970aaeba468ad147370391c792235c79bbc1 Mon Sep 17 00:00:00 2001 From: Dan Harrison Date: Thu, 10 Mar 2016 09:56:26 -0600 Subject: [PATCH 083/140] Removed Python 3.2 support (since tox has deprecated it) --- setup.py | 1 - tox.ini | 2 +- toxtest.sh | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 34b9629..bb43eab 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,6 @@ def get_long_description(): "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.2", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Topic :: Software Development :: Libraries", diff --git a/tox.ini b/tox.ini index 93045a2..8b329df 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py32,py33,py34,pypy +envlist = py27,py33,py34,pypy [testenv] passenv = * diff --git a/toxtest.sh b/toxtest.sh index 7c38aad..1d1d72f 100755 --- a/toxtest.sh +++ b/toxtest.sh @@ -7,7 +7,6 @@ # pyversions=(2.7.7 - 3.2.5 3.3.5 3.4.3 pypy-2.3.1) From ab91fef2fa002770391ccf55e9f975a0a4f1c5e0 Mon Sep 17 00:00:00 2001 From: Dan Harrison Date: Thu, 10 Mar 2016 11:00:40 -0600 Subject: [PATCH 084/140] devicecloud/__init__.py: fixed Filesystem API accessor --- devicecloud/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index 81cd393..c6f3508 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -366,6 +366,7 @@ def __init__(self, username, password, base_url=None, ) self._streams_api = None # streams property api ref self._filedata_api = None # filedata property api ref + self._fss_api = None # fileservice property api ref self._devicecore_api = None # devicecore property api ref self._sci_api = None # sci property api ref self._monitor_api = None # monitor property of api ref From 781b9289ebc9f95d450aa68f8661ad8e1707c67e Mon Sep 17 00:00:00 2001 From: Dan Harrison Date: Tue, 5 Apr 2016 14:03:15 -0500 Subject: [PATCH 085/140] devicecloud/file_system_service.py: made get_file properly handle empty files --- devicecloud/file_system_service.py | 5 ++++- devicecloud/test/unit/test_file_system_service.py | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/devicecloud/file_system_service.py b/devicecloud/file_system_service.py index 4be4217..801dd78 100644 --- a/devicecloud/file_system_service.py +++ b/devicecloud/file_system_service.py @@ -403,7 +403,10 @@ def parse_response(cls, response, **kwargs): return _parse_error_tree(error) data = six.b(response.find('./data').text) - return base64.b64decode(data) + if data: + return base64.b64decode(data) + else: + return six.b('') class PutCommand(FileSystemServiceCommandABC): diff --git a/devicecloud/test/unit/test_file_system_service.py b/devicecloud/test/unit/test_file_system_service.py index 68808ab..3112da9 100644 --- a/devicecloud/test/unit/test_file_system_service.py +++ b/devicecloud/test/unit/test_file_system_service.py @@ -228,6 +228,10 @@ def test_init_values(self): self.assertEqual(0, len(list(et))) self.assertEqual(None, et.text) + def test_parse_empty(self): + data = GetCommand.parse_response(ET.fromstring(GET_FILE_BLOCK.format(data=''))) + self.assertEqual(six.b(""), data) + def test_parse(self): data_str = base64.b64encode(six.b("File Data")).decode('ascii') data = GetCommand.parse_response(ET.fromstring(GET_FILE_BLOCK.format(data=data_str))) From fd8a78bf1f655b330bc6a4de113475d839ad7409 Mon Sep 17 00:00:00 2001 From: Dan Harrison Date: Tue, 5 Apr 2016 14:07:39 -0500 Subject: [PATCH 086/140] devicecloud/file_system_service.py: fixed tox unit tests --- devicecloud/file_system_service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devicecloud/file_system_service.py b/devicecloud/file_system_service.py index 801dd78..15eb6a4 100644 --- a/devicecloud/file_system_service.py +++ b/devicecloud/file_system_service.py @@ -402,9 +402,9 @@ def parse_response(cls, response, **kwargs): if error is not None: return _parse_error_tree(error) - data = six.b(response.find('./data').text) - if data: - return base64.b64decode(data) + text = response.find('./data').text + if text: + return base64.b64decode(six.b(text)) else: return six.b('') From 7f3055625440d15d36214bb46d30d3d3c9337932 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Mon, 11 Jun 2018 14:40:43 -0500 Subject: [PATCH 087/140] Update requirements to latest libraries. Closes #17 --- requirements.txt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1f7a761..147559a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,9 @@ -six==1.8.0 -requests==2.5.1 -arrow==0.4.4 +arrow==0.12.1 +backports.functools-lru-cache==1.5 +certifi==2018.4.16 +chardet==3.0.4 +idna==2.6 +python-dateutil==2.7.3 +requests==2.18.4 +six==1.11.0 +urllib3==1.22 From 1d5bb35935c06c79dd9a6f93e4bfb15e81001286 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Mon, 11 Jun 2018 14:43:27 -0500 Subject: [PATCH 088/140] Update Device Cloud CRT to latest. Closes #16 --- devicecloud/data/devicecloud.crt | 199 ++++++++++++++++++------------- 1 file changed, 113 insertions(+), 86 deletions(-) diff --git a/devicecloud/data/devicecloud.crt b/devicecloud/data/devicecloud.crt index 9641564..928955f 100644 --- a/devicecloud/data/devicecloud.crt +++ b/devicecloud/data/devicecloud.crt @@ -1,94 +1,121 @@ -----BEGIN CERTIFICATE----- -MIID7jCCAtagAwIBAgIQHywezy3AzTcnh1pWVDNUujANBgkqhkiG9w0BAQUFADA8 -MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMVGhhd3RlLCBJbmMuMRYwFAYDVQQDEw1U -aGF3dGUgU1NMIENBMB4XDTEyMDEyNTAwMDAwMFoXDTE3MDEyMzIzNTk1OVowgYYx -CzAJBgNVBAYTAlVTMRIwEAYDVQQIEwlNaW5uZXNvdGExEzARBgNVBAcUCk1pbm5l -dG9ua2ExGzAZBgNVBAoUEkRpZ2kgSW50ZXJuYXRpb25hbDEbMBkGA1UECxQSRGln -aSBJbnRlcm5hdGlvbmFsMRQwEgYDVQQDFAsqLmlkaWdpLmNvbTCCASIwDQYJKoZI -hvcNAQEBBQADggEPADCCAQoCggEBAMhPXg6GdwDI/z8K13CQzyIPbebrjG1KrV1F -qKBOshm9Iqf4DrZ+pNSA/ckOa2WrSZjMHJPTprAaRL+gh/Ymh3WDCLCKIZy3fm7Q -jqTNRLDJipNypBrc8OJmaz7I7wBfAKo6mJU+GlyTvR3fyBaqeihVbqE+aOhtMQvq -I+PjcvNXRx9c4YZrz1swKDwZo36A3zDtefQo/ZK08W+eGP4HEasJeRl63hfo24CT -Se4r02c4TRRJcNbTPG2jueiUrawPy4hh9f2y/luQ6UZfnqxO/CTep5AVGQcDZt+O -ItDl42EMmPvhYFy74fHYlEOVWkIR836uExPHBAD/YMEQuq3BeckCAwEAAaOBoDCB -nTAMBgNVHRMBAf8EAjAAMDoGA1UdHwQzMDEwL6AtoCuGKWh0dHA6Ly9zdnItb3Yt -Y3JsLnRoYXd0ZS5jb20vVGhhd3RlT1YuY3JsMB0GA1UdJQQWMBQGCCsGAQUFBwMB -BggrBgEFBQcDAjAyBggrBgEFBQcBAQQmMCQwIgYIKwYBBQUHMAGGFmh0dHA6Ly9v -Y3NwLnRoYXd0ZS5jb20wDQYJKoZIhvcNAQEFBQADggEBADOWE49SbClVY/X7TWDf -UIJk6ItfQBS+h2i9e5uKmiAGu9akkM9weVOLGfs+jsGSJnqKlossfqfMKB74hO8q -gmYgYtSx+LLjzSxkcHkGSYGErtAs2EemddB4zkUurK+oZ3qNpfCYPnWbzboNM4Ip -nJ7I7K9bNkbEa9Q0m4iu06Uc+yUmA/DvgStmBHHRsLfVB+KiuuwXcLPSxr0mi7IQ -eaj8GKGhBMGPVr3mvhFrKusCNEX4gxiX35s6JiF5gKnZ3AQitWez5gpOjhi14Yv3 -UyRLm2WhxcYY+zzWlfzfyJyNL+rlfoC1IQbfwISvBW14VjbCqgiOmPfXuoYrhCAX -EHk= +MIIHXTCCBkWgAwIBAgIJANZr7DJFVmHfMA0GCSqGSIb3DQEBCwUAMIG0MQswCQYD +VQQGEwJVUzEQMA4GA1UECBMHQXJpem9uYTETMBEGA1UEBxMKU2NvdHRzZGFsZTEa +MBgGA1UEChMRR29EYWRkeS5jb20sIEluYy4xLTArBgNVBAsTJGh0dHA6Ly9jZXJ0 +cy5nb2RhZGR5LmNvbS9yZXBvc2l0b3J5LzEzMDEGA1UEAxMqR28gRGFkZHkgU2Vj +dXJlIENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTE3MDIxMzIxMzIwMVoX +DTE5MDMxMjEzMTgzOFowgdcxEzARBgsrBgEEAYI3PAIBAxMCVVMxGjAYBgsrBgEE +AYI3PAIBAhMJTWlubmVzb3RhMR0wGwYDVQQPExRQcml2YXRlIE9yZ2FuaXphdGlv +bjEOMAwGA1UEBRMFNDg4NTkxCzAJBgNVBAYTAlVTMRIwEAYDVQQIEwlNaW5uZXNv +dGExEzARBgNVBAcTCk1pbm5ldG9ua2ExIDAeBgNVBAoTF0RpZ2kgSW50ZXJuYXRp +b25hbCBJbmMuMR0wGwYDVQQDExRkZXZpY2VjbG91ZC5kaWdpLmNvbTCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBAOiQG1upAjqcisuXt+Kz/3RTbbWli8q1 +a4KtcIg0y19gClNsSWVioHH4KFBlYEafcL42MT3LP5+WMEGedBzf6rm7b7MW4IvY +zMUqilUWxXhMtgFRZP/NQlugFCGVL529qult/ZZ6Oo+DTdBukduEdCtXU1KJuFwh +MP1zL85TxTyAt2t7TrWO5Pat9lQGxd/LjC0mI7TdQ++9VNsAzDcYWrP9TSGM8wsr +lU739jLQhRz6Q4UYUPFk8PEvp6x4wdxFMhTqOIQ4McOkjokHLd6WXcjMhY4tPgTJ +hNuiYgtoGj+EGI2XDWPDAc+dQTKWfMa+ykZxbyb4lXyqldARir/OrwsCAwEAAaOC +A0swggNHMAwGA1UdEwEB/wQCMAAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUF +BwMCMA4GA1UdDwEB/wQEAwIFoDA1BgNVHR8ELjAsMCqgKKAmhiRodHRwOi8vY3Js +LmdvZGFkZHkuY29tL2dkaWcyczMtNy5jcmwwXAYDVR0gBFUwUzBIBgtghkgBhv1t +AQcXAzA5MDcGCCsGAQUFBwIBFitodHRwOi8vY2VydGlmaWNhdGVzLmdvZGFkZHku +Y29tL3JlcG9zaXRvcnkvMAcGBWeBDAEBMHYGCCsGAQUFBwEBBGowaDAkBggrBgEF +BQcwAYYYaHR0cDovL29jc3AuZ29kYWRkeS5jb20vMEAGCCsGAQUFBzAChjRodHRw +Oi8vY2VydGlmaWNhdGVzLmdvZGFkZHkuY29tL3JlcG9zaXRvcnkvZ2RpZzIuY3J0 +MB8GA1UdIwQYMBaAFEDCvSeOzDSDMKIz1/tss/C0LIDOMDkGA1UdEQQyMDCCFGRl +dmljZWNsb3VkLmRpZ2kuY29tghh3d3cuZGV2aWNlY2xvdWQuZGlnaS5jb20wHQYD +VR0OBBYEFFvYGl4ynfmb11I5vDrDq1/excTEMIIBfgYKKwYBBAHWeQIEAgSCAW4E +ggFqAWgAdgBWFAaaL9fC7NP14b1Esj7HRna5vJkRXMDvlJhV1onQ3QAAAVo5ZQR8 +AAAEAwBHMEUCIEQfzk/nT9zPlxJEnYsI0jdcPqIhFTYCevqyAE4EXrgYAiEAmeWb +BVXzM52QxKPOtXpgKOMPjUihpqqgJ9mKjmejgI4AdQDuS723dc5guuFCaR+r4Z5m +ow9+X7By2IMAxHuJeqj9ywAAAVo5ZQhLAAAEAwBGMEQCIAjkl2UpriTtD4bdTjTD +4IUMaYZMN0NdVsDk5oxSJxCiAiAOtR7KSAKkRn0gEdyTrJpQPolgCLNHj55GPOwZ +oDzX3AB3AKS5CZC0GFgUh7sTosxncAo8NZgE+RvfuON3zQ7IDdwQAAABWjllCWQA +AAQDAEgwRgIhAMw0R6P4MFmCGO1x+QgHeBBWYeVQvMXhGbwN+Ffh4OXoAiEA/kOC +UfexoAx+PmKQ1cQKTKkRCzLhTxzCAOaySMrm/iAwDQYJKoZIhvcNAQELBQADggEB +AGLrc578DVbChRu29j/+c4Q9jlLW8WARTcAbK3dlAX28Hx3jlLEkfWA2aFCoQsva +thgGik6sMOUTkAzV8Qof+6akyeNaVh2yIByNupsnXgsC1SM5qD4UmfBdPGwzW3Xd +RWGN+LM6cmGBzxNOVH2NxdZ/XCGm282tmBnDsXioIQ7fo0EW681svJRCUwbYv4mm +8wwX8ALNVlODfhs4TwxASOMB2+uFEXRzJ15BawGeXDPg7C1o/giHru6vbO2DVCga +7eeCxCnM4NJ/dCzc2+K6FqNGL3ddQzAmEtwh4frHEs2KK6/fP2FeZA2WzOTJ4ZXx +jcicTeajht5zfh8/I94rwJI= -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIEbDCCA1SgAwIBAgIQTV8sNAiyTCDNbVB+JE3J7DANBgkqhkiG9w0BAQUFADCB -qTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjEoMCYGA1UECxMf -Q2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYGA1UECxMvKGMpIDIw -MDYgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxHzAdBgNV -BAMTFnRoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EwHhcNMTAwMjA4MDAwMDAwWhcNMjAw -MjA3MjM1OTU5WjA8MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMVGhhd3RlLCBJbmMu -MRYwFAYDVQQDEw1UaGF3dGUgU1NMIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAmeSFW3ZJfS8F2MWsyMip09yY5tc0pi8M8iIm2KPJFEyPBaRF6BQM -WJAFGrfFwQalgK+7HUlrUjSIw1nn72vEJ0GMK2Yd0OCjl5gZNEtB1ZjVxwWtouTX -7QytT8G1sCH9PlBTssSQ0NQwZ2ya8Q50xMLciuiX/8mSrgGKVgqYMrAAI+yQGmDD -7bs6yw9jnw1EyVLhJZa/7VCViX9WFLG3YR0cB4w6LPf/gN45RdWvGtF42MdxaqMZ -pzJQIenyDqHGEwNESNFmqFJX1xG0k4vlmZ9d53hR5U32t1m0drUJN00GOBN6HAiY -XMRISstSoKn4sZ2Oe3mwIC88lqgRYke7EQIDAQABo4H7MIH4MDIGCCsGAQUFBwEB -BCYwJDAiBggrBgEFBQcwAYYWaHR0cDovL29jc3AudGhhd3RlLmNvbTASBgNVHRMB -Af8ECDAGAQH/AgEAMDQGA1UdHwQtMCswKaAnoCWGI2h0dHA6Ly9jcmwudGhhd3Rl -LmNvbS9UaGF3dGVQQ0EuY3JsMA4GA1UdDwEB/wQEAwIBBjAoBgNVHREEITAfpB0w -GzEZMBcGA1UEAxMQVmVyaVNpZ25NUEtJLTItOTAdBgNVHQ4EFgQUp6KDuzRFQD38 -1TBPErk+oQGf9tswHwYDVR0jBBgwFoAUe1tFz6/Oy3r9MZIaarbzRutXSFAwDQYJ -KoZIhvcNAQEFBQADggEBAIAigOBsyJUW11cmh/NyNNvGclYnPtOW9i4lkaU+M5en -S+Uv+yV9Lwdh+m+DdExMU3IgpHrPUVFWgYiwbR82LMgrsYiZwf5Eq0hRfNjyRGQq -2HGn+xov+RmNNLIjv8RMVR2OROiqXZrdn/0Dx7okQ40tR0Tb9tiYyLL52u/tKVxp -EvrRI5YPv5wN8nlFUzeaVi/oVxBw9u6JDEmJmsEj9cIqzEHPIqtlbreUgm0vQF9Y -3uuVK6ZyaFIZkSqudZ1OkubK3lTqGKslPOZkpnkfJn1h7X3S5XFV2JMXfBQ4MDzf -huNMrUnjl1nOG5srztxl1Asoa06ERlFE9zMILViXIa4= +MIIE0DCCA7igAwIBAgIBBzANBgkqhkiG9w0BAQsFADCBgzELMAkGA1UEBhMCVVMx +EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxGjAYBgNVBAoT +EUdvRGFkZHkuY29tLCBJbmMuMTEwLwYDVQQDEyhHbyBEYWRkeSBSb290IENlcnRp +ZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTExMDUwMzA3MDAwMFoXDTMxMDUwMzA3 +MDAwMFowgbQxCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6b25hMRMwEQYDVQQH +EwpTY290dHNkYWxlMRowGAYDVQQKExFHb0RhZGR5LmNvbSwgSW5jLjEtMCsGA1UE +CxMkaHR0cDovL2NlcnRzLmdvZGFkZHkuY29tL3JlcG9zaXRvcnkvMTMwMQYDVQQD +EypHbyBEYWRkeSBTZWN1cmUgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IC0gRzIwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC54MsQ1K92vdSTYuswZLiBCGzD +BNliF44v/z5lz4/OYuY8UhzaFkVLVat4a2ODYpDOD2lsmcgaFItMzEUz6ojcnqOv +K/6AYZ15V8TPLvQ/MDxdR/yaFrzDN5ZBUY4RS1T4KL7QjL7wMDge87Am+GZHY23e +cSZHjzhHU9FGHbTj3ADqRay9vHHZqm8A29vNMDp5T19MR/gd71vCxJ1gO7GyQ5HY +pDNO6rPWJ0+tJYqlxvTV0KaudAVkV4i1RFXULSo6Pvi4vekyCgKUZMQWOlDxSq7n +eTOvDCAHf+jfBDnCaQJsY1L6d8EbyHSHyLmTGFBUNUtpTrw700kuH9zB0lL7AgMB +AAGjggEaMIIBFjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNV +HQ4EFgQUQMK9J47MNIMwojPX+2yz8LQsgM4wHwYDVR0jBBgwFoAUOpqFBxBnKLbv +9r0FQW4gwZTaD94wNAYIKwYBBQUHAQEEKDAmMCQGCCsGAQUFBzABhhhodHRwOi8v +b2NzcC5nb2RhZGR5LmNvbS8wNQYDVR0fBC4wLDAqoCigJoYkaHR0cDovL2NybC5n +b2RhZGR5LmNvbS9nZHJvb3QtZzIuY3JsMEYGA1UdIAQ/MD0wOwYEVR0gADAzMDEG +CCsGAQUFBwIBFiVodHRwczovL2NlcnRzLmdvZGFkZHkuY29tL3JlcG9zaXRvcnkv +MA0GCSqGSIb3DQEBCwUAA4IBAQAIfmyTEMg4uJapkEv/oV9PBO9sPpyIBslQj6Zz +91cxG7685C/b+LrTW+C05+Z5Yg4MotdqY3MxtfWoSKQ7CC2iXZDXtHwlTxFWMMS2 +RJ17LJ3lXubvDGGqv+QqG+6EnriDfcFDzkSnE3ANkR/0yBOtg2DZ2HKocyQetawi +DsoXiWJYRBuriSUBAA/NxBti21G00w9RKpv0vHP8ds42pM3Z2Czqrpv1KrKQ0U11 +GIo/ikGQI31bS/6kA1ibRrLDYGCD+H1QQc7CoZDDu+8CL9IVVO5EFdkKrqeKM+2x +LXY2JtwE65/3YR8V3Idv7kaWKK2hJn0KCacuBKONvPi8BDAB -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIERTCCA66gAwIBAgIQM2VQCHmtc+IwueAdDX+skTANBgkqhkiG9w0BAQUFADCB -zjELMAkGA1UEBhMCWkExFTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJ -Q2FwZSBUb3duMR0wGwYDVQQKExRUaGF3dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UE -CxMfQ2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjEhMB8GA1UEAxMYVGhh -d3RlIFByZW1pdW0gU2VydmVyIENBMSgwJgYJKoZIhvcNAQkBFhlwcmVtaXVtLXNl -cnZlckB0aGF3dGUuY29tMB4XDTA2MTExNzAwMDAwMFoXDTIwMTIzMDIzNTk1OVow -gakxCzAJBgNVBAYTAlVTMRUwEwYDVQQKEwx0aGF3dGUsIEluYy4xKDAmBgNVBAsT -H0NlcnRpZmljYXRpb24gU2VydmljZXMgRGl2aXNpb24xODA2BgNVBAsTLyhjKSAy -MDA2IHRoYXd0ZSwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MR8wHQYD -VQQDExZ0aGF3dGUgUHJpbWFyeSBSb290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEArKDw+4BZ1JzHpM+doVlzCRBFDA0sbmjxbFtIaElZN/wLMxnC -d3/MEC2VNBzm600JpxzSuMmXNgK3idQkXwbAzESUlI0CYm/rWt0RjSiaXISQEHoN -vXRmL2o4oOLVVETrHQefB7pv7un9Tgsp9T6EoAHxnKv4HH6JpOih2HFlDaNRe+68 -0iJgDblbnd+6/FFbC6+Ysuku6QToYofeK8jXTsFMZB7dz4dYukpPymgHHRydSsbV -L5HMfHFyHMXAZ+sy/cmSXJTahcCbv1N9Kwn0jJ2RH5dqUsveCTakd9h7h1BE1T5u -KWn7OUkmHgmlgHtALevoJ4XJ/mH9fuZ8lx3VnQIDAQABo4HCMIG/MA8GA1UdEwEB -/wQFMAMBAf8wOwYDVR0gBDQwMjAwBgRVHSAAMCgwJgYIKwYBBQUHAgEWGmh0dHBz -Oi8vd3d3LnRoYXd0ZS5jb20vY3BzMA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQU -e1tFz6/Oy3r9MZIaarbzRutXSFAwQAYDVR0fBDkwNzA1oDOgMYYvaHR0cDovL2Ny -bC50aGF3dGUuY29tL1RoYXd0ZVByZW1pdW1TZXJ2ZXJDQS5jcmwwDQYJKoZIhvcN -AQEFBQADgYEAhKhMyT4qvJrizI8LsiV3xGGJiWNa1KMVQNT7Xj+0Q+pjFytrmXSe -Cajd1FYVLnp5MV9jllMbNNkV6k9tcMq+9oKp7dqFd8x2HGqBCiHYQZl/Xi6Cweiq -95OBBaqStB+3msAHF/XLxrRMDtdW3HEgdDjWdMbWj2uvi42gbCkLYeA= +MIIEfTCCA2WgAwIBAgIDG+cVMA0GCSqGSIb3DQEBCwUAMGMxCzAJBgNVBAYTAlVT +MSEwHwYDVQQKExhUaGUgR28gRGFkZHkgR3JvdXAsIEluYy4xMTAvBgNVBAsTKEdv +IERhZGR5IENsYXNzIDIgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTQwMTAx +MDcwMDAwWhcNMzEwNTMwMDcwMDAwWjCBgzELMAkGA1UEBhMCVVMxEDAOBgNVBAgT +B0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxGjAYBgNVBAoTEUdvRGFkZHku +Y29tLCBJbmMuMTEwLwYDVQQDEyhHbyBEYWRkeSBSb290IENlcnRpZmljYXRlIEF1 +dGhvcml0eSAtIEcyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv3Fi +CPH6WTT3G8kYo/eASVjpIoMTpsUgQwE7hPHmhUmfJ+r2hBtOoLTbcJjHMgGxBT4H +Tu70+k8vWTAi56sZVmvigAf88xZ1gDlRe+X5NbZ0TqmNghPktj+pA4P6or6KFWp/ +3gvDthkUBcrqw6gElDtGfDIN8wBmIsiNaW02jBEYt9OyHGC0OPoCjM7T3UYH3go+ +6118yHz7sCtTpJJiaVElBWEaRIGMLKlDliPfrDqBmg4pxRyp6V0etp6eMAo5zvGI +gPtLXcwy7IViQyU0AlYnAZG0O3AqP26x6JyIAX2f1PnbU21gnb8s51iruF9G/M7E +GwM8CetJMVxpRrPgRwIDAQABo4IBFzCCARMwDwYDVR0TAQH/BAUwAwEB/zAOBgNV +HQ8BAf8EBAMCAQYwHQYDVR0OBBYEFDqahQcQZyi27/a9BUFuIMGU2g/eMB8GA1Ud +IwQYMBaAFNLEsNKR1EwRcbNhyz2h/t2oatTjMDQGCCsGAQUFBwEBBCgwJjAkBggr +BgEFBQcwAYYYaHR0cDovL29jc3AuZ29kYWRkeS5jb20vMDIGA1UdHwQrMCkwJ6Al +oCOGIWh0dHA6Ly9jcmwuZ29kYWRkeS5jb20vZ2Ryb290LmNybDBGBgNVHSAEPzA9 +MDsGBFUdIAAwMzAxBggrBgEFBQcCARYlaHR0cHM6Ly9jZXJ0cy5nb2RhZGR5LmNv +bS9yZXBvc2l0b3J5LzANBgkqhkiG9w0BAQsFAAOCAQEAWQtTvZKGEacke+1bMc8d +H2xwxbhuvk679r6XUOEwf7ooXGKUwuN+M/f7QnaF25UcjCJYdQkMiGVnOQoWCcWg +OJekxSOTP7QYpgEGRJHjp2kntFolfzq3Ms3dhP8qOCkzpN1nsoX+oYggHFCJyNwq +9kIDN0zmiN/VryTyscPfzLXs4Jlet0lUIDyUGAzHHFIYSaRt4bNYC8nY7NmuHDKO +KHAN4v6mF56ED71XcLNa6R+ghlO773z/aQvgSMO3kwvIClTErF0UZzdsyqUvMQg3 +qm5vjLyb4lddJIGvl5echK1srDdMZvNhkREg5L4wn3qkKQmw4TRfZHcYQFHfjDCm +rw== -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIDJzCCApCgAwIBAgIBATANBgkqhkiG9w0BAQQFADCBzjELMAkGA1UEBhMCWkEx -FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYD -VQQKExRUaGF3dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlv -biBTZXJ2aWNlcyBEaXZpc2lvbjEhMB8GA1UEAxMYVGhhd3RlIFByZW1pdW0gU2Vy -dmVyIENBMSgwJgYJKoZIhvcNAQkBFhlwcmVtaXVtLXNlcnZlckB0aGF3dGUuY29t -MB4XDTk2MDgwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgc4xCzAJBgNVBAYTAlpB -MRUwEwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEdMBsG -A1UEChMUVGhhd3RlIENvbnN1bHRpbmcgY2MxKDAmBgNVBAsTH0NlcnRpZmljYXRp -b24gU2VydmljZXMgRGl2aXNpb24xITAfBgNVBAMTGFRoYXd0ZSBQcmVtaXVtIFNl -cnZlciBDQTEoMCYGCSqGSIb3DQEJARYZcHJlbWl1bS1zZXJ2ZXJAdGhhd3RlLmNv -bTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA0jY2aovXwlue2oFBYo847kkE -VdbQ7xwblRZH7xhINTpS9CtqBo87L+pW46+GjZ4X9560ZXUCTe/LCaIhUdib0GfQ -ug2SBhRz1JPLlyoAnFxODLz6FVL88kRu2hFKbgifLy3j+ao6hnO2RlNYyIkFvYMR -uHM/qgeN9EJN50CdHDcCAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG -9w0BAQQFAAOBgQAmSCwWwlj66BZ0DKqqX1Q/8tfJeGBeXm43YyJ3Nn6yF8Q0ufUI -hfzJATj/Tb7yFkJD57taRvvBxhEf8UqwKEbJw8RCfbz6q1lu1bdRiBHjpIUZa4JM -pAwSremkrj/xw0llmozFyD4lt5SZu5IycQfwhl7tUCemDaYj+bvLpgcUQg== +MIIEADCCAuigAwIBAgIBADANBgkqhkiG9w0BAQUFADBjMQswCQYDVQQGEwJVUzEh +MB8GA1UEChMYVGhlIEdvIERhZGR5IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBE +YWRkeSBDbGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTA0MDYyOTE3 +MDYyMFoXDTM0MDYyOTE3MDYyMFowYzELMAkGA1UEBhMCVVMxITAfBgNVBAoTGFRo +ZSBHbyBEYWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28gRGFkZHkgQ2xhc3Mg +MiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASAwDQYJKoZIhvcNAQEBBQADggEN +ADCCAQgCggEBAN6d1+pXGEmhW+vXX0iG6r7d/+TvZxz0ZWizV3GgXne77ZtJ6XCA +PVYYYwhv2vLM0D9/AlQiVBDYsoHUwHU9S3/Hd8M+eKsaA7Ugay9qK7HFiH7Eux6w +wdhFJ2+qN1j3hybX2C32qRe3H3I2TqYXP2WYktsqbl2i/ojgC95/5Y0V4evLOtXi +EqITLdiOr18SPaAIBQi2XKVlOARFmR6jYGB0xUGlcmIbYsUfb18aQr4CUWWoriMY +avx4A6lNf4DD+qta/KFApMoZFv6yyO9ecw3ud72a9nmYvLEHZ6IVDd2gWMZEewo+ +YihfukEHU1jPEX44dMX4/7VpkI+EdOqXG68CAQOjgcAwgb0wHQYDVR0OBBYEFNLE +sNKR1EwRcbNhyz2h/t2oatTjMIGNBgNVHSMEgYUwgYKAFNLEsNKR1EwRcbNhyz2h +/t2oatTjoWekZTBjMQswCQYDVQQGEwJVUzEhMB8GA1UEChMYVGhlIEdvIERhZGR5 +IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBEYWRkeSBDbGFzcyAyIENlcnRpZmlj +YXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQAD +ggEBADJL87LKPpH8EsahB4yOd6AzBhRckB4Y9wimPQoZ+YeAEW5p5JYXMP80kWNy +OO7MHAGjHZQopDH2esRU1/blMVgDoszOYtuURXO1v0XJJLXVggKtI3lpjbi2Tc7P +TMozI+gciKqdi0FuFskg5YmezTvacPd+mSYgFFQlq25zheabIZ0KbIIOqPjCDPoQ +HmyW74cNxA9hi63ugyuV+I6ShHI56yDqg+2DzZduCLzrTia2cyvk0/ZM/iZx4mER +dEr/VxqHD3VILs9RaRegAhJhldXRQLIQTO7ErBBDpqWeCtWVYpoNz4iCxTIM5Cuf +ReYNnyicsbkqWletNw+vHX/bvZ8= -----END CERTIFICATE----- From 8303f23302d3698f09598eb43c4f25fbe1fb52d0 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Mon, 11 Jun 2018 14:44:27 -0500 Subject: [PATCH 089/140] release: prepare release 0.4.3 Signed-off-by: Brandon Moser --- devicecloud/version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/devicecloud/version.py b/devicecloud/version.py index 36e50c1..ba081ef 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -4,4 +4,4 @@ # # Copyright (c) 2015 Digi International, Inc. -__version__ = "0.4.2" +__version__ = "0.4.3" diff --git a/setup.py b/setup.py index 34b9629..8f18c14 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ def get_long_description(): long_description=get_long_description(), url="https://github.com/digidotcom/python-devicecloud", author="Digi International, Inc.", - author_email="paul.osborne@digi.com", + author_email="brandon.moser@digi.com", packages=find_packages(), install_requires=open('requirements.txt').read().split(), classifiers=[ From 13c3703a5be5e2c3b201df882f845d92cc09f2b2 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Thu, 19 Jul 2018 16:12:29 -0500 Subject: [PATCH 090/140] update changelog to include notes for 0.4.3 --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23fa9a8..889f893 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ ## Python Devicecloud Library Changelog +### 0.4.3 / 2018-06-11 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.4.2...0.4.3) + +Bug Fixes: + +* core: updated Device Cloud CRT to latest +* core: updated requirements to latest + ### 0.4.2 / 2015-11-20 [Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.4.1...0.4.2) From 23f7773d1399faaec19e94a8cd897e87ce9e2383 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Fri, 20 Jul 2018 15:44:38 -0500 Subject: [PATCH 091/140] Add support to add and remove tags --- devicecloud/devicecore.py | 47 ++++++++++++++++++++-- devicecloud/test/unit/test_devicecore.py | 51 ++++++++++++++++++------ 2 files changed, 82 insertions(+), 16 deletions(-) diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index 8a66b61..a63d62a 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -20,13 +20,21 @@ ADD_GROUP_TEMPLATE = \ -""" + """ {connectware_id} {group_path} """ +TAGS_TEMPLATE = \ + """ + + {connectware_id} + {tags} + +""" + class DeviceCoreAPI(APIBase): """Encapsulate DeviceCore interface""" @@ -383,7 +391,6 @@ class Device(object): """Interface to a device in the device cloud""" # TODO: provide ability to set/update available data items - # TODO: add/remove tags def __init__(self, conn, sci, device_json): self._conn = conn @@ -587,4 +594,38 @@ def remove_from_group(self): self._conn.put('/ws/DeviceCore', post_data) # Invalidate cache - self._device_json = None \ No newline at end of file + self._device_json = None + + def add_tag(self, tag): + """Add a tag to existing device tags + + :param tag: the tag to be added + """ + + tags = self.get_tags() + tags.append(tag) + + post_data = TAGS_TEMPLATE.format(connectware_id=self.get_connectware_id(), + tags=",".join(tags)) + self._conn.put('/ws/DeviceCore', post_data) + + # Invalidate cache + self._device_json = None + + def remove_tag(self, tag): + """Remove tag from existing device tags + + :param tag: the tag to be removed from the list + + :raises ValueError: If tag does not exist in list + """ + + tags = self.get_tags() + tags.remove(tag) + + post_data = TAGS_TEMPLATE.format(connectware_id=self.get_connectware_id(), + tags=",".join(tags)) + self._conn.put('/ws/DeviceCore', post_data) + + # Invalidate cache + self._device_json = None diff --git a/devicecloud/test/unit/test_devicecore.py b/devicecloud/test/unit/test_devicecore.py index 0c63a85..ad8a64c 100644 --- a/devicecloud/test/unit/test_devicecore.py +++ b/devicecloud/test/unit/test_devicecore.py @@ -12,7 +12,7 @@ from devicecloud.devicecore import dev_mac, group_id from devicecloud.test.unit.test_utilities import HttpTestBase import httpretty -from devicecloud.devicecore import ADD_GROUP_TEMPLATE +from devicecloud.devicecore import ADD_GROUP_TEMPLATE, TAGS_TEMPLATE import six import mock @@ -172,6 +172,7 @@ """ + class TestDeviceCoreGroups(HttpTestBase): def test_get_groups(self): @@ -205,7 +206,7 @@ def test_repr_and_tree_print(self): self.assertEqual(len(fobj.getvalue()), 471) # no u'' on repr for strings def test_get_groups_condition(self): - self.prepare_response("GET", "/ws/Group", EXAMPLE_GET_GROUPS) + self.prepare_response("GET", "/ws/Group", EXAMPLE_GET_GROUPS) list(self.dc.devicecore.get_groups(group_id == "123")) params = self._get_last_request_params() self.assertEqual(params["condition"], "grpId='123'") @@ -242,7 +243,7 @@ def test_provision_imei(self): res = self.dc.devicecore.provision_device(imei="990000862471854") req = self._get_last_request() self.assertEqual(req.body, six.b( - "" + "" "" "990000862471854" "" @@ -283,15 +284,15 @@ def test_provision_multiple_simple(self): ]) req = self._get_last_request() self.assertEqual(req.body, six.b( - '' - '' - '00000000-00000000-0000DEFF-FFADBEEFF' - '' - '' - '' - 'DE:AD:BE:EF:00:00' - '' - '')) + '' + '' + '00000000-00000000-0000DEFF-FFADBEEFF' + '' + '' + '' + 'DE:AD:BE:EF:00:00' + '' + '')) self.assertTrue(len(res), 2) self.assertDictEqual(res[0], {"error": False, "error_msg": None, "location": "DeviceCore/1397876/0"}) self.assertDictEqual(res[1], {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) @@ -398,7 +399,7 @@ def test_dc_get_devices(self): self.assertEqual(dev1.get_last_known_ip(), '10.35.1.107') self.assertEqual(dev1.get_global_ip(), '204.182.3.237') self.assertEqual(dev1.get_last_connected_dt(), - datetime.datetime(2013, 4, 8, 4, 1, 20, 633, tzinfo=tzutc())) + datetime.datetime(2013, 4, 8, 4, 1, 20, 633, tzinfo=tzutc())) self.assertEqual(dev1.get_contact(), '') self.assertEqual(dev1.get_description(), '') self.assertEqual(dev1.get_location(), '') @@ -465,5 +466,29 @@ def test_remove_device_from_group(self): self.assertIsNone(dev._device_json) self.assertEqual(six.b(expected), httpretty.last_request().body) + def test_add_device_tag(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + expected = TAGS_TEMPLATE.format(connectware_id=dev.get_connectware_id(), + tags='test') + dev.add_tag('test') + self.assertIsNone(dev._device_json) + self.assertEqual(six.b(expected), httpretty.last_request().body) + + def test_remove_device_tag(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + try: + dev.remove_tag('test') + except ValueError: + pass + else: + assert False, "should have thrown exception" + + if __name__ == '__main__': unittest.main() From e7a21a09397535644c53ebb771f44c0a087a27fc Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Fri, 20 Jul 2018 15:47:10 -0500 Subject: [PATCH 092/140] Add python versions 3.5, 3.6, 3.7 to test suite --- .travis.yml | 3 +++ setup.py | 3 +++ tox.ini | 2 +- toxtest.sh | 3 +++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4d73613..8679145 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,9 @@ env: - TOX_ENV=py32 - TOX_ENV=py33 - TOX_ENV=py34 + - TOX_ENV=py35 + - TOX_ENV=py36 + - TOX_ENV=py37 - TOX_ENV=pypy - TOX_ENV=coverage diff --git a/setup.py b/setup.py index 9e2009c..aa69f27 100644 --- a/setup.py +++ b/setup.py @@ -66,6 +66,9 @@ def get_long_description(): "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "Topic :: Software Development :: Libraries", "Operating System :: OS Independent", ], diff --git a/tox.ini b/tox.ini index 8b329df..86deb32 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py33,py34,pypy +envlist = py27,py33,py34,py35,py36,py37,pypy [testenv] passenv = * diff --git a/toxtest.sh b/toxtest.sh index 1d1d72f..ec71a56 100755 --- a/toxtest.sh +++ b/toxtest.sh @@ -9,6 +9,9 @@ pyversions=(2.7.7 3.3.5 3.4.3 + 3.5.5 + 3.6.6 + 3.7.0 pypy-2.3.1) # first make sure that pyenv is installed From 885ec9fb919d5333fd8072a6a0e3e790b5cc03ab Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Fri, 20 Jul 2018 15:47:33 -0500 Subject: [PATCH 093/140] Fix tests --- devicecloud/test/unit/test_devicecore.py | 988 +++++++++++------------ devicecloud/test/unit/test_filedata.py | 8 +- devicecloud/test/unit/test_streams.py | 11 +- 3 files changed, 504 insertions(+), 503 deletions(-) diff --git a/devicecloud/test/unit/test_devicecore.py b/devicecloud/test/unit/test_devicecore.py index ad8a64c..a86b756 100644 --- a/devicecloud/test/unit/test_devicecore.py +++ b/devicecloud/test/unit/test_devicecore.py @@ -1,494 +1,494 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -# -# Copyright (c) 2015 Digi International, Inc. -import copy -import datetime -import unittest - -from dateutil.tz import tzutc -from devicecloud import DeviceCloudHttpException -from devicecloud.devicecore import dev_mac, group_id -from devicecloud.test.unit.test_utilities import HttpTestBase -import httpretty -from devicecloud.devicecore import ADD_GROUP_TEMPLATE, TAGS_TEMPLATE -import six -import mock - - -EXAMPLE_GET_DEVICES = { - "resultTotalRows": "2", - "requestedStartRow": "0", - "resultSize": "2", - "requestedSize": "1000", - "remainingSize": "0", - "items": [ - { - "id": { - "devId": "702077", - "devVersion": "6" - }, - "devRecordStartDate": "2013-02-28T19:54:00.000Z", - "devMac": "00:40:9D:58:17:5B", - "devCellularModemId": "354374042391400", - "devConnectwareId": "00000000-00000000-00409DFF-FF58175B", - "cstId": "1872", - "grpId": "2331", - "devEffectiveStartDate": "2013-02-28T19:53:00.000Z", - "devTerminated": "false", - "dvVendorId": "4261412864", - "dpDeviceType": "ConnectPort X5 R", - "dpFirmwareLevel": "34537482", - "dpFirmwareLevelDesc": "2.15.0.10", - "dpRestrictedStatus": "0", - "dpLastKnownIp": "10.35.1.107", - "dpGlobalIp": "204.182.3.237", - "dpConnectionStatus": "0", - "dpLastConnectTime": "2013-04-08T04:01:20.633Z", - "dpContact": "", - "dpDescription": "", - "dpLocation": "", - "dpMapLat": "34.964465", - "dpMapLong": "40.268198", - "dpServerId": "", - "dpZigbeeCapabilities": "0", - "dpCapabilities": "6707", - "grpPath": "", - "dpLastDisconnectTime": "2013-04-16T19:46:06.557Z" - }, - { - "id": { - "devId": "714038", - "devVersion": "7" - }, - "devRecordStartDate": "2013-07-16T18:05:00.000Z", - "devMac": "00:1d:09:2b:7d:8c", - "devConnectwareId": "00000000-00000000-001D09FF-FF2B7D8C", - "cstId": "1872", - "grpId": "2331", - "devEffectiveStartDate": "2013-07-16T18:05:00.000Z", - "devTerminated": "false", - "dvVendorId": "50331982", - "dpDeviceType": "IntelligentSystem", - "dpFirmwareLevel": "0", - "dpRestrictedStatus": "0", - "dpLastKnownIp": "10.35.1.113", - "dpGlobalIp": "204.182.3.238", - "dpConnectionStatus": "0", - "dpLastConnectTime": "2013-07-24T00:40:20.363Z", - "dpServerId": "", - "dpCapabilities": "66114", - "grpPath": "", - "dpLastDisconnectTime": "2013-07-24T00:40:36.537Z" - } - ] -} - - -GET_DEVICES_PAGE1 = """\ -{ - "resultTotalRows": "2", - "requestedStartRow": "0", - "resultSize": "1", - "requestedSize": "1", - "remainingSize": "1", - "items": [ - {"id": {"devId": "702077","devVersion": "6"},"devRecordStartDate": "2013-02-28T19:54:00.000Z","devMac": "00:40:9D:58:17:5B","devCellularModemId": "354374042391400","devConnectwareId": "00000000-00000000-00409DFF-FF58175B","cstId": "1872","grpId": "2331","devEffectiveStartDate": "2013-02-28T19:53:00.000Z","devTerminated": "false","dvVendorId": "4261412864","dpDeviceType": "ConnectPort X5 R","dpFirmwareLevel": "34537482","dpFirmwareLevelDesc": "2.15.0.10","dpRestrictedStatus": "0","dpLastKnownIp": "10.35.1.107","dpGlobalIp": "204.182.3.237","dpConnectionStatus": "0","dpLastConnectTime": "2013-04-08T04:01:20.633Z","dpContact": "","dpDescription": "","dpLocation": "","dpMapLat": "34.964465","dpMapLong": "40.268198","dpServerId": "","dpZigbeeCapabilities": "0","dpCapabilities": "6707","grpPath": "","dpLastDisconnectTime": "2013-04-16T19:46:06.557Z"} - ] - } -""" - -GET_DEVICES_PAGE2 = """\ -{ - "resultTotalRows": "2", - "requestedStartRow": "1", - "resultSize": "1", - "requestedSize": "1", - "remainingSize": "0", - "items": [ - {"id": {"devId": "702078","devVersion": "6"},"devRecordStartDate": "2013-02-28T19:54:00.000Z","devMac": "00:40:9D:58:17:5B","devCellularModemId": "354374042391400","devConnectwareId": "00000000-00000000-00409DFF-FF58175B","cstId": "1872","grpId": "2331","devEffectiveStartDate": "2013-02-28T19:53:00.000Z","devTerminated": "false","dvVendorId": "4261412864","dpDeviceType": "ConnectPort X5 R","dpFirmwareLevel": "34537482","dpFirmwareLevelDesc": "2.15.0.10","dpRestrictedStatus": "0","dpLastKnownIp": "10.35.1.107","dpGlobalIp": "204.182.3.237","dpConnectionStatus": "0","dpLastConnectTime": "2013-04-08T04:01:20.633Z","dpContact": "","dpDescription": "","dpLocation": "","dpMapLat": "34.964465","dpMapLong": "40.268198","dpServerId": "","dpZigbeeCapabilities": "0","dpCapabilities": "6707","grpPath": "","dpLastDisconnectTime": "2013-04-16T19:46:06.557Z"} - ] - } -""" - -EXAMPLE_GET_GROUPS = """\ -{ - "resultTotalRows": "2", - "requestedStartRow": "0", - "resultSize": "2", - "requestedSize": "1000", - "remainingSize": "0", - "items": [ - { "grpId": "11817", "grpName": "7603_Digi", "grpDescription": "7603_Digi root group", "grpPath": "\/7603_Digi\/", "grpParentId": "1"}, - { "grpId": "13542", "grpName": "Demo", "grpPath": "\/7603_Digi\/Demo\/", "grpParentId": "11817"} - ] -} -""" - -EXAMPLE_GET_GROUPS_EXTENDED = """\ -{ - "resultTotalRows": "4", - "requestedStartRow": "0", - "resultSize": "4", - "requestedSize": "1000", - "remainingSize": "0", - "items": [ - { "grpId": "11817", "grpName": "7603_Digi", "grpDescription": "7603_Digi root group", "grpPath": "\/7603_Digi\/", "grpParentId": "1"}, - { "grpId": "13542", "grpName": "Demo", "grpPath": "\/7603_Digi\/Demo\/", "grpParentId": "11817"}, - { "grpId": "13544", "grpName": "SubDir2", "grpPath": "\/7603_Digi\/Demo\/SubDir2\/", "grpParentId": "13542"}, - { "grpId": "13545", "grpName": "Another Second Level", "grpDescription": "Another Second Level", "grpPath": "\/7603_Digi\/Another Second Level\/", "grpParentId": "11817"} - ] -} -""" - -PROVISION_SUCCESS_RESPONSE1 = """\ - - - DeviceCore/946246/0 - -""" - -PROVISION_MULTIPLE_SUCCESS_RESPONSE1 = """\ - - - DeviceCore/1397876/0 - DeviceCore/946246/0 - -""" - -PROVISION_ERROR1 = """\ - - - The device 00000000-00000000-BC5FF4FF-FFF7908A is already provisioned. - -""" - -PROVISION_MIXED_RESULT_RESPONSE = """\ - - - DeviceCore/1397876/0 - The device 00000000-00000000-D48564FF-FF9D4FEE is already provisioned. - -""" - - -class TestDeviceCoreGroups(HttpTestBase): - - def test_get_groups(self): - self.prepare_response("GET", "/ws/Group", EXAMPLE_GET_GROUPS) - it = self.dc.devicecore.get_groups() - - grp = six.next(it) - self.assertEqual(grp.is_root(), True) - self.assertEqual(grp.get_id(), "11817") - self.assertEqual(grp.get_name(), "7603_Digi") - self.assertEqual(grp.get_description(), "7603_Digi root group") - self.assertEqual(grp.get_path(), "/7603_Digi/") - self.assertEqual(grp.get_parent_id(), "1") - - grp = six.next(it) - self.assertEqual(grp.is_root(), False) - self.assertEqual(grp.get_id(), "13542") - self.assertEqual(grp.get_name(), "Demo") - self.assertEqual(grp.get_description(), "") - self.assertEqual(grp.get_path(), "/7603_Digi/Demo/") - self.assertEqual(grp.get_parent_id(), "11817") - - def test_repr_and_tree_print(self): - self.prepare_response("GET", "/ws/Group", EXAMPLE_GET_GROUPS_EXTENDED) - fobj = six.StringIO() - root = self.dc.devicecore.get_group_tree_root() - root.print_subtree(fobj) # the order of the traversal can vary, so just assert on the length - if six.PY2: - self.assertEqual(len(fobj.getvalue()), 489) - elif six.PY3: - self.assertEqual(len(fobj.getvalue()), 471) # no u'' on repr for strings - - def test_get_groups_condition(self): - self.prepare_response("GET", "/ws/Group", EXAMPLE_GET_GROUPS) - list(self.dc.devicecore.get_groups(group_id == "123")) - params = self._get_last_request_params() - self.assertEqual(params["condition"], "grpId='123'") - - -class TestDeviceCoreProvisioning(HttpTestBase): - - def test_provision_one_simple_device_id(self): - self.prepare_response("POST", "/ws/DeviceCore", PROVISION_SUCCESS_RESPONSE1, status=207) - res = self.dc.devicecore.provision_device(device_id='00000000-00000000-0000DEFF-FFADBEEFF') - req = self._get_last_request() - self.assertEqual(req.body, six.b( - "" - "" - "00000000-00000000-0000DEFF-FFADBEEFF" - "" - "")) - self.assertDictEqual(res, {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) - - def test_provision_one_simple_mac(self): - self.prepare_response("POST", "/ws/DeviceCore", PROVISION_SUCCESS_RESPONSE1, status=207) - res = self.dc.devicecore.provision_device(mac_address="DE:AD:BE:EF:00:00") - req = self._get_last_request() - self.assertEqual(req.body, six.b( - "" - "" - "DE:AD:BE:EF:00:00" - "" - "")) - self.assertDictEqual(res, {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) - - def test_provision_imei(self): - self.prepare_response("POST", "/ws/DeviceCore", PROVISION_SUCCESS_RESPONSE1, status=207) - res = self.dc.devicecore.provision_device(imei="990000862471854") - req = self._get_last_request() - self.assertEqual(req.body, six.b( - "" - "" - "990000862471854" - "" - "")) - self.assertDictEqual(res, {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) - - def test_provision_all_the_fixins(self): - self.prepare_response("POST", "/ws/DeviceCore", PROVISION_SUCCESS_RESPONSE1, status=207) - res = self.dc.devicecore.provision_device( - mac_address="DE:AD:BE:EF:00:00", - group_path="/group/path", - metadata="Sweet, sweet metadata", - map_lat=44.9807496, - map_long=-93.1397815, - contact="Saint Paul Parks Department", - description="Buried Treasure", - ) - req = self._get_last_request() - self.assertEqual(req.body, six.b( - '' - '' - 'DE:AD:BE:EF:00:00' - '/group/path' - 'Sweet, sweet metadata' - '-93.1397815' - '44.9807496' - 'Saint Paul Parks Department' - 'Buried Treasure' - '' - '')) - self.assertDictEqual(res, {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) - - def test_provision_multiple_simple(self): - self.prepare_response("POST", "/ws/DeviceCore", PROVISION_MULTIPLE_SUCCESS_RESPONSE1, status=207) - res = self.dc.devicecore.provision_devices([ - {'device_id': "00000000-00000000-0000DEFF-FFADBEEFF"}, - {'mac_address': 'DE:AD:BE:EF:00:00'} - ]) - req = self._get_last_request() - self.assertEqual(req.body, six.b( - '' - '' - '00000000-00000000-0000DEFF-FFADBEEFF' - '' - '' - '' - 'DE:AD:BE:EF:00:00' - '' - '')) - self.assertTrue(len(res), 2) - self.assertDictEqual(res[0], {"error": False, "error_msg": None, "location": "DeviceCore/1397876/0"}) - self.assertDictEqual(res[1], {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) - - def test_without_required_param(self): - self.assertRaises(ValueError, self.dc.devicecore.provision_device, description="I should not work") - - def test_bad_request_400(self): - self.prepare_response('POST', '/ws/DeviceCore', 'Bad Request', status=400) - self.assertRaises(DeviceCloudHttpException, - self.dc.devicecore.provision_device, mac_address="DE:AD:BE:EF:00:00") - - def test_bad_request_500(self): - self.prepare_response('POST', '/ws/DeviceCore', 'Bad Request', status=500) - self.assertRaises(DeviceCloudHttpException, - self.dc.devicecore.provision_device, mac_address="DE:AD:BE:EF:00:00") - - def test_error_response(self): - self.prepare_response("POST", "/ws/DeviceCore", PROVISION_ERROR1, status=207) - res = self.dc.devicecore.provision_device(imei="990000862471854") - self.assertDictEqual(res, { - "error": True, - "error_msg": 'The device 00000000-00000000-BC5FF4FF-FFF7908A is already provisioned.', - "location": None} - ) - - def test_mixed_error_success_response(self): - self.prepare_response("POST", "/ws/DeviceCore", PROVISION_MIXED_RESULT_RESPONSE, status=207) - res = self.dc.devicecore.provision_devices([ - {'device_id': "00000000-00000000-0000DEFF-FFADBEEFF"}, - {'mac_address': 'DE:AD:BE:EF:00:00'} - ]) - self.assertTrue(len(res), 2) - self.assertDictEqual(res[0], { - 'error': False, - 'error_msg': None, - 'location': 'DeviceCore/1397876/0', - }) - self.assertDictEqual(res[1], { - 'error': True, - 'error_msg': 'The device 00000000-00000000-D48564FF-FF9D4FEE is already provisioned.', - 'location': None, - }) - - -class TestDeviceCoreDeleting(HttpTestBase): - - def test_delete_device_good(self): - fake_device = mock.MagicMock() - fake_device.get_device_id.return_value = '1234' - self.prepare_response("DELETE", "/ws/DeviceCore/1234", "1 items deleted", status=200) - self.dc.devicecore.delete_device(fake_device) - req = self._get_last_request() - self.assertEqual(req.path, "/ws/DeviceCore/1234") - - def test_delete_device_not_exist(self): - fake_device = mock.MagicMock() - fake_device.get_device_id.return_value = '1234' - self.prepare_response("DELETE", "/ws/DeviceCore/1234", "0 items deleted", status=200) - self.dc.devicecore.delete_device(fake_device) - req = self._get_last_request() - self.assertEqual(req.path, "/ws/DeviceCore/1234") - - def test_delete_device_bad_status(self): - fake_device = mock.MagicMock() - fake_device.get_device_id.return_value = '1234' - self.prepare_response("DELETE", "/ws/DeviceCore/1234", "I pity da foo' who don' know about API changes.", status=400) - try: - self.dc.devicecore.delete_device(fake_device) - except DeviceCloudHttpException: - pass - else: - assert False, "should have thrown exception" - req = self._get_last_request() - self.assertEqual(req.path, "/ws/DeviceCore/1234") - - -class TestDeviceCoreDevices(HttpTestBase): - - def test_dc_get_devices(self): - self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - devices = self.dc.devicecore.get_devices() - dev1 = six.next(devices) - dev2 = six.next(devices) - self.assertRaises(StopIteration, six.next, devices) - - self.assertEqual(dev1.get_mac(), "00:40:9D:58:17:5B") - self.assertEqual(dev1.get_mac_last4(), "175B") - self.assertEqual(dev1.get_device_id(), "702077") - self.assertEqual(dev1.get_connectware_id(), "00000000-00000000-00409DFF-FF58175B") - self.assertEqual(dev1.get_ip(), "10.35.1.107") - self.assertEqual(dev1.get_tags(), []) - self.assertEqual(dev1.get_registration_dt(), - datetime.datetime(2013, 2, 28, 19, 54, tzinfo=tzutc())) - self.assertEqual(dev1.get_meid(), '354374042391400') - self.assertEqual(dev1.get_customer_id(), '1872') - self.assertEqual(dev1.get_group_id(), '2331') - self.assertEqual(dev1.get_group_path(), '') - self.assertEqual(dev1.get_vendor_id(), '4261412864') - self.assertEqual(dev1.get_device_type(), 'ConnectPort X5 R') - self.assertEqual(dev1.get_firmware_level(), '34537482') - self.assertEqual(dev1.get_firmware_level_description(), '2.15.0.10') - self.assertEqual(dev1.get_restricted_status(), "0") - self.assertEqual(dev1.get_last_known_ip(), '10.35.1.107') - self.assertEqual(dev1.get_global_ip(), '204.182.3.237') - self.assertEqual(dev1.get_last_connected_dt(), - datetime.datetime(2013, 4, 8, 4, 1, 20, 633, tzinfo=tzutc())) - self.assertEqual(dev1.get_contact(), '') - self.assertEqual(dev1.get_description(), '') - self.assertEqual(dev1.get_location(), '') - self.assertEqual(dev1.get_latlon(), (34.964465, 40.268198)) - self.assertEqual(dev1.get_user_metadata(), None) - self.assertEqual(dev1.get_zb_pan_id(), None) - self.assertEqual(dev1.get_zb_extended_address(), None) - self.assertEqual(dev1.get_server_id(), '') - self.assertEqual(dev1.get_provision_id(), None) - self.assertEqual(dev1.get_current_connect_pw(), None) - - def test_dc_get_devices_paged(self): - self.prepare_response("GET", "/ws/DeviceCore", GET_DEVICES_PAGE1) - gen = self.dc.devicecore.get_devices(page_size=1) - dev1 = six.next(gen) - self.prepare_response("GET", "/ws/DeviceCore", GET_DEVICES_PAGE2) - dev2 = six.next(gen) - self.assertRaises(StopIteration, six.next, gen) - self.assertEqual(dev1.get_device_id(), '702077') - self.assertEqual(dev2.get_device_id(), '702078') - - def test_dc_get_devices_with_condition(self): - self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - gen = self.dc.devicecore.get_devices(dev_mac == 'xx:xx:xx:xx:xx', page_size=1) - six.next(gen) - qs = httpretty.last_request().querystring - self.assertEqual(qs['condition'][0], "devMac='xx:xx:xx:xx:xx'") - self.assertEqual(qs['size'][0], "1") - self.assertEqual(qs['embed'][0], "true") - self.assertEqual(qs['start'][0], "0") - - def test_refresh_from_cache(self): - get_devices_update = copy.deepcopy(EXAMPLE_GET_DEVICES) - get_devices_update["items"][0]["dpDeviceType"] = "Turboencabulator" - del get_devices_update["items"][1] # remove the other item... close enough - self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - devices = self.dc.devicecore.get_devices() - device = six.next(devices) - self.prepare_json_response("GET", "/ws/DeviceCore/702077", get_devices_update) - self.assertEqual(device.get_device_type(), "ConnectPort X5 R") - self.assertEqual(device.get_device_type(False), "Turboencabulator") - self.assertEqual(device.get_device_type(), "Turboencabulator") # make sure cache updated - - def test_add_device_to_group(self): - self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - self.prepare_response("PUT", "/ws/DeviceCore", '') - gen = self.dc.devicecore.get_devices(page_size=1) - dev = six.next(gen) - expected = ADD_GROUP_TEMPLATE.format(connectware_id=dev.get_connectware_id(), - group_path='testgrp') - dev.add_to_group('testgrp') - self.assertIsNone(dev._device_json) - self.assertEqual(six.b(expected), httpretty.last_request().body) - - def test_remove_device_from_group(self): - self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - self.prepare_response("PUT", "/ws/DeviceCore", '') - gen = self.dc.devicecore.get_devices(page_size=1) - dev = six.next(gen) - dev.get_group_path = lambda: 'something other than empty string' - expected = ADD_GROUP_TEMPLATE.format(connectware_id=dev.get_connectware_id(), - group_path='') - dev.remove_from_group() - self.assertIsNone(dev._device_json) - self.assertEqual(six.b(expected), httpretty.last_request().body) - - def test_add_device_tag(self): - self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - self.prepare_response("PUT", "/ws/DeviceCore", '') - gen = self.dc.devicecore.get_devices(page_size=1) - dev = six.next(gen) - expected = TAGS_TEMPLATE.format(connectware_id=dev.get_connectware_id(), - tags='test') - dev.add_tag('test') - self.assertIsNone(dev._device_json) - self.assertEqual(six.b(expected), httpretty.last_request().body) - - def test_remove_device_tag(self): - self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) - self.prepare_response("PUT", "/ws/DeviceCore", '') - gen = self.dc.devicecore.get_devices(page_size=1) - dev = six.next(gen) - try: - dev.remove_tag('test') - except ValueError: - pass - else: - assert False, "should have thrown exception" - - -if __name__ == '__main__': - unittest.main() +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015 Digi International, Inc. +import copy +import datetime +import unittest + +from dateutil.tz import tzutc +from devicecloud import DeviceCloudHttpException +from devicecloud.devicecore import dev_mac, group_id +from devicecloud.test.unit.test_utilities import HttpTestBase +import httpretty +from devicecloud.devicecore import ADD_GROUP_TEMPLATE, TAGS_TEMPLATE +import six +import mock + + +EXAMPLE_GET_DEVICES = { + "resultTotalRows": "2", + "requestedStartRow": "0", + "resultSize": "2", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { + "id": { + "devId": "702077", + "devVersion": "6" + }, + "devRecordStartDate": "2013-02-28T19:54:00.000Z", + "devMac": "00:40:9D:58:17:5B", + "devCellularModemId": "354374042391400", + "devConnectwareId": "00000000-00000000-00409DFF-FF58175B", + "cstId": "1872", + "grpId": "2331", + "devEffectiveStartDate": "2013-02-28T19:53:00.000Z", + "devTerminated": "false", + "dvVendorId": "4261412864", + "dpDeviceType": "ConnectPort X5 R", + "dpFirmwareLevel": "34537482", + "dpFirmwareLevelDesc": "2.15.0.10", + "dpRestrictedStatus": "0", + "dpLastKnownIp": "10.35.1.107", + "dpGlobalIp": "204.182.3.237", + "dpConnectionStatus": "0", + "dpLastConnectTime": "2013-04-08T04:01:20.633Z", + "dpContact": "", + "dpDescription": "", + "dpLocation": "", + "dpMapLat": "34.964465", + "dpMapLong": "40.268198", + "dpServerId": "", + "dpZigbeeCapabilities": "0", + "dpCapabilities": "6707", + "grpPath": "", + "dpLastDisconnectTime": "2013-04-16T19:46:06.557Z" + }, + { + "id": { + "devId": "714038", + "devVersion": "7" + }, + "devRecordStartDate": "2013-07-16T18:05:00.000Z", + "devMac": "00:1d:09:2b:7d:8c", + "devConnectwareId": "00000000-00000000-001D09FF-FF2B7D8C", + "cstId": "1872", + "grpId": "2331", + "devEffectiveStartDate": "2013-07-16T18:05:00.000Z", + "devTerminated": "false", + "dvVendorId": "50331982", + "dpDeviceType": "IntelligentSystem", + "dpFirmwareLevel": "0", + "dpRestrictedStatus": "0", + "dpLastKnownIp": "10.35.1.113", + "dpGlobalIp": "204.182.3.238", + "dpConnectionStatus": "0", + "dpLastConnectTime": "2013-07-24T00:40:20.363Z", + "dpServerId": "", + "dpCapabilities": "66114", + "grpPath": "", + "dpLastDisconnectTime": "2013-07-24T00:40:36.537Z" + } + ] +} + + +GET_DEVICES_PAGE1 = """\ +{ + "resultTotalRows": "2", + "requestedStartRow": "0", + "resultSize": "1", + "requestedSize": "1", + "remainingSize": "1", + "items": [ + {"id": {"devId": "702077","devVersion": "6"},"devRecordStartDate": "2013-02-28T19:54:00.000Z","devMac": "00:40:9D:58:17:5B","devCellularModemId": "354374042391400","devConnectwareId": "00000000-00000000-00409DFF-FF58175B","cstId": "1872","grpId": "2331","devEffectiveStartDate": "2013-02-28T19:53:00.000Z","devTerminated": "false","dvVendorId": "4261412864","dpDeviceType": "ConnectPort X5 R","dpFirmwareLevel": "34537482","dpFirmwareLevelDesc": "2.15.0.10","dpRestrictedStatus": "0","dpLastKnownIp": "10.35.1.107","dpGlobalIp": "204.182.3.237","dpConnectionStatus": "0","dpLastConnectTime": "2013-04-08T04:01:20.633Z","dpContact": "","dpDescription": "","dpLocation": "","dpMapLat": "34.964465","dpMapLong": "40.268198","dpServerId": "","dpZigbeeCapabilities": "0","dpCapabilities": "6707","grpPath": "","dpLastDisconnectTime": "2013-04-16T19:46:06.557Z"} + ] + } +""" + +GET_DEVICES_PAGE2 = """\ +{ + "resultTotalRows": "2", + "requestedStartRow": "1", + "resultSize": "1", + "requestedSize": "1", + "remainingSize": "0", + "items": [ + {"id": {"devId": "702078","devVersion": "6"},"devRecordStartDate": "2013-02-28T19:54:00.000Z","devMac": "00:40:9D:58:17:5B","devCellularModemId": "354374042391400","devConnectwareId": "00000000-00000000-00409DFF-FF58175B","cstId": "1872","grpId": "2331","devEffectiveStartDate": "2013-02-28T19:53:00.000Z","devTerminated": "false","dvVendorId": "4261412864","dpDeviceType": "ConnectPort X5 R","dpFirmwareLevel": "34537482","dpFirmwareLevelDesc": "2.15.0.10","dpRestrictedStatus": "0","dpLastKnownIp": "10.35.1.107","dpGlobalIp": "204.182.3.237","dpConnectionStatus": "0","dpLastConnectTime": "2013-04-08T04:01:20.633Z","dpContact": "","dpDescription": "","dpLocation": "","dpMapLat": "34.964465","dpMapLong": "40.268198","dpServerId": "","dpZigbeeCapabilities": "0","dpCapabilities": "6707","grpPath": "","dpLastDisconnectTime": "2013-04-16T19:46:06.557Z"} + ] + } +""" + +EXAMPLE_GET_GROUPS = """\ +{ + "resultTotalRows": "2", + "requestedStartRow": "0", + "resultSize": "2", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { "grpId": "11817", "grpName": "7603_Digi", "grpDescription": "7603_Digi root group", "grpPath": "\/7603_Digi\/", "grpParentId": "1"}, + { "grpId": "13542", "grpName": "Demo", "grpPath": "\/7603_Digi\/Demo\/", "grpParentId": "11817"} + ] +} +""" + +EXAMPLE_GET_GROUPS_EXTENDED = """\ +{ + "resultTotalRows": "4", + "requestedStartRow": "0", + "resultSize": "4", + "requestedSize": "1000", + "remainingSize": "0", + "items": [ + { "grpId": "11817", "grpName": "7603_Digi", "grpDescription": "7603_Digi root group", "grpPath": "\/7603_Digi\/", "grpParentId": "1"}, + { "grpId": "13542", "grpName": "Demo", "grpPath": "\/7603_Digi\/Demo\/", "grpParentId": "11817"}, + { "grpId": "13544", "grpName": "SubDir2", "grpPath": "\/7603_Digi\/Demo\/SubDir2\/", "grpParentId": "13542"}, + { "grpId": "13545", "grpName": "Another Second Level", "grpDescription": "Another Second Level", "grpPath": "\/7603_Digi\/Another Second Level\/", "grpParentId": "11817"} + ] +} +""" + +PROVISION_SUCCESS_RESPONSE1 = """\ + + + DeviceCore/946246/0 + +""" + +PROVISION_MULTIPLE_SUCCESS_RESPONSE1 = """\ + + + DeviceCore/1397876/0 + DeviceCore/946246/0 + +""" + +PROVISION_ERROR1 = """\ + + + The device 00000000-00000000-BC5FF4FF-FFF7908A is already provisioned. + +""" + +PROVISION_MIXED_RESULT_RESPONSE = """\ + + + DeviceCore/1397876/0 + The device 00000000-00000000-D48564FF-FF9D4FEE is already provisioned. + +""" + + +class TestDeviceCoreGroups(HttpTestBase): + + def test_get_groups(self): + self.prepare_response("GET", "/ws/Group", EXAMPLE_GET_GROUPS) + it = self.dc.devicecore.get_groups() + + grp = six.next(it) + self.assertEqual(grp.is_root(), True) + self.assertEqual(grp.get_id(), "11817") + self.assertEqual(grp.get_name(), "7603_Digi") + self.assertEqual(grp.get_description(), "7603_Digi root group") + self.assertEqual(grp.get_path(), "/7603_Digi/") + self.assertEqual(grp.get_parent_id(), "1") + + grp = six.next(it) + self.assertEqual(grp.is_root(), False) + self.assertEqual(grp.get_id(), "13542") + self.assertEqual(grp.get_name(), "Demo") + self.assertEqual(grp.get_description(), "") + self.assertEqual(grp.get_path(), "/7603_Digi/Demo/") + self.assertEqual(grp.get_parent_id(), "11817") + + def test_repr_and_tree_print(self): + self.prepare_response("GET", "/ws/Group", EXAMPLE_GET_GROUPS_EXTENDED) + fobj = six.StringIO() + root = self.dc.devicecore.get_group_tree_root() + root.print_subtree(fobj) # the order of the traversal can vary, so just assert on the length + if six.PY2: + self.assertEqual(len(fobj.getvalue()), 489) + elif six.PY3: + self.assertEqual(len(fobj.getvalue()), 471) # no u'' on repr for strings + + def test_get_groups_condition(self): + self.prepare_response("GET", "/ws/Group", EXAMPLE_GET_GROUPS) + list(self.dc.devicecore.get_groups(group_id == "123")) + params = self._get_last_request_params() + self.assertEqual(params["condition"], "grpId='123'") + + +class TestDeviceCoreProvisioning(HttpTestBase): + + def test_provision_one_simple_device_id(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_SUCCESS_RESPONSE1, status=207) + res = self.dc.devicecore.provision_device(device_id='00000000-00000000-0000DEFF-FFADBEEFF') + req = self._get_last_request() + self.assertEqual(req.body, six.b( + "" + "" + "00000000-00000000-0000DEFF-FFADBEEFF" + "" + "")) + self.assertDictEqual(res, {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) + + def test_provision_one_simple_mac(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_SUCCESS_RESPONSE1, status=207) + res = self.dc.devicecore.provision_device(mac_address="DE:AD:BE:EF:00:00") + req = self._get_last_request() + self.assertEqual(req.body, six.b( + "" + "" + "DE:AD:BE:EF:00:00" + "" + "")) + self.assertDictEqual(res, {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) + + def test_provision_imei(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_SUCCESS_RESPONSE1, status=207) + res = self.dc.devicecore.provision_device(imei="990000862471854") + req = self._get_last_request() + self.assertEqual(req.body, six.b( + "" + "" + "990000862471854" + "" + "")) + self.assertDictEqual(res, {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) + + def test_provision_all_the_fixins(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_SUCCESS_RESPONSE1, status=207) + res = self.dc.devicecore.provision_device( + mac_address="DE:AD:BE:EF:00:00", + group_path="/group/path", + metadata="Sweet, sweet metadata", + map_lat=44.9807496, + map_long=-93.1397815, + contact="Saint Paul Parks Department", + description="Buried Treasure", + ) + req = self._get_last_request() + self.assertEqual(req.body, six.b( + '' + '' + 'DE:AD:BE:EF:00:00' + '/group/path' + 'Sweet, sweet metadata' + '-93.1397815' + '44.9807496' + 'Saint Paul Parks Department' + 'Buried Treasure' + '' + '')) + self.assertDictEqual(res, {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) + + def test_provision_multiple_simple(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_MULTIPLE_SUCCESS_RESPONSE1, status=207) + res = self.dc.devicecore.provision_devices([ + {'device_id': "00000000-00000000-0000DEFF-FFADBEEFF"}, + {'mac_address': 'DE:AD:BE:EF:00:00'} + ]) + req = self._get_last_request() + self.assertEqual(req.body, six.b( + '' + '' + '00000000-00000000-0000DEFF-FFADBEEFF' + '' + '' + '' + 'DE:AD:BE:EF:00:00' + '' + '')) + self.assertTrue(len(res), 2) + self.assertDictEqual(res[0], {"error": False, "error_msg": None, "location": "DeviceCore/1397876/0"}) + self.assertDictEqual(res[1], {"error": False, "error_msg": None, "location": "DeviceCore/946246/0"}) + + def test_without_required_param(self): + self.assertRaises(ValueError, self.dc.devicecore.provision_device, description="I should not work") + + def test_bad_request_400(self): + self.prepare_response('POST', '/ws/DeviceCore', 'Bad Request', status=400) + self.assertRaises(DeviceCloudHttpException, + self.dc.devicecore.provision_device, mac_address="DE:AD:BE:EF:00:00") + + def test_bad_request_500(self): + self.prepare_response('POST', '/ws/DeviceCore', 'Bad Request', status=500) + self.assertRaises(DeviceCloudHttpException, + self.dc.devicecore.provision_device, mac_address="DE:AD:BE:EF:00:00") + + def test_error_response(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_ERROR1, status=207) + res = self.dc.devicecore.provision_device(imei="990000862471854") + self.assertDictEqual(res, { + "error": True, + "error_msg": 'The device 00000000-00000000-BC5FF4FF-FFF7908A is already provisioned.', + "location": None} + ) + + def test_mixed_error_success_response(self): + self.prepare_response("POST", "/ws/DeviceCore", PROVISION_MIXED_RESULT_RESPONSE, status=207) + res = self.dc.devicecore.provision_devices([ + {'device_id': "00000000-00000000-0000DEFF-FFADBEEFF"}, + {'mac_address': 'DE:AD:BE:EF:00:00'} + ]) + self.assertTrue(len(res), 2) + self.assertDictEqual(res[0], { + 'error': False, + 'error_msg': None, + 'location': 'DeviceCore/1397876/0', + }) + self.assertDictEqual(res[1], { + 'error': True, + 'error_msg': 'The device 00000000-00000000-D48564FF-FF9D4FEE is already provisioned.', + 'location': None, + }) + + +class TestDeviceCoreDeleting(HttpTestBase): + + def test_delete_device_good(self): + fake_device = mock.MagicMock() + fake_device.get_device_id.return_value = '1234' + self.prepare_response("DELETE", "/ws/DeviceCore/1234", "1 items deleted", status=200) + self.dc.devicecore.delete_device(fake_device) + req = self._get_last_request() + self.assertEqual(req.path, "/ws/DeviceCore/1234") + + def test_delete_device_not_exist(self): + fake_device = mock.MagicMock() + fake_device.get_device_id.return_value = '1234' + self.prepare_response("DELETE", "/ws/DeviceCore/1234", "0 items deleted", status=200) + self.dc.devicecore.delete_device(fake_device) + req = self._get_last_request() + self.assertEqual(req.path, "/ws/DeviceCore/1234") + + def test_delete_device_bad_status(self): + fake_device = mock.MagicMock() + fake_device.get_device_id.return_value = '1234' + self.prepare_response("DELETE", "/ws/DeviceCore/1234", "I pity da foo' who don' know about API changes.", status=400) + try: + self.dc.devicecore.delete_device(fake_device) + except DeviceCloudHttpException: + pass + else: + assert False, "should have thrown exception" + req = self._get_last_request() + self.assertEqual(req.path, "/ws/DeviceCore/1234") + + +class TestDeviceCoreDevices(HttpTestBase): + + def test_dc_get_devices(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + devices = self.dc.devicecore.get_devices() + dev1 = six.next(devices) + dev2 = six.next(devices) + self.assertRaises(StopIteration, six.next, devices) + + self.assertEqual(dev1.get_mac(), "00:40:9D:58:17:5B") + self.assertEqual(dev1.get_mac_last4(), "175B") + self.assertEqual(dev1.get_device_id(), "702077") + self.assertEqual(dev1.get_connectware_id(), "00000000-00000000-00409DFF-FF58175B") + self.assertEqual(dev1.get_ip(), "10.35.1.107") + self.assertEqual(dev1.get_tags(), []) + self.assertEqual(dev1.get_registration_dt(), + datetime.datetime(2013, 2, 28, 19, 54, tzinfo=tzutc())) + self.assertEqual(dev1.get_meid(), '354374042391400') + self.assertEqual(dev1.get_customer_id(), '1872') + self.assertEqual(dev1.get_group_id(), '2331') + self.assertEqual(dev1.get_group_path(), '') + self.assertEqual(dev1.get_vendor_id(), '4261412864') + self.assertEqual(dev1.get_device_type(), 'ConnectPort X5 R') + self.assertEqual(dev1.get_firmware_level(), '34537482') + self.assertEqual(dev1.get_firmware_level_description(), '2.15.0.10') + self.assertEqual(dev1.get_restricted_status(), "0") + self.assertEqual(dev1.get_last_known_ip(), '10.35.1.107') + self.assertEqual(dev1.get_global_ip(), '204.182.3.237') + self.assertEqual(dev1.get_last_connected_dt(), + datetime.datetime(2013, 4, 8, 4, 1, 20, 633000, tzinfo=tzutc())) + self.assertEqual(dev1.get_contact(), '') + self.assertEqual(dev1.get_description(), '') + self.assertEqual(dev1.get_location(), '') + self.assertEqual(dev1.get_latlon(), (34.964465, 40.268198)) + self.assertEqual(dev1.get_user_metadata(), None) + self.assertEqual(dev1.get_zb_pan_id(), None) + self.assertEqual(dev1.get_zb_extended_address(), None) + self.assertEqual(dev1.get_server_id(), '') + self.assertEqual(dev1.get_provision_id(), None) + self.assertEqual(dev1.get_current_connect_pw(), None) + + def test_dc_get_devices_paged(self): + self.prepare_response("GET", "/ws/DeviceCore", GET_DEVICES_PAGE1) + gen = self.dc.devicecore.get_devices(page_size=1) + dev1 = six.next(gen) + self.prepare_response("GET", "/ws/DeviceCore", GET_DEVICES_PAGE2) + dev2 = six.next(gen) + self.assertRaises(StopIteration, six.next, gen) + self.assertEqual(dev1.get_device_id(), '702077') + self.assertEqual(dev2.get_device_id(), '702078') + + def test_dc_get_devices_with_condition(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + gen = self.dc.devicecore.get_devices(dev_mac == 'xx:xx:xx:xx:xx', page_size=1) + six.next(gen) + qs = httpretty.last_request().querystring + self.assertEqual(qs['condition'][0], "devMac='xx:xx:xx:xx:xx'") + self.assertEqual(qs['size'][0], "1") + self.assertEqual(qs['embed'][0], "true") + self.assertEqual(qs['start'][0], "0") + + def test_refresh_from_cache(self): + get_devices_update = copy.deepcopy(EXAMPLE_GET_DEVICES) + get_devices_update["items"][0]["dpDeviceType"] = "Turboencabulator" + del get_devices_update["items"][1] # remove the other item... close enough + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + devices = self.dc.devicecore.get_devices() + device = six.next(devices) + self.prepare_json_response("GET", "/ws/DeviceCore/702077", get_devices_update) + self.assertEqual(device.get_device_type(), "ConnectPort X5 R") + self.assertEqual(device.get_device_type(False), "Turboencabulator") + self.assertEqual(device.get_device_type(), "Turboencabulator") # make sure cache updated + + def test_add_device_to_group(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + expected = ADD_GROUP_TEMPLATE.format(connectware_id=dev.get_connectware_id(), + group_path='testgrp') + dev.add_to_group('testgrp') + self.assertIsNone(dev._device_json) + self.assertEqual(six.b(expected), httpretty.last_request().body) + + def test_remove_device_from_group(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + dev.get_group_path = lambda: 'something other than empty string' + expected = ADD_GROUP_TEMPLATE.format(connectware_id=dev.get_connectware_id(), + group_path='') + dev.remove_from_group() + self.assertIsNone(dev._device_json) + self.assertEqual(six.b(expected), httpretty.last_request().body) + + def test_add_device_tag(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + expected = TAGS_TEMPLATE.format(connectware_id=dev.get_connectware_id(), + tags='test') + dev.add_tag('test') + self.assertIsNone(dev._device_json) + self.assertEqual(six.b(expected), httpretty.last_request().body) + + def test_remove_device_tag(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + try: + dev.remove_tag('test') + except ValueError: + pass + else: + assert False, "should have thrown exception" + + +if __name__ == '__main__': + unittest.main() diff --git a/devicecloud/test/unit/test_filedata.py b/devicecloud/test/unit/test_filedata.py index 699f14d..3c32230 100644 --- a/devicecloud/test/unit/test_filedata.py +++ b/devicecloud/test/unit/test_filedata.py @@ -192,9 +192,9 @@ def test_file_metadata_access(self): self.assertEqual(obj.get_type(), "file") self.assertEqual(obj.get_content_type(), "application/binary") self.assertEqual(obj.get_last_modified_date(), - datetime.datetime(2014, 7, 20, 18, 46, 45, 123, tzinfo=tzutc())) + datetime.datetime(2014, 7, 20, 18, 46, 45, 123000, tzinfo=tzutc())) self.assertEqual(obj.get_created_date(), - datetime.datetime(2014, 7, 20, 18, 46, 45, 123, tzinfo=tzutc())) + datetime.datetime(2014, 7, 20, 18, 46, 45, 123000, tzinfo=tzutc())) self.assertEqual(obj.get_customer_id(), "1234") self.assertEqual(obj.get_full_path(), "/db/blah/test.txt") self.assertEqual(obj.get_size(), 1234) @@ -210,9 +210,9 @@ def test_directory_metadata_access(self): self.assertEqual(obj.get_type(), "directory") self.assertEqual(obj.get_content_type(), "application/xml") self.assertEqual(obj.get_last_modified_date(), - datetime.datetime(2014, 7, 20, 18, 46, 45, 123, tzinfo=tzutc())) + datetime.datetime(2014, 7, 20, 18, 46, 45, 123000, tzinfo=tzutc())) self.assertEqual(obj.get_created_date(), - datetime.datetime(2014, 7, 20, 18, 46, 45, 123, tzinfo=tzutc())) + datetime.datetime(2014, 7, 20, 18, 46, 45, 123000, tzinfo=tzutc())) self.assertEqual(obj.get_customer_id(), "1234") self.assertEqual(obj.get_full_path(), "/db/blah/") self.assertEqual(obj.get_size(), 0) diff --git a/devicecloud/test/unit/test_streams.py b/devicecloud/test/unit/test_streams.py index 71dff09..deb800b 100644 --- a/devicecloud/test/unit/test_streams.py +++ b/devicecloud/test/unit/test_streams.py @@ -420,6 +420,7 @@ def test_bulk_write_datapoints_datapoint_has_no_stream_id(self): def test_bulk_write_multiple_pages(self): # Actual response has a ton of locations for the new data points requests = [] + def handle_request(request, uri, headers): requests.append(request) return (200, headers, '') @@ -436,7 +437,6 @@ def handle_request(request, uri, headers): self.dc.streams.bulk_write_datapoints(datapoints) self.assertEqual(len(requests), 2) - def parse_for_data(response): root = ET.fromstring(response) return [int(x.text) for x in root.iter('data')] @@ -566,6 +566,7 @@ def test_bulk_write_multiple_pages(self): stream = self._get_stream("test") requests = [] + def handle_request(request, uri, headers): requests.append(request) return (200, headers, '') @@ -791,8 +792,8 @@ def test_accessors(self): dp = stream.get_current_value() self.assertEqual(dp.get_id(), "07d77854-0557-11e4-ab44-fa163e7ebc6b") - self.assertEqual(dp.get_timestamp(), datetime.datetime(2014, 7, 6, 21, 46, 47, 981, tzinfo=tzutc())) - self.assertEqual(dp.get_server_timestamp(), datetime.datetime(2014, 7, 6, 21, 46, 47, 981, tzinfo=tzutc())) + self.assertEqual(dp.get_timestamp(), datetime.datetime(2014, 7, 6, 21, 46, 47, 981000, tzinfo=tzutc())) + self.assertEqual(dp.get_server_timestamp(), datetime.datetime(2014, 7, 6, 21, 46, 47, 981000, tzinfo=tzutc())) self.assertEqual(dp.get_data(), 123.1) self.assertEqual(dp.get_description(), "Test") self.assertEqual(dp.get_quality(), 20) @@ -814,7 +815,7 @@ def test_json_encode_to_xml(self): dp = DataPoint( data_type=STREAM_TYPE_JSON, data=my_dict, - ) + ) xml = dp.to_xml() self.assertIsNotNone(re.search('\{[ ",a-zA-Z0-9:[\]]+\}', xml)) @@ -852,7 +853,7 @@ def test_get_data_no_conversion(self): data_type=STREAM_TYPE_FLOAT, data=my_float, quality=0 - ) + ) self.assertEqual(my_float, dp.get_data()) self.assertFalse(mfloat.called) DSTREAM_TYPE_MAP[STREAM_TYPE_FLOAT] = old_float_conversion From 2b427c93a6051836b4d5e0e1b8aac261b18bc7ea Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Fri, 20 Jul 2018 15:49:20 -0500 Subject: [PATCH 094/140] bump to version 0.5.0, added tag support --- devicecloud/test/unit/test_version.py | 9 +++++++++ devicecloud/version.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 devicecloud/test/unit/test_version.py diff --git a/devicecloud/test/unit/test_version.py b/devicecloud/test/unit/test_version.py new file mode 100644 index 0000000..21dd404 --- /dev/null +++ b/devicecloud/test/unit/test_version.py @@ -0,0 +1,9 @@ +import unittest + +from devicecloud.version import __version__ + + +class TestVersion(unittest.TestCase): + + def test_version_format(self): + self.assertTrue(len(__version__.split('.')) >= 3) diff --git a/devicecloud/version.py b/devicecloud/version.py index ba081ef..d48a27e 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -4,4 +4,4 @@ # # Copyright (c) 2015 Digi International, Inc. -__version__ = "0.4.3" +__version__ = "0.5.0" From 4ccce9b9fccf0cc8c246e19e942d39c3790ff708 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Fri, 20 Jul 2018 16:10:12 -0500 Subject: [PATCH 095/140] Remove python 3.2 and 3.3, as they are no longer supported --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8679145..b38dea5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,6 @@ language: python python: 2.7 env: - TOX_ENV=py27 - - TOX_ENV=py32 - - TOX_ENV=py33 - TOX_ENV=py34 - TOX_ENV=py35 - TOX_ENV=py36 From ef84767316157e0eeb342c43482af15e09749dee Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Fri, 20 Jul 2018 16:48:40 -0500 Subject: [PATCH 096/140] Update tox local tests for pypy3 and new github url --- tox.ini | 2 +- toxtest.sh | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 86deb32..b4265f9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py33,py34,py35,py36,py37,pypy +envlist = py27,py33,py34,py35,py36,py37,pypy,pypy3 [testenv] passenv = * diff --git a/toxtest.sh b/toxtest.sh index ec71a56..27c914c 100755 --- a/toxtest.sh +++ b/toxtest.sh @@ -12,15 +12,16 @@ pyversions=(2.7.7 3.5.5 3.6.6 3.7.0 - pypy-2.3.1) + pypy2.7-6.0.0 + pypy3.5-6.0.0) # first make sure that pyenv is installed if [ ! -s "$HOME/.pyenv/bin/pyenv" ]; then - curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash + curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash fi # Update pyenv (required for new python versions to be available) -(cd $HOME/.pyenv && git pull) +pyenv update # add pyenv to our path and initialize (if this has not already been done) export PATH="$HOME/.pyenv/bin:$PATH" From b469dcb15394c9ae2c26c22f60e979e8810ed2d1 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Fri, 20 Jul 2018 16:49:09 -0500 Subject: [PATCH 097/140] Remove unsupported python/tox versions on Travis-CI --- .travis.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index b38dea5..3e18590 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,13 @@ +# +# This is the Travis-CI configuration. +# +# The actual dependency installation and test execution is done via tox as a +# way to share the same process between Travis-CI and Buildbot. +# language: python +# allow travis to use new, faster container based +# infrastructure to perform the testing +sudo: false # this version of python is only used to run tox - the version specified by TOX_ENV # is used to install and run tests @@ -6,11 +15,6 @@ python: 2.7 env: - TOX_ENV=py27 - TOX_ENV=py34 - - TOX_ENV=py35 - - TOX_ENV=py36 - - TOX_ENV=py37 - - TOX_ENV=pypy - - TOX_ENV=coverage # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors install: @@ -21,6 +25,3 @@ install: script: - tox -e $TOX_ENV -# allow travis to use new, faster container based -# infrastructure to perform the testing -sudo: false From 15a6579971f6a8282fa84d0faaf11873f326ab85 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Fri, 20 Jul 2018 17:23:38 -0500 Subject: [PATCH 098/140] release: publish version 0.5.0 - device tag support Signed-off-by: Brandon Moser --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 889f893..ba821e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ ## Python Devicecloud Library Changelog +### 0.5.0 / 2018-07-20 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.4.3...0.5.0) + +Enhancements: + +* devicecore: add support for adding and removing tags from a device + + ### 0.4.3 / 2018-06-11 [Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.4.2...0.4.3) From c838e36c6758b1174c89dffd1078d922b268fe4d Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Fri, 20 Jul 2018 21:24:19 -0500 Subject: [PATCH 099/140] Update copyright --- HACKING.md | 200 +++++++++--------- README.md | 2 +- devicecloud/__init__.py | 3 +- devicecloud/apibase.py | 2 +- devicecloud/conditions.py | 2 +- devicecloud/data/__init__.py | 2 +- devicecloud/devicecore.py | 3 +- devicecloud/examples/__init__.py | 2 +- devicecloud/examples/devicecore_playground.py | 2 +- devicecloud/examples/example_helpers.py | 2 +- devicecloud/examples/filedata_playground.py | 2 +- devicecloud/examples/streams_playground.py | 2 +- devicecloud/file_system_service.py | 2 +- devicecloud/filedata.py | 2 +- devicecloud/monitor.py | 6 +- devicecloud/monitor_tcp.py | 2 +- devicecloud/sci.py | 2 +- devicecloud/streams.py | 2 +- devicecloud/test/__init__.py | 2 +- devicecloud/test/integration/__init__.py | 2 +- .../test/integration/inttest_monitor_tcp.py | 4 +- .../test/integration/inttest_streams.py | 2 +- .../test/integration/inttest_utilities.py | 2 +- devicecloud/test/unit/__init__.py | 2 +- devicecloud/test/unit/test_conditions.py | 7 + devicecloud/test/unit/test_core.py | 2 +- devicecloud/test/unit/test_devicecore.py | 3 +- .../test/unit/test_file_system_service.py | 13 +- devicecloud/test/unit/test_filedata.py | 6 + devicecloud/test/unit/test_monitor.py | 4 +- devicecloud/test/unit/test_monitor_tcp.py | 2 +- devicecloud/test/unit/test_sci.py | 2 +- devicecloud/test/unit/test_streams.py | 2 +- devicecloud/test/unit/test_utilities.py | 2 +- devicecloud/test/unit/test_version.py | 6 + devicecloud/test/unit/test_ws.py | 7 + devicecloud/util.py | 2 +- devicecloud/version.py | 2 +- devicecloud/ws.py | 2 +- setup.py | 2 +- 40 files changed, 178 insertions(+), 140 deletions(-) diff --git a/HACKING.md b/HACKING.md index 74e9957..e849465 100644 --- a/HACKING.md +++ b/HACKING.md @@ -1,100 +1,100 @@ -Developer's Guide -================= - -Environment Setup ------------------ - -All the requirements in order to perform development on the product -should be installable in a virtualenv. - - $ pip install -r dev-requirements.txt - -In order to build a release you will also need to install pandoc. On -Ubuntu, you should be able to do: - - $ sudo apt-get install pandoc - - -Running the Unit Tests ----------------------- - -### Running Tests with Nose - -Running the tests is easy with nose (included in -test-requirements.txt). From the project root: - - $ nosetests . - -To run the unit tests with coverage results (view cover/index.html), -do the following: - - $ nosetests --with-coverage --cover-html --cover-package=devicecloud . - -New contributions to the library will only be accepted if they include -unit test coverage (with some exceptions). - -### Testing All Versions with Tox - -We also support running the tests against all supported versions of -python using a combination of -[tox](http://tox.readthedocs.org/en/latest/) and -[pyenv](https://github.com/yyuu/pyenv). To run all of the tests -against all supported versions of python, just do the following: - - $ ./toxtest.sh - -This might take awhile the first time as it will build from source a -version of the interpreter for each version supported. If you recieve -errors from pyenv, there may be addition dependencies required. -Please visit https://github.com/yyuu/pyenv/wiki/Common-build-problems -for additional pointers. - -### Running Integration and Unittests - -There are some additional integration tests that run against an actual -device cloud account. These are a bit more fragile and when something -fails, you may need to go to your device cloud account to clean things -up. - -To run those tests, you can just do the following. This script runs -the toxtest.sh script with environment variables set with your -account information. The tests that were skipped before will now -be run with each supported version of the interpreter: - - $ ./inttest.sh - -Build the Documentation ------------------------ - -Documentation (outside of this file and the README) is done via -Sphinx. To build the docs, just do the following (with virtualenv -activated): - - $ cd docs - $ make html - -The docs that are built will be located at -docs/_build/html/index.html. - -The documentation for the project is published on github using a [Github -Pages](https://pages.github.com/) Project Site. The process for -releasing a new set of documentation is the following: - -1. Create a fresh clone of the project and checkout the `gh-pages` - branch. Although this is the same repo, the tree is completely - separate from the main python-devicecloud codebase. -2. Remove all contents from the working area -3. From the python-devicecloud repo, `cp -r docs/_build/html/* - /path/to/other/repo/`. -4. Commit and push the update `gh-pages` branch to github - -Open Source License Header --------------------------- - -Each source file should be prefixed with the following header: - - # This Source Code Form is subject to the terms of the Mozilla Public - # License, v. 2.0. If a copy of the MPL was not distributed with this - # file, You can obtain one at http://mozilla.org/MPL/2.0/. - # - # Copyright (c) 2015 Digi International, Inc. All rights reserved. +Developer's Guide +================= + +Environment Setup +----------------- + +All the requirements in order to perform development on the product +should be installable in a virtualenv. + + $ pip install -r dev-requirements.txt + +In order to build a release you will also need to install pandoc. On +Ubuntu, you should be able to do: + + $ sudo apt-get install pandoc + + +Running the Unit Tests +---------------------- + +### Running Tests with Nose + +Running the tests is easy with nose (included in +test-requirements.txt). From the project root: + + $ nosetests . + +To run the unit tests with coverage results (view cover/index.html), +do the following: + + $ nosetests --with-coverage --cover-html --cover-package=devicecloud . + +New contributions to the library will only be accepted if they include +unit test coverage (with some exceptions). + +### Testing All Versions with Tox + +We also support running the tests against all supported versions of +python using a combination of +[tox](http://tox.readthedocs.org/en/latest/) and +[pyenv](https://github.com/yyuu/pyenv). To run all of the tests +against all supported versions of python, just do the following: + + $ ./toxtest.sh + +This might take awhile the first time as it will build from source a +version of the interpreter for each version supported. If you recieve +errors from pyenv, there may be addition dependencies required. +Please visit https://github.com/yyuu/pyenv/wiki/Common-build-problems +for additional pointers. + +### Running Integration and Unittests + +There are some additional integration tests that run against an actual +device cloud account. These are a bit more fragile and when something +fails, you may need to go to your device cloud account to clean things +up. + +To run those tests, you can just do the following. This script runs +the toxtest.sh script with environment variables set with your +account information. The tests that were skipped before will now +be run with each supported version of the interpreter: + + $ ./inttest.sh + +Build the Documentation +----------------------- + +Documentation (outside of this file and the README) is done via +Sphinx. To build the docs, just do the following (with virtualenv +activated): + + $ cd docs + $ make html + +The docs that are built will be located at +docs/_build/html/index.html. + +The documentation for the project is published on github using a [Github +Pages](https://pages.github.com/) Project Site. The process for +releasing a new set of documentation is the following: + +1. Create a fresh clone of the project and checkout the `gh-pages` + branch. Although this is the same repo, the tree is completely + separate from the main python-devicecloud codebase. +2. Remove all contents from the working area +3. From the python-devicecloud repo, `cp -r docs/_build/html/* + /path/to/other/repo/`. +4. Commit and push the update `gh-pages` branch to github + +Open Source License Header +-------------------------- + +Each source file should be prefixed with the following header: + + # This Source Code Form is subject to the terms of the Mozilla Public + # License, v. 2.0. If a copy of the MPL was not distributed with this + # file, You can obtain one at http://mozilla.org/MPL/2.0/. + # + # Copyright (c) 2015-2018 Digi International Inc. All rights reserved. diff --git a/README.md b/README.md index 7315a94..15ffe3f 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ License This software is open-source software. -Copyright (c) 2015 Digi International, Inc. +Copyright (c) 2015-2018 Digi International Inc. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index c6f3508..869650a 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -2,7 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. + import logging import time import json diff --git a/devicecloud/apibase.py b/devicecloud/apibase.py index 56708df..e7b7715 100644 --- a/devicecloud/apibase.py +++ b/devicecloud/apibase.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. class APIBase(object): """Base class for all API Classes diff --git a/devicecloud/conditions.py b/devicecloud/conditions.py index 4231f3b..83791c7 100644 --- a/devicecloud/conditions.py +++ b/devicecloud/conditions.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. """Module with functionality for building queries against cloud resources diff --git a/devicecloud/data/__init__.py b/devicecloud/data/__init__.py index cbfb263..1c52222 100644 --- a/devicecloud/data/__init__.py +++ b/devicecloud/data/__init__.py @@ -2,5 +2,5 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. All rights reserved. +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. # diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index a63d62a..cedf6df 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -2,7 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. + import sys import xml.etree.ElementTree as ET diff --git a/devicecloud/examples/__init__.py b/devicecloud/examples/__init__.py index 02c23fd..afdd51d 100644 --- a/devicecloud/examples/__init__.py +++ b/devicecloud/examples/__init__.py @@ -2,4 +2,4 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. diff --git a/devicecloud/examples/devicecore_playground.py b/devicecloud/examples/devicecore_playground.py index f94bb40..cdee37b 100644 --- a/devicecloud/examples/devicecore_playground.py +++ b/devicecloud/examples/devicecore_playground.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. from devicecloud.devicecore import dev_mac, group_path from devicecloud.examples.example_helpers import get_authenticated_dc diff --git a/devicecloud/examples/example_helpers.py b/devicecloud/examples/example_helpers.py index d548bf9..011c57b 100644 --- a/devicecloud/examples/example_helpers.py +++ b/devicecloud/examples/example_helpers.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. from getpass import getpass import os from six.moves import input diff --git a/devicecloud/examples/filedata_playground.py b/devicecloud/examples/filedata_playground.py index 5c4bd42..3533a25 100644 --- a/devicecloud/examples/filedata_playground.py +++ b/devicecloud/examples/filedata_playground.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. from devicecloud.examples.example_helpers import get_authenticated_dc from devicecloud.filedata import fd_path diff --git a/devicecloud/examples/streams_playground.py b/devicecloud/examples/streams_playground.py index 91e1d04..15f2678 100644 --- a/devicecloud/examples/streams_playground.py +++ b/devicecloud/examples/streams_playground.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. from math import pi import pprint import time diff --git a/devicecloud/file_system_service.py b/devicecloud/file_system_service.py index 15eb6a4..20bb522 100644 --- a/devicecloud/file_system_service.py +++ b/devicecloud/file_system_service.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. """Provide access to the device cloud file system service API""" import base64 diff --git a/devicecloud/filedata.py b/devicecloud/filedata.py index 9cc05dd..826f0ac 100644 --- a/devicecloud/filedata.py +++ b/devicecloud/filedata.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. """Provide access to the device cloud filedata API""" diff --git a/devicecloud/monitor.py b/devicecloud/monitor.py index 449b9e9..a2f4146 100644 --- a/devicecloud/monitor.py +++ b/devicecloud/monitor.py @@ -2,10 +2,11 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. All rights reserved. +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. # # This code is originally from another Digi Open Source Library: # https://github.com/digidotcom/idigi-python-monitor-api + import xml.etree.ElementTree as ET import logging import textwrap @@ -209,7 +210,7 @@ def create_tcp_monitor(self, topics, batch_size=1, batch_duration=0, monitor_id = int(location.split('/')[-1]) return TCPDeviceCloudMonitor(self._conn, monitor_id, self._tcp_client_manager) - def create_http_monitor(self, topics, transport_url, transport_token=None, transport_method='PUT',connect_timeout=0, + def create_http_monitor(self, topics, transport_url, transport_token=None, transport_method='PUT', connect_timeout=0, response_timeout=0, batch_size=1, batch_duration=0, compression='none', format_type='json'): """Creates a HTTP Monitor instance in the device cloud for a given list of topics @@ -363,6 +364,7 @@ def delete(self): """Delete this monitor form the device cloud""" self._conn.delete("/ws/Monitor/{id}".format(id=self._id)) + class HTTPDeviceCloudMonitor(DeviceCloudMonitor): """Device Cloud Monitor with HTTP transport type""" diff --git a/devicecloud/monitor_tcp.py b/devicecloud/monitor_tcp.py index 94954a6..0cee12a 100644 --- a/devicecloud/monitor_tcp.py +++ b/devicecloud/monitor_tcp.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. All rights reserved. +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. # # This code is originally from another Digi Open Source Library: # https://github.com/digidotcom/idigi-python-monitor-api diff --git a/devicecloud/sci.py b/devicecloud/sci.py index a4069ac..134af77 100644 --- a/devicecloud/sci.py +++ b/devicecloud/sci.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. """Server Command Interface functionality""" from devicecloud.apibase import APIBase diff --git a/devicecloud/streams.py b/devicecloud/streams.py index 1da3c17..0b77de3 100644 --- a/devicecloud/streams.py +++ b/devicecloud/streams.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. r"""Module providing classes for interacting with device cloud data streams""" import json diff --git a/devicecloud/test/__init__.py b/devicecloud/test/__init__.py index 02c23fd..afdd51d 100644 --- a/devicecloud/test/__init__.py +++ b/devicecloud/test/__init__.py @@ -2,4 +2,4 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. diff --git a/devicecloud/test/integration/__init__.py b/devicecloud/test/integration/__init__.py index ce587c6..14b3231 100644 --- a/devicecloud/test/integration/__init__.py +++ b/devicecloud/test/integration/__init__.py @@ -2,4 +2,4 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. All rights reserved. +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. diff --git a/devicecloud/test/integration/inttest_monitor_tcp.py b/devicecloud/test/integration/inttest_monitor_tcp.py index 94fd890..babc25c 100644 --- a/devicecloud/test/integration/inttest_monitor_tcp.py +++ b/devicecloud/test/integration/inttest_monitor_tcp.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. import pprint import time from devicecloud.streams import DataPoint @@ -58,4 +58,4 @@ def receive_notification(notification): if __name__ == '__main__': import unittest - unittest.main() \ No newline at end of file + unittest.main() diff --git a/devicecloud/test/integration/inttest_streams.py b/devicecloud/test/integration/inttest_streams.py index ae51fb3..4a3f30e 100644 --- a/devicecloud/test/integration/inttest_streams.py +++ b/devicecloud/test/integration/inttest_streams.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. All rights reserved. +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. """Integration tests for streams functionality diff --git a/devicecloud/test/integration/inttest_utilities.py b/devicecloud/test/integration/inttest_utilities.py index 64b4be3..4ec10fd 100644 --- a/devicecloud/test/integration/inttest_utilities.py +++ b/devicecloud/test/integration/inttest_utilities.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. All rights reserved. +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. from getpass import getpass import unittest diff --git a/devicecloud/test/unit/__init__.py b/devicecloud/test/unit/__init__.py index ce587c6..14b3231 100644 --- a/devicecloud/test/unit/__init__.py +++ b/devicecloud/test/unit/__init__.py @@ -2,4 +2,4 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. All rights reserved. +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. diff --git a/devicecloud/test/unit/test_conditions.py b/devicecloud/test/unit/test_conditions.py index f7c0ba5..3f1afff 100644 --- a/devicecloud/test/unit/test_conditions.py +++ b/devicecloud/test/unit/test_conditions.py @@ -1,3 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. + import unittest import datetime @@ -44,5 +50,6 @@ def test_multi_combination(self): self.assertEqual(((a > 1) & (a > 2) & (a > 3)).compile(), "a>'1' and a>'2' and a>'3'") + if __name__ == '__main__': unittest.main() diff --git a/devicecloud/test/unit/test_core.py b/devicecloud/test/unit/test_core.py index e896fe1..aa13f33 100644 --- a/devicecloud/test/unit/test_core.py +++ b/devicecloud/test/unit/test_core.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. import unittest from devicecloud import DeviceCloudHttpException diff --git a/devicecloud/test/unit/test_devicecore.py b/devicecloud/test/unit/test_devicecore.py index a86b756..fd57759 100644 --- a/devicecloud/test/unit/test_devicecore.py +++ b/devicecloud/test/unit/test_devicecore.py @@ -2,7 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. + import copy import datetime import unittest diff --git a/devicecloud/test/unit/test_file_system_service.py b/devicecloud/test/unit/test_file_system_service.py index 3112da9..a4a7f5a 100644 --- a/devicecloud/test/unit/test_file_system_service.py +++ b/devicecloud/test/unit/test_file_system_service.py @@ -1,3 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. + import base64 import unittest from xml.etree import ElementTree as ET @@ -287,7 +293,7 @@ def test_put_command_no_data(self): def test_put_command_both_data(self): self.assertRaises(FileSystemServiceException, PutCommand, path='/a/path/here', - file_data=six.b("some file data"), server_file='/a/file/on/server') + file_data=six.b("some file data"), server_file='/a/file/on/server') def test_parse(self): self.assertIsNone(PutCommand.parse_response(ET.fromstring(''))) @@ -362,9 +368,9 @@ def setUp(self): # Create some file, directory, and error info objects to use in tests self.file1 = FileInfo(self.fss_api, self.dev1_id, '/a/path/file1.txt', 1436276773, 7989, - "967FDA522517B9CE0C3E056EDEB485BB", 'md5') + "967FDA522517B9CE0C3E056EDEB485BB", 'md5') self.file2 = FileInfo(self.fss_api, self.dev1_id, '/a/path/file2.py', 1434377919, 181, - "DEA17715739E46079C1A6DDCB38344DF", 'md5') + "DEA17715739E46079C1A6DDCB38344DF", 'md5') self.dir1 = DirectoryInfo(self.fss_api, self.dev1_id, '/a/path/dir', 1436203917) self.errinfo = ErrorInfo(errno=1, message="an error message") @@ -848,5 +854,6 @@ def test_send_command_block_errors(self): self.assertEqual("an error message", out_dict[self.dev1_id][1].message) self.assertEqual("an error message", out_dict[self.dev2_id][1].message) + if __name__ == '__main__': unittest.main() diff --git a/devicecloud/test/unit/test_filedata.py b/devicecloud/test/unit/test_filedata.py index 3c32230..16da0c2 100644 --- a/devicecloud/test/unit/test_filedata.py +++ b/devicecloud/test/unit/test_filedata.py @@ -1,3 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. + import base64 import unittest from xml.etree import ElementTree diff --git a/devicecloud/test/unit/test_monitor.py b/devicecloud/test/unit/test_monitor.py index 98cd629..dd0a6bb 100644 --- a/devicecloud/test/unit/test_monitor.py +++ b/devicecloud/test/unit/test_monitor.py @@ -2,7 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. + from devicecloud.monitor import MON_TOPIC_ATTR, MON_TRANSPORT_TYPE_ATTR from devicecloud.test.unit.test_utilities import HttpTestBase import six @@ -188,7 +189,6 @@ """ - class TestMonitorAPI(HttpTestBase): def test_create_tcp_monitor(self): diff --git a/devicecloud/test/unit/test_monitor_tcp.py b/devicecloud/test/unit/test_monitor_tcp.py index 1eb3acc..1293273 100644 --- a/devicecloud/test/unit/test_monitor_tcp.py +++ b/devicecloud/test/unit/test_monitor_tcp.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. from devicecloud.monitor_tcp import TCPClientManager from devicecloud.test.unit.test_utilities import HttpTestBase diff --git a/devicecloud/test/unit/test_sci.py b/devicecloud/test/unit/test_sci.py index 6acd9d3..f7195f9 100644 --- a/devicecloud/test/unit/test_sci.py +++ b/devicecloud/test/unit/test_sci.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. import unittest diff --git a/devicecloud/test/unit/test_streams.py b/devicecloud/test/unit/test_streams.py index deb800b..6c3d8e8 100644 --- a/devicecloud/test/unit/test_streams.py +++ b/devicecloud/test/unit/test_streams.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. import unittest import datetime diff --git a/devicecloud/test/unit/test_utilities.py b/devicecloud/test/unit/test_utilities.py index 2ed5fcf..c080465 100644 --- a/devicecloud/test/unit/test_utilities.py +++ b/devicecloud/test/unit/test_utilities.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. import unittest import json diff --git a/devicecloud/test/unit/test_version.py b/devicecloud/test/unit/test_version.py index 21dd404..1d020f7 100644 --- a/devicecloud/test/unit/test_version.py +++ b/devicecloud/test/unit/test_version.py @@ -1,3 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. + import unittest from devicecloud.version import __version__ diff --git a/devicecloud/test/unit/test_ws.py b/devicecloud/test/unit/test_ws.py index 2ae06b8..4561ff4 100644 --- a/devicecloud/test/unit/test_ws.py +++ b/devicecloud/test/unit/test_ws.py @@ -1,3 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. + import unittest from devicecloud import DeviceCloudException @@ -36,5 +42,6 @@ def test_method_access_args_kwargs(self): self.assertEqual(res[0], ("/ws/a/test/path", "foo")) self.assertDictEqual(res[1], {"bar": "baz"}) + if __name__ == '__main__': unittest.main() diff --git a/devicecloud/util.py b/devicecloud/util.py index 509ba0c..00e1698 100644 --- a/devicecloud/util.py +++ b/devicecloud/util.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. import datetime import arrow diff --git a/devicecloud/version.py b/devicecloud/version.py index d48a27e..dfc6b43 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -2,6 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. +# Copyright (c) 2015-2018 Digi International Inc. __version__ = "0.5.0" diff --git a/devicecloud/ws.py b/devicecloud/ws.py index afac53e..08d35a5 100644 --- a/devicecloud/ws.py +++ b/devicecloud/ws.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015 Digi International, Inc. All rights reserved. +# Copyright (c) 2015-2018 Digi International Inc. All rights reserved. import functools import inspect diff --git a/setup.py b/setup.py index aa69f27..2a2713a 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ def get_long_description(): description="Python API to the Digi Device Cloud", long_description=get_long_description(), url="https://github.com/digidotcom/python-devicecloud", - author="Digi International, Inc.", + author="Digi International Inc.", author_email="brandon.moser@digi.com", packages=find_packages(), install_requires=open('requirements.txt').read().split(), From de283338ee34106e285b25a99bfafe996c619f11 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Tue, 24 Jul 2018 00:12:11 -0500 Subject: [PATCH 100/140] Update documentation to formalize Device Cloud --- README.md | 66 ++++++------------------- devicecloud/__init__.py | 66 ++++++++++++------------- devicecloud/conditions.py | 5 +- devicecloud/devicecore.py | 16 +++--- devicecloud/file_system_service.py | 8 +-- devicecloud/filedata.py | 7 ++- devicecloud/monitor.py | 24 ++++----- devicecloud/monitor_tcp.py | 12 ++--- devicecloud/streams.py | 52 +++++++++---------- devicecloud/test/unit/test_streams.py | 4 +- devicecloud/test/unit/test_utilities.py | 2 +- devicecloud/util.py | 2 +- docs/conf.py | 26 +++++----- docs/cookbook.rst | 24 ++++++++- docs/filedata.rst | 6 +-- docs/filesystem.rst | 2 +- docs/index.rst | 10 ++-- docs/monitor.rst | 4 +- docs/sci.rst | 4 +- docs/streams.rst | 8 +-- docs/ws.rst | 2 +- 21 files changed, 168 insertions(+), 182 deletions(-) diff --git a/README.md b/README.md index 15ffe3f..cc16ef9 100644 --- a/README.md +++ b/README.md @@ -7,36 +7,21 @@ Python Device Cloud Library [![Latest Version](https://img.shields.io/pypi/v/devicecloud.svg)](https://pypi.python.org/pypi/devicecloud/) [![License](https://img.shields.io/badge/license-MPL%202.0-blue.svg)](https://github.com/digidotcom/python-devicecloud/blob/master/LICENSE) -Be sure to check out the [full documentation](http://digidotcom.github.io/python-devicecloud). -A [Changelog](https://github.com/digidotcom/python-devicecloud/blob/master/CHANGELOG.md) -is also available. +Be sure to check out the [full documentation](http://digidotcom.github.io/python-devicecloud). A [Changelog](https://github.com/digidotcom/python-devicecloud/blob/master/CHANGELOG.md) is also available. Overview -------- -Python-devicecloud is a library providing simple, intuitive access to -the [Digi Device Cloud](http://www.digi.com/products/cloud/digi-device-cloud) -for clients written in Python. +Python-devicecloud is a library providing simple, intuitive access to [Digi Device Cloud(sm)](http://www.digi.com/products/cloud/digi-device-cloud) for clients written in Python. -The library wraps the Device Cloud REST API and hides the details of -forming HTTP requests in order to gain access to device information, -file data, streams, and other features of the device cloud. The API -wrapped can be found -[here](http://ftp1.digi.com/support/documentation/90002008_redirect.htm). +The library wraps Device Cloud's REST API and hides the details of forming HTTP requests in order to gain access to device information, file data, streams, and other features of Device Cloud. The API can be found [here](http://ftp1.digi.com/support/documentation/90002008_redirect.htm). -The primary target audience for this library is individuals -interfacing with the device cloud from the server side or developers -writing tools to aid device development. For efficient connectivity -from devices, we suggest that you first look at using the [Device Cloud -Connector](http://www.digi.com/support/productdetail?pid=5575). -That being said, this library could also be used on devices if deemed -suitable. +The primary target audience for this library is individuals interfacing with Device Cloud from the server side or developers writing tools to aid device development. For efficient connectivity from devices, we suggest that you first look at using the [Device Cloud Connector](http://www.digi.com/support/productdetail?pid=5575). That being said, this library could also be used on devices if deemed suitable. Example ------- -The library provides access to a wide array of features, but here is a -couple quick examples to give you a taste of what the API looks like. +The library provides access to a wide array of features, but here is a couple quick examples to give you a taste of what the API looks like. ```python from devicecloud import DeviceCloud @@ -45,7 +30,7 @@ dc = DeviceCloud('user', 'pass') # show the MAC address of all devices that are currently connected # -# This is done using the device cloud DeviceCore functionality +# This is done using Device Cloud DeviceCore functionality print "== Connected Devices ==" for device in dc.devicecore.get_devices(): if device.is_connected(): @@ -54,28 +39,24 @@ for device in dc.devicecore.get_devices(): # get the name and current value of all data streams having values # with a floating point type # -# This is done using the device cloud stream functionality +# This is done using Device Cloud stream functionality for stream in dc.streams.get_streams(): if stream.get_data_type().lower() in ('float', 'double'): print "%s -> %s" % (stream.get_stream_id(), stream.get_current_value()) ``` -For more examples and detailed documentation, be sure to checkout out -the [Full API Documentation](https://digidotcom.github.io/python-devicecloud). +For more examples and detailed documentation, be sure to checkout out the [Full API Documentation](https://digidotcom.github.io/python-devicecloud). Installation ------------ -This library can be installed using -[pip](https://github.com/pypa/pip). Python versions 2.7+ (including -Python 3) are supported by the library. +This library can be installed using [pip](https://github.com/pypa/pip). Python versions 2.7+ and 3+ are supported by the library. ```sh pip install devicecloud ``` -If you already have an older version of the library installed, you can -upgrade to the latest version by doing +If you already have an older version of the library installed, you can upgrade to the latest version by doing ```sh pip install --upgrade devicecloud @@ -84,9 +65,7 @@ pip install --upgrade devicecloud Supported Features ------------------ -Eventually, it is hoped that there will be complete feature parity -between the device cloud API and this library. For now, however, that -is not the case. The current features are supported by the library: +Eventually, it is hoped that there will be complete feature parity between Device Cloud API and this library. For now, however, that is not the case. The current features are supported by the library: * Getting basic device information via DeviceCore * Provision and Delete devices via DeviceCore @@ -110,7 +89,7 @@ is not the case. The current features are supported by the library: * Get full metadata and contents of files and directories. * Low level support for performing basic SCI commands with limited parsing of results and support for only a subset of available services/commands. -* APIs to make direct web service calls to the device cloud with some details +* APIs to make direct web service calls to Device Cloud with some details handled by the library (see DeviceCloudConnection and 'ws' documentation) * Device Provisioning via Mac Address, IMEI or Device ID * Monitors @@ -130,23 +109,15 @@ which features should be highest priority is always welcome. * XBee specific support (XBeeCore) * Smart Energy APIs * SMS Support -* Satellite/Iridium Support * SM/UDP Support * Carrier Information Access Contributing ------------ -Contributions to the library are very welcome in whatever form can be -provided. This could include issue reports, bug fixes, or features -additions. For issue reports, please [create an issue against the -Github -project](https://github.com/digidotcom/python-devicecloud/issues). +Contributions to the library are very welcome in whatever form can be provided. This could include issue reports, bug fixes, or features additions. For issue reports, please [create an issue against the Github project](https://github.com/digidotcom/python-devicecloud/issues). -For code changes, feel free to fork the project on Github and submit a -pull request with your changes. Additional instructions for -developers contributing to the project can be found in the [Developer's -Guide](https://github.com/digidotcom/python-devicecloud/blob/master/HACKING.md). +For code changes, feel free to fork the project on Github and submit a pull request with your changes. Additional instructions for developers contributing to the project can be found in the [Developer's Guide](https://github.com/digidotcom/python-devicecloud/blob/master/HACKING.md). License ------- @@ -155,14 +126,9 @@ This software is open-source software. Copyright (c) 2015-2018 Digi International Inc. -This Source Code Form is subject to the terms of the Mozilla Public -License, v. 2.0. If a copy of the MPL was not distributed with this file, -you can obtain one at http://mozilla.org/MPL/2.0/. +This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, you can obtain one at http://mozilla.org/MPL/2.0/. -Digi, Digi International, the Digi logo, the Digi website, Digi Device Cloud, and Digi Cloud Connector are trademarks or registered trademarks of -Digi International, Inc. in the United States and other countries -worldwide. All other trademarks are the property of their respective -owners. +Digi, Digi International, the Digi logo, the Digi website, Digi Device Cloud, Digi Remote Manager, and Digi Cloud Connector are trademarks or registered trademarks of Digi International Inc. in the United States and other countries worldwide. All other trademarks are the property of their respective owners. THE SOFTWARE AND RELATED TECHNICAL INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index 869650a..88060ed 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -63,10 +63,10 @@ class DeviceCloudHttpException(DeviceCloudException): """Exception raised when we failed a request to the DC over HTTP This exception will be raised whenever a non-success HTTP status - code is received from the device cloud and there is no other logic + code is received from Device Cloud and there is no other logic in place for gracefully handling the error case. - Often, if there is an error with a request to the device cloud, the device + Often, if there is an error with a request to Device Cloud, the device cloud will respond with an error status and include additional information about the nature of the error in the response body. This information can be accessed via the :attr:`~response` property. @@ -102,7 +102,7 @@ def response(self): class DeviceCloudConnection(object): - """Provide low-level access to the Device Cloud web services + """Provide low-level access to Device Cloud web services This is a convenience object that provides methods that make sending requests to the device cloud easier. This object is used extensively within the library but can @@ -189,9 +189,9 @@ def iter_json_pages(self, path, page_size=1000, **params): :param str path: The base path to the resource being requested (e.g. /ws/Group) :param int page_size: The number of items that should be requested for each page. A larger page_size may mean fewer HTTP requests but could also increase the time to get a first - result back from the device cloud. + result back from Device Cloud. :param params: These are additional query parameters that should be sent with each - request to the device cloud. + request to Device Cloud. """ path = validate_type(path, *six.string_types) @@ -209,7 +209,7 @@ def iter_json_pages(self, path, page_size=1000, **params): yield item_json def ping(self): - """Ping the Device Cloud using the authorization provided + """Ping Device Cloud using the authorization provided :return: The response of getting a single device from DeviceCore on success :raises: :class:`.DeviceCloudHttpException` if there is a problem @@ -218,19 +218,19 @@ def ping(self): return self.get("/ws/DeviceCore?size=1") def get(self, path, **kwargs): - """Perform an HTTP GET request of the specified path in the device cloud + """Perform an HTTP GET request of the specified path in Device Cloud - Make an HTTP GET request against the device cloud with this accounts + Make an HTTP GET request against Device Cloud with this accounts credentials and base url. This method uses the `requests `_ library `request method `_ and all keyword arguments will be passed on to that method. - :param str path: The device cloud path to GET + :param str path: Device Cloud path to GET :param int retries: The number of times the request should be retried if an unsuccessful response is received. Most likely, you should leave this at 0. :raises DeviceCloudHttpException: if a non-success response to the request is received - from the device cloud + from Device Cloud :returns: A requests ``Response`` object """ @@ -238,24 +238,24 @@ def get(self, path, **kwargs): return self._make_request("GET", url, **kwargs) def get_json(self, path, **kwargs): - """Perform an HTTP GET request with JSON headers of the specified path against the device cloud + """Perform an HTTP GET request with JSON headers of the specified path against Device Cloud - Make an HTTP GET request against the device cloud with this accounts + Make an HTTP GET request against Device Cloud with this accounts credentials and base url. This method uses the `requests `_ library `request method `_ and all keyword arguments will be passed on to that method. This method will automatically add the ``Accept: application/json`` and parse the - JSON response from the device cloud. + JSON response from Device Cloud. - :param str path: The device cloud path to GET + :param str path: Device Cloud path to GET :param int retries: The number of times the request should be retried if an unsuccessful response is received. Most likely, you should leave this at 0. :raises DeviceCloudHttpException: if a non-success response to the request is received - from the device cloud + from Device Cloud :returns: A python data structure containing the results of calling ``json.loads`` on the - body of the response from the device cloud. + body of the response from Device Cloud. """ @@ -266,21 +266,21 @@ def get_json(self, path, **kwargs): return json.loads(response.text) def post(self, path, data, **kwargs): - """Perform an HTTP POST request of the specified path in the device cloud + """Perform an HTTP POST request of the specified path in Device Cloud - Make an HTTP POST request against the device cloud with this accounts + Make an HTTP POST request against Device Cloud with this accounts credentials and base url. This method uses the `requests `_ library `request method `_ and all keyword arguments will be passed on to that method. - :param str path: The device cloud path to POST + :param str path: Device Cloud path to POST :param int retries: The number of times the request should be retried if an unsuccessful response is received. Most likely, you should leave this at 0. :param data: The data to be posted in the body of the POST request (see docs for ``requests.post`` :raises DeviceCloudHttpException: if a non-success response to the request is received - from the device cloud + from Device Cloud :returns: A requests ``Response`` object """ @@ -288,21 +288,21 @@ def post(self, path, data, **kwargs): return self._make_request("POST", url, data=data, **kwargs) def put(self, path, data, **kwargs): - """Perform an HTTP PUT request of the specified path in the device cloud + """Perform an HTTP PUT request of the specified path in Device Cloud - Make an HTTP PUT request against the device cloud with this accounts + Make an HTTP PUT request against Device Cloud with this accounts credentials and base url. This method uses the `requests `_ library `request method `_ and all keyword arguments will be passed on to that method. - :param str path: The device cloud path to PUT + :param str path: Device Cloud path to PUT :param int retries: The number of times the request should be retried if an unsuccessful response is received. Most likely, you should leave this at 0. :param data: The data to be posted in the body of the POST request (see docs for ``requests.post`` :raises DeviceCloudHttpException: if a non-success response to the request is received - from the device cloud + from Device Cloud :returns: A requests ``Response`` object """ @@ -311,19 +311,19 @@ def put(self, path, data, **kwargs): return self._make_request("PUT", url, data=data, **kwargs) def delete(self, path, retries=DEFAULT_THROTTLE_RETRIES, **kwargs): - """Perform an HTTP DELETE request of the specified path in the device cloud + """Perform an HTTP DELETE request of the specified path in Device Cloud - Make an HTTP DELETE request against the device cloud with this accounts + Make an HTTP DELETE request against Device Cloud with this accounts credentials and base url. This method uses the `requests `_ library `request method `_ and all keyword arguments will be passed on to that method. - :param str path: The device cloud path to DELETE + :param str path: Device Cloud path to DELETE :param int retries: The number of times the request should be retried if an unsuccessful response is received. Most likely, you should leave this at 0. :raises DeviceCloudHttpException: if a non-success response to the request is received - from the device cloud + from Device Cloud :returns: A requests ``Response`` object """ @@ -334,7 +334,7 @@ def delete(self, path, retries=DEFAULT_THROTTLE_RETRIES, **kwargs): class DeviceCloud(object): """Provide access to core device cloud features - This class is the primary interface to the device cloud through which access to individual + This class is the primary interface to Device Cloud through which access to individual device cloud services is provided. Creating a ``DeviceCloud`` object is as easy as doing the following:: @@ -344,7 +344,7 @@ class DeviceCloud(object): if dc.has_valid_credentials(): print list(dc.devicecore.get_devices()) - From there, access to all of the device clouds features are possible. In some cases, methods + From there, access to all of Device Clouds features are possible. In some cases, methods for quickly performing selected actions may be provided directly via the ``DeviceCloud`` object while advanced usage requires using functionality exposed through other interfaces. @@ -374,9 +374,9 @@ def __init__(self, username, password, base_url=None, self._legacy_api = None # legacy property api ref def has_valid_credentials(self): - """Verify that the device cloud url, username, and password are valid + """Verify that Device Cloud url, username, and password are valid - This method will attempt to "ping" the device cloud in order to ensure that all + This method will attempt to "ping" Device Cloud in order to ensure that all of the provided information is correct. :return: True if the credentials are valid and false if not @@ -441,7 +441,7 @@ def get_connection(self): """Get the low-level :class:`~DeviceCloudConnection` for this device cloud instance This object provides a low-level interface for making authenticated requests - to the device cloud. + to Device Cloud. """ return self._conn diff --git a/devicecloud/conditions.py b/devicecloud/conditions.py index 83791c7..b6844ab 100644 --- a/devicecloud/conditions.py +++ b/devicecloud/conditions.py @@ -6,7 +6,7 @@ """Module with functionality for building queries against cloud resources -This functionality is somewhat poorly documented in the device cloud documentation +This functionality is somewhat poorly documented in Device Cloud documentation in the `Compound Queries `_ section. @@ -20,7 +20,7 @@ def _quoted(value): """Return a single-quoted and escaped (percent-encoded) version of value This function will also perform transforms of known data types to a representation - that will be handled by the device cloud. For instance, datetime objects will be + that will be handled by Device Cloud. For instance, datetime objects will be converted to ISO8601. """ @@ -32,7 +32,6 @@ def _quoted(value): return "'{}'".format(value) - class Expression(object): r"""A condition is an evaluable filter diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index cedf6df..566cdea 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -168,7 +168,7 @@ def get_groups(self, condition=None, page_size=1000): def delete_device(self, dev): """ Delete a from the cloud account associated with the handle. - :raises DeviceCloudHttpException: If there is an unexpected error reported by the device cloud. + :raises DeviceCloudHttpException: If there is an unexpected error reported by Device Cloud. :param dev: Device object of the device to delete. :return: the Response from the delete request. """ @@ -206,12 +206,12 @@ def provision_device(self, **kwargs): :param str contact: (optional) Contact associted with this device (or whatever you want). :param str description: (optional) Textual description of this device. - :raises DeviceCloudHttpException: If there is an unexpected error reported by the device cloud. + :raises DeviceCloudHttpException: If there is an unexpected error reported by Device Cloud. :raises ValueError: If any input fields are known to have a bad form. :return: A dictionary matching the format specified above. """ - # This snippet is from the device cloud API Explorer and shows the pieces of + # This snippet is from Device Cloud API Explorer and shows the pieces of # information that may be specified when adding a device. # # @@ -243,7 +243,7 @@ def provision_devices(self, devices): :param list devices: An iterable of dictionaries each containing information about a device to be provision. The form of the dictionary should match the keyword arguments taken by :meth:`provision_device`. - :raises DeviceCloudHttpException: If there is an unexpected error reported by the device cloud. + :raises DeviceCloudHttpException: If there is an unexpected error reported by Device Cloud. :raises ValueError: If any input fields are known to have a bad form. :return: A list of dictionaries in the form described for :meth:`provision_device` in the order matching the requested device list. Note that it is possible for there to @@ -312,7 +312,7 @@ def maybe_write_element(tag, val): class Group(object): - """Provides access to information about a group in the device cloud + """Provides access to information about a group in Device Cloud .. note:: @@ -389,7 +389,7 @@ def get_parent_id(self): class Device(object): - """Interface to a device in the device cloud""" + """Interface to a device in Device Cloud""" # TODO: provide ability to set/update available data items @@ -461,7 +461,7 @@ def get_mac_last4(self, use_cached=True): return mac4.upper() def get_registration_dt(self, use_cached=True): - """Get the datetime of when this device was added to the device cloud""" + """Get the datetime of when this device was added to Device Cloud""" device_json = self.get_device_json(use_cached) start_date_iso8601 = device_json.get("devRecordStartDate") if start_date_iso8601: @@ -523,7 +523,7 @@ def get_global_ip(self, use_cached=True): return self.get_device_json(use_cached).get("dpGlobalIp") def get_last_connected_dt(self, use_cached=True): - """Get the datetime that the device last connected to the device cloud""" + """Get the datetime that the device last connected to Device Cloud""" return iso8601_to_dt(self.get_device_json(use_cached).get("dpLastConnectTime")) def get_contact(self, use_cached=True): diff --git a/devicecloud/file_system_service.py b/devicecloud/file_system_service.py index 20bb522..5791c91 100644 --- a/devicecloud/file_system_service.py +++ b/devicecloud/file_system_service.py @@ -4,7 +4,7 @@ # # Copyright (c) 2015-2018 Digi International Inc. -"""Provide access to the device cloud file system service API""" +"""Provide access to Device Cloud file system service API""" import base64 from collections import namedtuple import xml.etree.ElementTree as ET @@ -308,7 +308,7 @@ def parse_response(cls, response, device_id=None, fssapi=None, **kwargs): if response.tag != cls.command_name: raise ResponseParseError( "Received response of type {}, LsCommand can only parse responses of type {}".format(response.tag, - cls.command_name)) + cls.command_name)) if fssapi is None: raise FileSystemServiceException("fssapi is required to parse an LsCommand response") @@ -526,7 +526,7 @@ def parse_response(cls, response, **kwargs): if response.tag != cls.command_name: raise ResponseParseError( "Received response of type {}, DeleteCommand can only parse responses of type {}".format(response.tag, - cls.command_name)) + cls.command_name)) error = response.find('./error') if error is not None: return _parse_error_tree(error) @@ -552,7 +552,7 @@ def send_command_block(self, target, command_block): :return: The response will be a dictionary where the keys are device_ids and the values are the parsed responses of each command sent in the order listed in the command response for that device. In practice it seems to be the same order as the commands were sent in, however, - the device cloud documentation does not explicitly state anywhere that is the case so I cannot + Device Cloud documentation does not explicitly state anywhere that is the case so I cannot guarantee it. This does mean that if you send different types of commands the response list will be different types. Please see the commands parse_response functions for what those types will be. (:meth:`LsCommand.parse_response`, :class:`GetCommand.parse_response`, diff --git a/devicecloud/filedata.py b/devicecloud/filedata.py index 826f0ac..1e14243 100644 --- a/devicecloud/filedata.py +++ b/devicecloud/filedata.py @@ -4,7 +4,7 @@ # # Copyright (c) 2015-2018 Digi International Inc. -"""Provide access to the device cloud filedata API""" +"""Provide access to Device Cloud filedata API""" import base64 @@ -25,7 +25,7 @@ class FileDataAPI(APIBase): - """Encapsulate data and logic required to interact with the device cloud file data store""" + """Encapsulate data and logic required to interact with Device Cloud file data store""" def get_filedata(self, condition=None, page_size=1000): """Return a generator over all results matching the provided condition @@ -123,9 +123,8 @@ def delete_file(self, path): self._conn.delete("/ws/FileData{path}".format(path=path)) - def walk(self, root="~/"): - """Emulation of os.walk behavior against the device cloud filedata store + """Emulation of os.walk behavior against Device Cloud filedata store This method will yield tuples in the form ``(dirpath, FileDataDirectory's, FileData's)`` recursively in pre-order (depth first from top down). diff --git a/devicecloud/monitor.py b/devicecloud/monitor.py index a2f4146..6a7942f 100644 --- a/devicecloud/monitor.py +++ b/devicecloud/monitor.py @@ -22,7 +22,7 @@ #: Device Cloud customer identifier. MON_CST_ID_ATTR = Attribute("cstId") -#: One or more topics to monitor separated by comma. See the device cloud +#: One or more topics to monitor separated by comma. See Device Cloud #: documentation for more details MON_TOPIC_ATTR = Attribute("monTopic") @@ -109,16 +109,16 @@ class MonitorAPI(APIBase): - """Provide access to the device cloud Monitor API for receiving push notifications + """Provide access to Device Cloud Monitor API for receiving push notifications - The Monitor API in the device cloud allows for the creation and destruction of + The Monitor API in Device Cloud allows for the creation and destruction of multiple "monitors." Each monitor is registered against one or more "topics" which describe the data in which it is interested. There are, in turn, two main ways to receive data matching the topics for a given monitor: - 1. Stream: The device cloud supports a protocol over TCP (optionally with SSL) over which + 1. Stream: Device Cloud supports a protocol over TCP (optionally with SSL) over which the batches of events will be sent when they are received. 2. HTTP: When batches of events are received, a configured web service endpoint will received a POST request with the new data. @@ -133,7 +133,7 @@ class MonitorAPI(APIBase): and associated listener that triggers a callback. Deletion of existing monitors matching the same topics is not necessary but sometimes done in order to ensure that changes to the monitor configuration in code always make it to the monitor - configuration in the device cloud:: + configuration in Device Cloud:: def monitor_callback(json_data): print(json_data) @@ -150,7 +150,7 @@ def monitor_callback(json_data): # later... dc.monitor.stop_listeners() - When updates to any DataPoint in the device cloud occurs, the callback will be called + When updates to any DataPoint in Device Cloud occurs, the callback will be called with a data structure like this one:: {'Document': {'Msg': {'DataPoint': {'cstId': 7603, @@ -175,7 +175,7 @@ def __init__(self, conn): def create_tcp_monitor(self, topics, batch_size=1, batch_duration=0, compression='gzip', format_type='json'): - """Creates a TCP Monitor instance in the device cloud for a given list of topics + """Creates a TCP Monitor instance in Device Cloud for a given list of topics :param topics: a string list of topics (e.g. ['DeviceCore[U]', 'FileDataCore']). @@ -212,7 +212,7 @@ def create_tcp_monitor(self, topics, batch_size=1, batch_duration=0, def create_http_monitor(self, topics, transport_url, transport_token=None, transport_method='PUT', connect_timeout=0, response_timeout=0, batch_size=1, batch_duration=0, compression='none', format_type='json'): - """Creates a HTTP Monitor instance in the device cloud for a given list of topics + """Creates a HTTP Monitor instance in Device Cloud for a given list of topics :param topics: a string list of topics (e.g. ['DeviceCore[U]', 'FileDataCore']). @@ -280,7 +280,7 @@ def get_monitors(self, condition=None, page_size=1000): :param condition: An :class:`.Expression` which defines the condition which must be matched on the monitor that will be retrieved from - the device cloud. If a condition is unspecified, an iterator over + Device Cloud. If a condition is unspecified, an iterator over all monitors for this account will be returned. :type condition: :class:`.Expression` or None :param int page_size: The number of results to fetch in a single page. @@ -310,7 +310,7 @@ def stop_listeners(self): class DeviceCloudMonitor(object): - """Provides access to a single monitor instance on the device cloud + """Provides access to a single monitor instance on Device Cloud This is a base class that should not be instantiated directly. @@ -361,7 +361,7 @@ def get_metadata(self): return self._conn.get_json("/ws/Monitor/{id}".format(id=self._id))["items"][0] def delete(self): - """Delete this monitor form the device cloud""" + """Delete this monitor form Device Cloud""" self._conn.delete("/ws/Monitor/{id}".format(id=self._id)) @@ -377,5 +377,5 @@ def __init__(self, conn, monitor_id, tcp_client_manager): self._tcp_client_manager = tcp_client_manager def add_callback(self, callback): - """Create a secure SSL/TCP listen session to the device cloud""" + """Create a secure SSL/TCP listen session to Device Cloud""" self._tcp_client_manager.create_session(callback, self._id) diff --git a/devicecloud/monitor_tcp.py b/devicecloud/monitor_tcp.py index 0cee12a..bf3d75c 100644 --- a/devicecloud/monitor_tcp.py +++ b/devicecloud/monitor_tcp.py @@ -115,7 +115,7 @@ class PushSession(object): """ def __init__(self, callback, monitor_id, client): - """Creates a PushSession for use with the device cloud + """Creates a PushSession for use with Device Cloud :param callback: The callback function to invoke when data received. Must have 1 required parameter that will contain the payload. @@ -201,7 +201,7 @@ def send_connection_request(self): raise exception def start(self): - """Creates a TCP connection to the device cloud and sends a ConnectionRequest message""" + """Creates a TCP connection to Device Cloud and sends a ConnectionRequest message""" self.log.info("Starting Insecure Session for Monitor %s" % self.monitor_id) if self.socket is not None: raise Exception("Socket already established for %s." % self) @@ -239,7 +239,7 @@ class SecurePushSession(PushSession): def __init__(self, callback, monitor_id, client, ca_certs=None): """ Creates a PushSession wrapped in SSL for use with interacting with - the device cloud push functionality. + Device Cloud push functionality. :param callback: The callback function to invoke when data is received. Must have 1 required parameter that will contain the @@ -357,7 +357,7 @@ def queue_callback(self, session, block_id, data): class TCPClientManager(object): - """A Client for the 'Push' feature in the device cloud""" + """A Client for the 'Push' feature in Device Cloud""" def __init__(self, conn, secure=True, ca_certs=None, workers=1): """ @@ -452,7 +452,7 @@ def _select(self): try: while not self.closed: try: - inputready = select.select(self.sessions.keys(), [], [], 0.1)[0] + inputready = select.select(self.sessions.keys(), [], [], 0.1)[0] for sock in inputready: session = self.sessions[sock] sck = session.socket @@ -501,7 +501,7 @@ def _select(self): # We received full payload, # clear session data and parse it. - data = session.data + data = session.data session.data = six.b("") session.message_length = 0 block_id = struct.unpack('!H', data[0:2])[0] diff --git a/devicecloud/streams.py b/devicecloud/streams.py index 0b77de3..c87c496 100644 --- a/devicecloud/streams.py +++ b/devicecloud/streams.py @@ -78,10 +78,10 @@ def _get_encoder_method(stream_type): def _get_decoder_method(stream_type): - """ A function to get the device cloud type to python type converter function. + """ A function to get Device Cloud type to python type converter function. :param stream_type: The streams data type - :return: A function that when called with the device cloud object will return the python + :return: A function that when called with Device Cloud object will return the python native type. If there is no function for the given type, or the `stream_type` is `None` the returned function will simply return the object unchanged. """ @@ -130,12 +130,12 @@ def _get_streams(self, uri_suffix=None): def create_stream(self, stream_id, data_type, description=None, data_ttl=None, rollup_ttl=None, units=None): - """Create a new data stream on the device cloud + """Create a new data stream on Device Cloud - This method will attempt to create a new data stream on the device cloud. + This method will attempt to create a new data stream on Device Cloud. This method will only succeed if the stream does not already exist. - :param str stream_id: The path/id of the stream being created on the device cloud. + :param str stream_id: The path/id of the stream being created on Device Cloud. :param str data_type: The type of this stream. This must be in the set `{ INTEGER, LONG, FLOAT, DOUBLE, STRING, BINARY, UNKNOWN }`. These values are available in constants like :attr:`~STREAM_TYPE_INTEGER`. @@ -177,7 +177,7 @@ def get_streams(self, stream_prefix=None): :param stream_prefix: An optional prefix to limit the iterator to; all streams are returned if it is not specified. - :return: iterator over all :class:`.DataStream` instances on the device cloud + :return: iterator over all :class:`.DataStream` instances on Device Cloud """ # TODO: deal with paging. We now return a generator, so the interface should look the same @@ -191,7 +191,7 @@ def get_stream(self, stream_id): stream exists, one can use :py:meth:`get_stream_if_exists` which will return None if the stream is not already created. - :param stream_id: The path of the stream on the device cloud + :param stream_id: The path of the stream on Device Cloud :raises TypeError: if the stream_id provided is the wrong type :raises ValueError: if the stream_id is not properly formed :return: datastream instance with the provided stream_id @@ -206,7 +206,7 @@ def get_stream_if_exists(self, stream_id): This works similar to :py:meth:`get_stream` but will return None if the stream is not already created. - :param stream_id: The path of the stream on the device cloud + :param stream_id: The path of the stream on Device Cloud :raises TypeError: if the stream_id provided is the wrong type :raises ValueError: if the stream_id is not properly formed :return: :class:`.DataStream` instance with the provided stream_id @@ -225,7 +225,7 @@ def bulk_write_datapoints(self, datapoints): """Perform a bulk write (or set of writes) of a collection of data points This method takes a list (or other iterable) of datapoints and writes them - to the device cloud in an efficient manner, minimizing the number of HTTP + to Device Cloud in an efficient manner, minimizing the number of HTTP requests that need to be made. As this call is performed from outside the context of any particular stream, @@ -248,14 +248,14 @@ def bulk_write_datapoints(self, datapoints): dc.streams.bulk_write_datapoints(datapoints) Depending on the size of the list of datapoints provided, this method may - need to make multiple calls to the device cloud (in chunks of 250). + need to make multiple calls to Device Cloud (in chunks of 250). - :param list datapoints: a list of datapoints to be written to the device cloud + :param list datapoints: a list of datapoints to be written to Device Cloud :raises TypeError: if a list of datapoints is not provided :raises ValueError: if any of the provided data points do not have all required information (such as information about the stream) :raises DeviceCloudHttpException: in the case of an unexpected error in communicating - with the device cloud. + with Device Cloud. """ datapoints = list(datapoints) # effectively performs validation that we have the right type @@ -288,7 +288,7 @@ class DataPoint(object): This class encapsulates the data required for both pushing data points to the device cloud as well as for storing and provding methods to access data from - streams that has been retrieved from the device cloud. + streams that has been retrieved from Device Cloud. """ @@ -297,7 +297,7 @@ def from_json(cls, stream, json_data): """Create a new DataPoint object from device cloud JSON data :param DataStream stream: The :class:`~DataStream` out of which this data is coming - :param dict json_data: Deserialized JSON data from the device cloud about this device + :param dict json_data: Deserialized JSON data from Device Cloud about this device :raises ValueError: if the data is malformed :return: (:class:`~DataPoint`) newly created :class:`~DataPoint` @@ -325,7 +325,7 @@ def from_rollup_json(cls, stream, json_data): """Rollup json data from the server looks slightly different :param DataStream stream: The :class:`~DataStream` out of which this data is coming - :param dict json_data: Deserialized JSON data from the device cloud about this device + :param dict json_data: Deserialized JSON data from Device Cloud about this device :raises ValueError: if the data is malformed :return: (:class:`~DataPoint`) newly created :class:`~DataPoint` """ @@ -475,7 +475,7 @@ def get_quality(self): def set_quality(self, quality): """Set the quality for this sample - Quality is stored on the device cloud as a 32-bit integer, so the input + Quality is stored on Device Cloud as a 32-bit integer, so the input to this function should be either None, an integer, or a string that can be converted to an integer. @@ -638,7 +638,7 @@ def __repr__(self): )) def _get_stream_metadata(self, use_cached): - """Retrieve metadata about this stream from the device cloud""" + """Retrieve metadata about this stream from Device Cloud""" if self._cached_data is None or not use_cached: try: self._cached_data = self._conn.get_json("/ws/DataStream/%s" % self._stream_id)["items"][0] @@ -674,7 +674,7 @@ def get_data_type(self, use_cached=True): * BINARY - Data with this type map to a python string. * UNKNOWN - Data with this type map to a python string. - :param bool use_cached: If False, the function will always request the latest from the device cloud. + :param bool use_cached: If False, the function will always request the latest from Device Cloud. If True, the device will not make a request if it already has cached data. :return: The data type of this stream as a string :rtype: str @@ -690,7 +690,7 @@ def get_units(self, use_cached=True): Units are a user-defined field stored as a string - :param bool use_cached: If False, the function will always request the latest from the device cloud. + :param bool use_cached: If False, the function will always request the latest from Device Cloud. If True, the device will not make a request if it already has cached data. :return: The unit of this stream as a string :rtype: str or None @@ -701,7 +701,7 @@ def get_units(self, use_cached=True): def get_description(self, use_cached=True): """Get the description associated with this data stream - :param bool use_cached: If False, the function will always request the latest from the device cloud. + :param bool use_cached: If False, the function will always request the latest from Device Cloud. If True, the device will not make a request if it already has cached data. :raises devicecloud.DeviceCloudHttpException: in the case of an unexpected http error :raises devicecloud.streams.NoSuchStreamException: if this stream has not yet been created @@ -717,7 +717,7 @@ def get_data_ttl(self, use_cached=True): The dataTtl is the time to live (TTL) in seconds for data points stored in the data stream. A data point expires after the configured amount of time and is automatically deleted. - :param bool use_cached: If False, the function will always request the latest from the device cloud. + :param bool use_cached: If False, the function will always request the latest from Device Cloud. If True, the device will not make a request if it already has cached data. :raises devicecloud.DeviceCloudHttpException: in the case of an unexpected http error :raises devicecloud.streams.NoSuchStreamException: if this stream has not yet been created @@ -736,7 +736,7 @@ def get_rollup_ttl(self, use_cached=True): stored in the stream. A roll-up expires after the configured amount of time and is automatically deleted. - :param bool use_cached: If False, the function will always request the latest from the device cloud. + :param bool use_cached: If False, the function will always request the latest from Device Cloud. If True, the device will not make a request if it already has cached data. :raises devicecloud.DeviceCloudHttpException: in the case of an unexpected http error :raises devicecloud.streams.NoSuchStreamException: if this stream has not yet been created @@ -752,7 +752,7 @@ def get_current_value(self, use_cached=False): The current value is the last recorded data point for this stream. - :param bool use_cached: If False, the function will always request the latest from the device cloud. + :param bool use_cached: If False, the function will always request the latest from Device Cloud. If True, the device will not make a request if it already has cached data. :raises devicecloud.DeviceCloudHttpException: in the case of an unexpected http error :raises devicecloud.streams.NoSuchStreamException: if this stream has not yet been created @@ -767,7 +767,7 @@ def get_current_value(self, use_cached=False): return None def delete(self): - """Delete this stream from the device cloud along with its history + """Delete this stream from Device Cloud along with its history This call will return None on success and raise an exception in the event of an error performing the deletion. @@ -872,7 +872,7 @@ def write(self, datapoint): Values already set on the datapoint will not be overridden (except for path) - :param DataPoint datapoint: The :class:`.DataPoint` that should be written to the device cloud + :param DataPoint datapoint: The :class:`.DataPoint` that should be written to Device Cloud """ if not isinstance(datapoint, DataPoint): @@ -889,7 +889,7 @@ def read(self, start_time=None, end_time=None, use_client_timeline=True, newest_ """Read one or more DataPoints from a stream .. warning:: - The data points from the device cloud is a paged data set. When iterating over the + The data points from Device Cloud is a paged data set. When iterating over the result set there could be delays when we hit the end of a page. If this is undesirable, the caller should collect all results into a data structure first before iterating over the result set. diff --git a/devicecloud/test/unit/test_streams.py b/devicecloud/test/unit/test_streams.py index 6c3d8e8..c7bc119 100644 --- a/devicecloud/test/unit/test_streams.py +++ b/devicecloud/test/unit/test_streams.py @@ -499,7 +499,7 @@ def test_write_simple(self): data=123.4, )) - # verify that the body sent to the device cloud is sufficiently minimal + # verify that the body sent to Device Cloud is sufficiently minimal self.assertEqual( httpretty.last_request().body, six.b('' @@ -519,7 +519,7 @@ def test_write_full(self): units="scolvilles", )) - # verify that the body sent to the device cloud is sufficiently minimal + # verify that the body sent to Device Cloud is sufficiently minimal self.assertEqual( httpretty.last_request().body, six.b('' diff --git a/devicecloud/test/unit/test_utilities.py b/devicecloud/test/unit/test_utilities.py index c080465..57dc6a1 100644 --- a/devicecloud/test/unit/test_utilities.py +++ b/devicecloud/test/unit/test_utilities.py @@ -15,7 +15,7 @@ class HttpTestBase(unittest.TestCase): def setUp(self): httpretty.enable() - # setup the Device cloud ping response + # setup Device Cloud ping response self.prepare_response("GET", "/ws/DeviceCore?size=1", "", status=200) self.dc = DeviceCloud('user', 'pass') diff --git a/devicecloud/util.py b/devicecloud/util.py index 00e1698..e573c77 100644 --- a/devicecloud/util.py +++ b/devicecloud/util.py @@ -17,7 +17,7 @@ def conditional_write(strm, fmt, value, *args, **kwargs): def iso8601_to_dt(iso8601): - """Given an ISO8601 string as returned by the device cloud, convert to a datetime object""" + """Given an ISO8601 string as returned by Device Cloud, convert to a datetime object""" # We could just use arrow.get() but that is more permissive than we actually want. # Internal (but still public) to arrow is the actual parser where we can be # a bit more specific diff --git a/docs/conf.py b/docs/conf.py index 1a9ef60..3ddf4e3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -55,7 +55,7 @@ # General information about the project. project = u'python-devicecloud' -copyright = u'2015, Digi International, Inc.' +copyright = u'2015-2018, Digi International Inc.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -193,22 +193,22 @@ # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # Additional stuff for the LaTeX preamble. + #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'python-devicecloud.tex', u'python-devicecloud Documentation', - u'Paul Osborne, Tom Manley, Stephen Stack', 'manual'), + ('index', 'python-devicecloud.tex', u'python-devicecloud Documentation', + u'Paul Osborne, Tom Manley, Stephen Stack, Brandon Moser', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -251,10 +251,10 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'python-devicecloud', u'python-devicecloud Documentation', - u'Paul Osborne, Tom Manley, Stephen Stack', 'python-devicecloud', - 'Device Cloud Web Services Client for Python', - 'Miscellaneous'), + ('index', 'python-devicecloud', u'python-devicecloud Documentation', + u'Paul Osborne, Tom Manley, Stephen Stack', 'python-devicecloud', + 'Device Cloud Web Services Client for Python', + 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. diff --git a/docs/cookbook.rst b/docs/cookbook.rst index 170907a..89085c0 100644 --- a/docs/cookbook.rst +++ b/docs/cookbook.rst @@ -158,8 +158,30 @@ First, get a reference to the device which you would like to add a specific grou Then you can add it to a group and fetch it to make sure it works:: device.add_to_group('mygroup') - device.get_group_path() # prints 'mygroup' (the DC sometimes needs a second to catch up) + device.get_group_path() # prints 'mygroup' (DC sometimes needs a second to catch up) Or remove it:: device.remove_from_group() + + +Device Core - Tags +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. note:: + + This assumes your device is provisioned. + +Similar to Groups, get a reference to the device which you would like to add a specific group:: + + device = devicecore.get_device('00:40:9D:50:B0:EA') + +Then you can add a tag and then get the new list:: + + device.add_tags('mytag') + device.get_tags() # prints ['mytags'] (DC sometimes needs a second to catch up) + +Or remove it:: + + device.remove_tag('mytag') + device.get_tags() # prints [] (DC sometimes needs a second to catch up) diff --git a/docs/filedata.rst b/docs/filedata.rst index 042a667..87d104f 100644 --- a/docs/filedata.rst +++ b/docs/filedata.rst @@ -4,7 +4,7 @@ FileData API FileData Overview ----------------- -The FileData store on the device cloud provides a hierarchical mechanism for temporarily +The FileData store on Device Cloud provides a hierarchical mechanism for temporarily storing information in files sent from devices. With the APIs provided by the device cloud, it is possible to use the FileData store in a number of different ways to implement various use cases. @@ -20,7 +20,7 @@ basis. If using the FileData store as a queue, one will likely want to setup mo on FileData paths matching certain criterion. The set of files matching some condition can then be thought of as a channel. -This library seeks to make using the device cloud for both of these use cases simple +This library seeks to make using Device Cloud for both of these use cases simple and robust. Navigating the FileData Store @@ -129,7 +129,7 @@ API Documentation ----------------- The filedata module provides function for reading, writing, and -deleting "files" from the device cloud FileData store. +deleting "files" from Device Cloud FileData store. .. automodule:: devicecloud.filedata :members: diff --git a/docs/filesystem.rst b/docs/filesystem.rst index 9622665..94ab1ce 100644 --- a/docs/filesystem.rst +++ b/docs/filesystem.rst @@ -4,7 +4,7 @@ File System Service API File System Service Overview ---------------------------- -Provide access to the device cloud File System commands that use SCI to +Provide access to Device Cloud File System commands that use SCI to get the data from your devices connected to the cloud. File System Service API Documentation diff --git a/docs/index.rst b/docs/index.rst index 5cab9a1..e176e8a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,15 +25,15 @@ the `Digi Device Cloud `_ for clients written in Python. -The library wraps the Device Cloud REST API and hides the details of +The library wraps Device Cloud REST API and hides the details of forming HTTP requests in order to gain access to device information, -file data, streams, and other features of the device cloud. The API +file data, streams, and other features of Device Cloud. The API wrapped can be found `here `_. The primary target audience for this library is individuals -interfacing with the device cloud from the server side or developers +interfacing with Device Cloud from the server side or developers writing tools to aid device development. For efficient connectivity from devices, we suggest that you first look at using the `Device Cloud Connector `_. @@ -49,7 +49,7 @@ quick example of what the API looks like:: # show the MAC address of all devices that are currently connected # - # This is done using the device cloud DeviceCore functionality + # This is done using Device Cloud DeviceCore functionality print "== Connected Devices ==" for device in dc.devicecore.get_devices(): if device.is_connected(): @@ -58,7 +58,7 @@ quick example of what the API looks like:: # get the name and current value of all data streams having values # with a floating point type # - # This is done using the device cloud stream functionality + # This is done using Device Cloud stream functionality for stream in dc.streams.get_streams(): if stream.get_data_type().lower() in ('float', 'double'): print "%s -> %s" % (stream.get_stream_id(), stream.get_current_value()) diff --git a/docs/monitor.rst b/docs/monitor.rst index df57f47..31356d0 100644 --- a/docs/monitor.rst +++ b/docs/monitor.rst @@ -4,9 +4,9 @@ Monitor API Monitor Overview ---------------- -Provide access to the device cloud monitor API which can be used to +Provide access to Device Cloud monitor API which can be used to subscribe to topics to receive notifications when data is received -on the device cloud. +on Device Cloud. SCI API Documentation --------------------- diff --git a/docs/sci.rst b/docs/sci.rst index e3281ff..7f7dbb2 100644 --- a/docs/sci.rst +++ b/docs/sci.rst @@ -4,8 +4,8 @@ SCI (Server Command Interface) API SCI Overview ------------ -Provide access to the device cloud Server Command Interface used for -sending messages to devices connected to the device cloud. +Provide access to Device Cloud Server Command Interface used for +sending messages to devices connected to Device Cloud. SCI API Documentation --------------------- diff --git a/docs/streams.rst b/docs/streams.rst index f78628c..5cbef2d 100644 --- a/docs/streams.rst +++ b/docs/streams.rst @@ -4,12 +4,12 @@ Streams API Streams Overview ---------------- -Data Streams on the device cloud provide a mechanism for storing time-series +Data Streams on Device Cloud provide a mechanism for storing time-series values over a long period of time. Each individual value in the time series is known as a Data Point. -There are a few basic operations supported by the device cloud on streams which -are supported by the device cloud and this library. Here we give examples of +There are a few basic operations supported by Device Cloud on streams which +are supported by Device Cloud and this library. Here we give examples of each. Listing Streams @@ -98,7 +98,7 @@ DataPoint objects The :class:`.DataPoint` class encapsulates all information required for both writing data points as well as retrieving information about data -points stored on the device cloud. +points stored on Device Cloud. API Documentation ----------------- diff --git a/docs/ws.rst b/docs/ws.rst index ee2a4a5..fe0bb81 100644 --- a/docs/ws.rst +++ b/docs/ws.rst @@ -1,7 +1,7 @@ Direct Web Services API ======================= -The Device Cloud exposes a large set of functionality to users and the +Device Cloud exposes a large set of functionality to users and the python-devicecloud library seeks to provide convenient and complete APIs for a majority of these. However, there are APIs which the library does not cover; some may have coverage in the future and others may never From 5d52ecb082a68edb046d9cd33c78a60acd275d70 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Tue, 24 Jul 2018 00:12:40 -0500 Subject: [PATCH 101/140] Add coverage to Travis-CI builds --- .gitignore | 2 +- .travis.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e768771..4af4102 100644 --- a/.gitignore +++ b/.gitignore @@ -73,4 +73,4 @@ tmp/ #------------------------------------------------------------------------------- # Generated Files #------------------------------------------------------------------------------- -README.rst \ No newline at end of file +README.rst diff --git a/.travis.yml b/.travis.yml index 3e18590..1124229 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ python: 2.7 env: - TOX_ENV=py27 - TOX_ENV=py34 + - TOX_ENV=coverage # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors install: From 84bddaa429ef1fb66ae6a97b2f72feba876b9455 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Tue, 24 Jul 2018 00:13:21 -0500 Subject: [PATCH 102/140] Remove empty tags, do not duplicate tag on add --- devicecloud/devicecore.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index 566cdea..ec06659 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -420,7 +420,7 @@ def get_tags(self, use_cached=True): device_json = self.get_device_json(use_cached) potential_tags = device_json.get("dpTags") if potential_tags: - return potential_tags.split(",") + return list(filter(None, potential_tags.split(","))) else: return [] @@ -598,20 +598,21 @@ def remove_from_group(self): self._device_json = None def add_tag(self, tag): - """Add a tag to existing device tags + """Add a tag to existing device tags. This method will not add a duplicate, if already in the list. :param tag: the tag to be added """ tags = self.get_tags() - tags.append(tag) + if not tag in tags: + tags.append(tag) - post_data = TAGS_TEMPLATE.format(connectware_id=self.get_connectware_id(), - tags=",".join(tags)) - self._conn.put('/ws/DeviceCore', post_data) + post_data = TAGS_TEMPLATE.format(connectware_id=self.get_connectware_id(), + tags=",".join(tags)) + self._conn.put('/ws/DeviceCore', post_data) - # Invalidate cache - self._device_json = None + # Invalidate cache + self._device_json = None def remove_tag(self, tag): """Remove tag from existing device tags From 54d532b97f9be35fe7640e7cac51842aae8ceb51 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Tue, 24 Jul 2018 00:28:38 -0500 Subject: [PATCH 103/140] Try to fix coverage on travi-ci --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1124229..70e51a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,10 +19,12 @@ env: # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors install: - - pip install tox + - pip install tox-travis coveralls - pip install -r test-requirements.txt # command to run tests, e.g. python setup.py test script: - - tox -e $TOX_ENV + - tox +after_success: + - coveralls From 503f242b8b6a1360533136748271ca178e21c765 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Tue, 24 Jul 2018 00:32:11 -0500 Subject: [PATCH 104/140] release: version 0.5.1 -- "4 - Beta on pypi Signed-off-by: Brandon Moser --- devicecloud/version.py | 3 ++- setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/devicecloud/version.py b/devicecloud/version.py index dfc6b43..cc6d822 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -4,4 +4,5 @@ # # Copyright (c) 2015-2018 Digi International Inc. -__version__ = "0.5.0" + +__version__ = "0.5.1" diff --git a/setup.py b/setup.py index 2a2713a..142bf45 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ def get_long_description(): packages=find_packages(), install_requires=open('requirements.txt').read().split(), classifiers=[ - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", "Programming Language :: Python :: 2.7", From d7d05ff479b9e2ff4dcfce15a49f5c8cd135ab4a Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Tue, 24 Jul 2018 15:01:10 -0500 Subject: [PATCH 105/140] Add support for adding multiple tags, escape XML chars --- devicecloud/devicecore.py | 29 ++++++++++---- devicecloud/test/unit/test_devicecore.py | 50 ++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index ec06659..2c49d98 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -11,6 +11,7 @@ from devicecloud.conditions import Attribute, Expression from devicecloud.util import iso8601_to_dt, validate_type import six +from xml.sax.saxutils import escape dev_mac = Attribute('devMac') @@ -597,22 +598,34 @@ def remove_from_group(self): # Invalidate cache self._device_json = None - def add_tag(self, tag): + def add_tag(self, new_tags): """Add a tag to existing device tags. This method will not add a duplicate, if already in the list. - :param tag: the tag to be added + :param new_tags: the tag(s) to be added. new_tags can be a comma-separated string or list """ tags = self.get_tags() - if not tag in tags: - tags.append(tag) + orig_tag_cnt = len(tags) + print("self.get_tags() {}".format(tags)) + if isinstance(new_tags, six.string_types): + new_tags = new_tags.split(',') + print("spliting tags :: {}".format(new_tags)) + + for tag in new_tags: + if not tag in tags: + tags.append(tag.strip()) + + if len(tags) > orig_tag_cnt: + xml_tags = escape(",".join(tags)) post_data = TAGS_TEMPLATE.format(connectware_id=self.get_connectware_id(), - tags=",".join(tags)) + tags=xml_tags) self._conn.put('/ws/DeviceCore', post_data) # Invalidate cache - self._device_json = None + # self._device_json = None + else: + print("skipping tag update") def remove_tag(self, tag): """Remove tag from existing device tags @@ -626,8 +639,8 @@ def remove_tag(self, tag): tags.remove(tag) post_data = TAGS_TEMPLATE.format(connectware_id=self.get_connectware_id(), - tags=",".join(tags)) + tags=escape(",".join(tags))) self._conn.put('/ws/DeviceCore', post_data) # Invalidate cache - self._device_json = None + # self._device_json = None diff --git a/devicecloud/test/unit/test_devicecore.py b/devicecloud/test/unit/test_devicecore.py index fd57759..8ab1fcd 100644 --- a/devicecloud/test/unit/test_devicecore.py +++ b/devicecloud/test/unit/test_devicecore.py @@ -16,6 +16,7 @@ from devicecloud.devicecore import ADD_GROUP_TEMPLATE, TAGS_TEMPLATE import six import mock +from xml.sax.saxutils import escape EXAMPLE_GET_DEVICES = { @@ -478,6 +479,55 @@ def test_add_device_tag(self): self.assertIsNone(dev._device_json) self.assertEqual(six.b(expected), httpretty.last_request().body) + def test_add_multiple_tags(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + expected = TAGS_TEMPLATE.format(connectware_id=dev.get_connectware_id(), + tags='test,test2,test3') + dev.add_tag('test,test2,test3') + self.assertIsNone(dev._device_json) + self.assertEqual(six.b(expected), httpretty.last_request().body) + + def test_add_tag_list(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + tags = ['test', 'test2', 'test3'] + expected = TAGS_TEMPLATE.format(connectware_id=dev.get_connectware_id(), + tags="{}".format(",".join(tags))) + dev.add_tag(tags) + self.assertIsNone(dev._device_json) + self.assertEqual(six.b(expected), httpretty.last_request().body) + + def test_add_tags_with_spaces(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + tags = 'test, test2, test3, compound tag' + clean_tags = [t.strip() for t in tags.split(',')] + expected = TAGS_TEMPLATE.format(connectware_id=dev.get_connectware_id(), + tags="{}".format(",".join(clean_tags))) + dev.add_tag(tags) + self.assertIsNone(dev._device_json) + self.assertEqual(six.b(expected), httpretty.last_request().body) + + def test_add_tags_with_special_chars(self): + self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) + self.prepare_response("PUT", "/ws/DeviceCore", '') + gen = self.dc.devicecore.get_devices(page_size=1) + dev = six.next(gen) + tags = 'test, test2, test3, this & that, < more >' + clean_tags = [t.strip() for t in tags.split(',')] + expected = TAGS_TEMPLATE.format(connectware_id=dev.get_connectware_id(), + tags=escape("{}".format(",".join(clean_tags)))) + dev.add_tag(tags) + self.assertIsNone(dev._device_json) + self.assertEqual(six.b(expected), httpretty.last_request().body) + def test_remove_device_tag(self): self.prepare_json_response("GET", "/ws/DeviceCore", EXAMPLE_GET_DEVICES) self.prepare_response("PUT", "/ws/DeviceCore", '') From 0fffdca5844f5a478e06b66ba519b3f59876df02 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Tue, 24 Jul 2018 15:01:37 -0500 Subject: [PATCH 106/140] release: 0.5.2 -- multiple add tags support Signed-off-by: Brandon Moser --- devicecloud/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devicecloud/version.py b/devicecloud/version.py index cc6d822..2fe5e92 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -5,4 +5,4 @@ # Copyright (c) 2015-2018 Digi International Inc. -__version__ = "0.5.1" +__version__ = "0.5.2" From 394d0f2979634b2181bc98e5e53c6797c46c4579 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Tue, 24 Jul 2018 15:08:17 -0500 Subject: [PATCH 107/140] Remove print stmnts, clear device cache --- devicecloud/devicecore.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/devicecloud/devicecore.py b/devicecloud/devicecore.py index 2c49d98..c5b5ee0 100644 --- a/devicecloud/devicecore.py +++ b/devicecloud/devicecore.py @@ -606,11 +606,11 @@ def add_tag(self, new_tags): tags = self.get_tags() orig_tag_cnt = len(tags) - print("self.get_tags() {}".format(tags)) + # print("self.get_tags() {}".format(tags)) if isinstance(new_tags, six.string_types): new_tags = new_tags.split(',') - print("spliting tags :: {}".format(new_tags)) + # print("spliting tags :: {}".format(new_tags)) for tag in new_tags: if not tag in tags: @@ -623,9 +623,9 @@ def add_tag(self, new_tags): self._conn.put('/ws/DeviceCore', post_data) # Invalidate cache - # self._device_json = None - else: - print("skipping tag update") + self._device_json = None + # else: + # print("skipping tag update") def remove_tag(self, tag): """Remove tag from existing device tags @@ -643,4 +643,4 @@ def remove_tag(self, tag): self._conn.put('/ws/DeviceCore', post_data) # Invalidate cache - # self._device_json = None + self._device_json = None From 3cf1bf7b085dc91092a5676c08bd53c95a2949a1 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Tue, 24 Jul 2018 15:14:10 -0500 Subject: [PATCH 108/140] release: bump version to 0.5.3, adding changelog and removed print stmts Signed-off-by: Brandon Moser --- CHANGELOG.md | 16 ++++++++++++++++ devicecloud/version.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba821e4..75c54ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ ## Python Devicecloud Library Changelog +### 0.5.3 / 2018-07-24 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.1...0.5.3) + +Enhancements: + +* devicecore: add support for adding multiple tags + + +### 0.5.1 / 2018-07-20 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.0...0.5.1) + +Changes: + +* pypi: upgrade Development Status to "4 - Beta" + + ### 0.5.0 / 2018-07-20 [Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.4.3...0.5.0) diff --git a/devicecloud/version.py b/devicecloud/version.py index 2fe5e92..34f635c 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -5,4 +5,4 @@ # Copyright (c) 2015-2018 Digi International Inc. -__version__ = "0.5.2" +__version__ = "0.5.3" From 4fee945427337174cda84ea5a9cb080c7ad805e0 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Fri, 27 Jul 2018 10:59:24 -0500 Subject: [PATCH 109/140] Rename HACKING to CONTRIBUTING to match expectations/Github Standards --- HACKING.md => CONTRIBUTING.md | 200 +++++++++++++++++----------------- README.md | 2 +- 2 files changed, 101 insertions(+), 101 deletions(-) rename HACKING.md => CONTRIBUTING.md (97%) diff --git a/HACKING.md b/CONTRIBUTING.md similarity index 97% rename from HACKING.md rename to CONTRIBUTING.md index e849465..384a262 100644 --- a/HACKING.md +++ b/CONTRIBUTING.md @@ -1,100 +1,100 @@ -Developer's Guide -================= - -Environment Setup ------------------ - -All the requirements in order to perform development on the product -should be installable in a virtualenv. - - $ pip install -r dev-requirements.txt - -In order to build a release you will also need to install pandoc. On -Ubuntu, you should be able to do: - - $ sudo apt-get install pandoc - - -Running the Unit Tests ----------------------- - -### Running Tests with Nose - -Running the tests is easy with nose (included in -test-requirements.txt). From the project root: - - $ nosetests . - -To run the unit tests with coverage results (view cover/index.html), -do the following: - - $ nosetests --with-coverage --cover-html --cover-package=devicecloud . - -New contributions to the library will only be accepted if they include -unit test coverage (with some exceptions). - -### Testing All Versions with Tox - -We also support running the tests against all supported versions of -python using a combination of -[tox](http://tox.readthedocs.org/en/latest/) and -[pyenv](https://github.com/yyuu/pyenv). To run all of the tests -against all supported versions of python, just do the following: - - $ ./toxtest.sh - -This might take awhile the first time as it will build from source a -version of the interpreter for each version supported. If you recieve -errors from pyenv, there may be addition dependencies required. -Please visit https://github.com/yyuu/pyenv/wiki/Common-build-problems -for additional pointers. - -### Running Integration and Unittests - -There are some additional integration tests that run against an actual -device cloud account. These are a bit more fragile and when something -fails, you may need to go to your device cloud account to clean things -up. - -To run those tests, you can just do the following. This script runs -the toxtest.sh script with environment variables set with your -account information. The tests that were skipped before will now -be run with each supported version of the interpreter: - - $ ./inttest.sh - -Build the Documentation ------------------------ - -Documentation (outside of this file and the README) is done via -Sphinx. To build the docs, just do the following (with virtualenv -activated): - - $ cd docs - $ make html - -The docs that are built will be located at -docs/_build/html/index.html. - -The documentation for the project is published on github using a [Github -Pages](https://pages.github.com/) Project Site. The process for -releasing a new set of documentation is the following: - -1. Create a fresh clone of the project and checkout the `gh-pages` - branch. Although this is the same repo, the tree is completely - separate from the main python-devicecloud codebase. -2. Remove all contents from the working area -3. From the python-devicecloud repo, `cp -r docs/_build/html/* - /path/to/other/repo/`. -4. Commit and push the update `gh-pages` branch to github - -Open Source License Header --------------------------- - -Each source file should be prefixed with the following header: - - # This Source Code Form is subject to the terms of the Mozilla Public - # License, v. 2.0. If a copy of the MPL was not distributed with this - # file, You can obtain one at http://mozilla.org/MPL/2.0/. - # - # Copyright (c) 2015-2018 Digi International Inc. All rights reserved. +Developer's Guide +================= + +Environment Setup +----------------- + +All the requirements in order to perform development on the product +should be installable in a virtualenv. + + $ pip install -r dev-requirements.txt + +In order to build a release you will also need to install pandoc. On +Ubuntu, you should be able to do: + + $ sudo apt-get install pandoc + + +Running the Unit Tests +---------------------- + +### Running Tests with Nose + +Running the tests is easy with nose (included in +test-requirements.txt). From the project root: + + $ nosetests . + +To run the unit tests with coverage results (view cover/index.html), +do the following: + + $ nosetests --with-coverage --cover-html --cover-package=devicecloud . + +New contributions to the library will only be accepted if they include +unit test coverage (with some exceptions). + +### Testing All Versions with Tox + +We also support running the tests against all supported versions of +python using a combination of +[tox](http://tox.readthedocs.org/en/latest/) and +[pyenv](https://github.com/yyuu/pyenv). To run all of the tests +against all supported versions of python, just do the following: + + $ ./toxtest.sh + +This might take awhile the first time as it will build from source a +version of the interpreter for each version supported. If you recieve +errors from pyenv, there may be addition dependencies required. +Please visit https://github.com/yyuu/pyenv/wiki/Common-build-problems +for additional pointers. + +### Running Integration and Unittests + +There are some additional integration tests that run against an actual +device cloud account. These are a bit more fragile and when something +fails, you may need to go to your device cloud account to clean things +up. + +To run those tests, you can just do the following. This script runs +the toxtest.sh script with environment variables set with your +account information. The tests that were skipped before will now +be run with each supported version of the interpreter: + + $ ./inttest.sh + +Build the Documentation +----------------------- + +Documentation (outside of this file and the README) is done via +Sphinx. To build the docs, just do the following (with virtualenv +activated): + + $ cd docs + $ make html + +The docs that are built will be located at +docs/_build/html/index.html. + +The documentation for the project is published on github using a [Github +Pages](https://pages.github.com/) Project Site. The process for +releasing a new set of documentation is the following: + +1. Create a fresh clone of the project and checkout the `gh-pages` + branch. Although this is the same repo, the tree is completely + separate from the main python-devicecloud codebase. +2. Remove all contents from the working area +3. From the python-devicecloud repo, `cp -r docs/_build/html/* + /path/to/other/repo/`. +4. Commit and push the update `gh-pages` branch to github + +Open Source License Header +-------------------------- + +Each source file should be prefixed with the following header: + + # This Source Code Form is subject to the terms of the Mozilla Public + # License, v. 2.0. If a copy of the MPL was not distributed with this + # file, You can obtain one at http://mozilla.org/MPL/2.0/. + # + # Copyright (c) 2015-2018 Digi International Inc. All rights reserved. diff --git a/README.md b/README.md index cc16ef9..6ebf0c0 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ Contributing Contributions to the library are very welcome in whatever form can be provided. This could include issue reports, bug fixes, or features additions. For issue reports, please [create an issue against the Github project](https://github.com/digidotcom/python-devicecloud/issues). -For code changes, feel free to fork the project on Github and submit a pull request with your changes. Additional instructions for developers contributing to the project can be found in the [Developer's Guide](https://github.com/digidotcom/python-devicecloud/blob/master/HACKING.md). +For code changes, feel free to fork the project on Github and submit a pull request with your changes. Additional instructions for developers contributing to the project can be found in the [Developer's Guide](https://github.com/digidotcom/python-devicecloud/blob/master/CONTRIBUTING.md). License ------- From 74d79b7413db2df82d273df7b57190578973771a Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Fri, 27 Jul 2018 10:59:48 -0500 Subject: [PATCH 110/140] Make sure all files have license --- setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.py b/setup.py index 142bf45..f9ea6ba 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright (c) 2015-2018 Digi International Inc. + import re import os from setuptools import setup, find_packages From 32529684a348a7830a269c32601604c78036bcb8 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Wed, 8 Aug 2018 14:43:47 -0500 Subject: [PATCH 111/140] Add more items to ignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 4af4102..2838413 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,8 @@ env3/ .env/ .env3/ .toxenv/ +venv/ +.venv/ #------------------------------------------------------------------------------- # Setuptools Stuff @@ -74,3 +76,5 @@ tmp/ # Generated Files #------------------------------------------------------------------------------- README.rst +*.csv +*.log From 72b75b48b6c86a4bb99a3abe2147dc1d4d979294 Mon Sep 17 00:00:00 2001 From: mlchan Date: Mon, 20 May 2019 08:23:10 -0400 Subject: [PATCH 112/140] ADded attribute to send sci operaton tag Signed-off-by: mlchan --- devicecloud/sci.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/devicecloud/sci.py b/devicecloud/sci.py index 134af77..8baf9af 100644 --- a/devicecloud/sci.py +++ b/devicecloud/sci.py @@ -12,7 +12,7 @@ SCI_TEMPLATE = """\ - <{operation}{reply}{synchronous}{cache}{sync_timeout}{allow_offline}{wait_for_reconnect}> + <{operation}{attribute}{reply}{synchronous}{cache}{sync_timeout}{allow_offline}{wait_for_reconnect}> {targets} @@ -131,7 +131,7 @@ def send_sci_async(self, operation, target, payload, **sci_options): return AsyncRequestProxy(job_id, self._conn) def send_sci(self, operation, target, payload, reply=None, synchronous=None, sync_timeout=None, - cache=None, allow_offline=None, wait_for_reconnect=None): + cache=None, allow_offline=None, wait_for_reconnect=None, attribute=None): """Send SCI request to 1 or more targets :param str operation: The operation is one of {send_message, update_firmware, disconnect, query_firmware_targets, @@ -204,6 +204,11 @@ def send_sci(self, operation, target, payload, reply=None, synchronous=None, syn else: wait_for_reconnect_xml = '' + if attribute is not None: + operation_attribute = ' ' + attribute + else: + operation_attribute = '' + full_request = SCI_TEMPLATE.format( operation=operation, targets=targets_xml, @@ -213,7 +218,8 @@ def send_sci(self, operation, target, payload, reply=None, synchronous=None, syn cache=cache_xml, allow_offline=allow_offline_xml, wait_for_reconnect=wait_for_reconnect_xml, - payload=payload + payload=payload, + attribute=operation_attribute ) # TODO: do parsing here? From 65eaac5934392c35ad86d513d0e71fcca2ad8212 Mon Sep 17 00:00:00 2001 From: mlchan Date: Wed, 22 May 2019 11:13:38 -0400 Subject: [PATCH 113/140] Added test_sci_update_firmware_attribute test for testing attribute parameter in send_sci function Signed-off-by: mlchan --- devicecloud/test/unit/test_sci.py | 31 +++++++++++++++++++++++++++++++ devicecloud/version.py | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/devicecloud/test/unit/test_sci.py b/devicecloud/test/unit/test_sci.py index f7195f9..0428f64 100644 --- a/devicecloud/test/unit/test_sci.py +++ b/devicecloud/test/unit/test_sci.py @@ -74,6 +74,16 @@ """ +EXAMPLE_UPDATE_FIRMWARE_INVALID_ATTRIBUTE_REQUEST_PAYLOAD = """ +aHNxcAbAADUct1cAAACAHEBAAAEABEAwAIBAAQAAACOFFzU +""" + +EXAMPLE_UPDATE_FIRMWARE_INVALID_ATTRIBUTE_RESPONSE = """\ + +Default target not availabe, specify a target number of filename + + +""" class TestSCI(HttpTestBase): def _prepare_sci_response(self, response, status=200): @@ -179,6 +189,27 @@ def test_sci_with_parameters(self): '' '')) + def test_sci_update_firmware_attribute(self): + + self._prepare_sci_response(EXAMPLE_UPDATE_FIRMWARE_INVALID_ATTRIBUTE_RESPONSE) + self.dc.get_sci_api().send_sci( + operation="update_firmware", + attribute="filename=\"abcd.bin\"", + target=DeviceTarget('00000000-00000000-00409dff-ffaabbcc'), + payload=EXAMPLE_UPDATE_FIRMWARE_INVALID_ATTRIBUTE_REQUEST_PAYLOAD) + + request = httpretty.last_request().body.decode('utf8') + request = ''.join([line.strip() for line in request.splitlines()]) + self.assertEqual(request, + six.u('' + '' + '' + '' + '' + 'aHNxcAbAADUct1cAAACAHEBAAAEABEAwAIBAAQAAACOFFzU' + '' + '')) + class TestGetAsync(HttpTestBase): def test_sci_get_async(self): diff --git a/devicecloud/version.py b/devicecloud/version.py index 34f635c..ecae618 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -5,4 +5,4 @@ # Copyright (c) 2015-2018 Digi International Inc. -__version__ = "0.5.3" +__version__ = "0.5.4" From 10c771a8a30141ce2dad3bfe6f0083287aebd79b Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Thu, 23 May 2019 13:16:08 -0500 Subject: [PATCH 114/140] Remove support for py3.3 --- setup.py | 1 - toxtest.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/setup.py b/setup.py index f9ea6ba..ecb6320 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,6 @@ def get_long_description(): "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", diff --git a/toxtest.sh b/toxtest.sh index 27c914c..a74779d 100755 --- a/toxtest.sh +++ b/toxtest.sh @@ -7,7 +7,6 @@ # pyversions=(2.7.7 - 3.3.5 3.4.3 3.5.5 3.6.6 From 5afbbc70fc3cece79bae848047e13226ba56e368 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Wed, 12 Jun 2019 08:46:08 -0500 Subject: [PATCH 115/140] Fix integration tests for TCP --- devicecloud/test/integration/inttest_monitor_tcp.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/devicecloud/test/integration/inttest_monitor_tcp.py b/devicecloud/test/integration/inttest_monitor_tcp.py index babc25c..466bd7d 100644 --- a/devicecloud/test/integration/inttest_monitor_tcp.py +++ b/devicecloud/test/integration/inttest_monitor_tcp.py @@ -40,15 +40,19 @@ def receive_notification(notification): for rec in rx: msg = rec['Document']['Msg'] fd = msg.get('FileData', None) - if fd: + if fd and 'id' in fd: if (fd['id']['fdName'] == 'test_file.txt' and fd['id']['fdPath'] == '/db/7603_Digi/inttest/monitor_tcp/'): fd_push_seen = True + # else: + # print('id not in test_event_reception/fd: {}'.format(rx)) dp = msg.get('DataPoint') - if dp: + if dp and 'streamId' in dp: + print('test_event_reception/dp: {}'.format(dp)) if dp['streamId'] == 'inttest/monitor_tcp': dp_push_seen = True - + # else: + # print('streamId not in test_event_reception/dp: {}'.format(rx)) self.assertTrue(fd_push_seen) self.assertTrue(dp_push_seen) except: From 3ffa60badc13ea426a35dff326789161c084f48f Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Wed, 12 Jun 2019 08:46:24 -0500 Subject: [PATCH 116/140] Update Tox versions to latest patch versions --- toxtest.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/toxtest.sh b/toxtest.sh index a74779d..8605c71 100755 --- a/toxtest.sh +++ b/toxtest.sh @@ -6,11 +6,11 @@ # testing using each of those interpreters. # -pyversions=(2.7.7 - 3.4.3 - 3.5.5 - 3.6.6 - 3.7.0 +pyversions=(2.7.16 + 3.4.10 + 3.5.7 + 3.6.8 + 3.7.3 pypy2.7-6.0.0 pypy3.5-6.0.0) From 91483e9b3ee0fb75639c6f612a13039e37492617 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Wed, 12 Jun 2019 10:30:51 -0500 Subject: [PATCH 117/140] Add make scripts --- make.bat | 13 +++++++++++++ make.sh | 13 +++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 make.bat create mode 100644 make.sh diff --git a/make.bat b/make.bat new file mode 100644 index 0000000..5552dd0 --- /dev/null +++ b/make.bat @@ -0,0 +1,13 @@ +@echo off + +rem Install/Upgrade Build Tools and Dependencies +pip install --upgrade setuptools wheel +pip install --upgrade twine + +rem Build distribution +python setup.py sdist bdist_wheel + +rem Upload to public Pypi Server +python -m twine upload dist/* + +rem "Package uploaded to pypi" diff --git a/make.sh b/make.sh new file mode 100644 index 0000000..25e9c86 --- /dev/null +++ b/make.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +# Install/Upgrade Build Tools and Dependencies +pip install --upgrade setuptools wheel +pip install --upgrade twine + +# Build distribution +python setup.py sdist bdist_wheel + +# Upload to public Pypi Server +python -m twine upload dist/* + +echo "Package uploaded to Pypi" From 7365336995115daa561451e2a649c5a17ac3d299 Mon Sep 17 00:00:00 2001 From: Corey Kline Date: Fri, 21 Jun 2019 16:10:22 -0500 Subject: [PATCH 118/140] Allow installation on Windows. Installation on Windows was failing because the which command doesn't exist. pyandoc now finds the pandoc executable internally anyways so this was redundant. Signed-off-by: Corey Kline --- setup.py | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/setup.py b/setup.py index ecb6320..8351e24 100644 --- a/setup.py +++ b/setup.py @@ -28,28 +28,15 @@ def get_long_description(): try: import subprocess import pandoc - - process = subprocess.Popen( - ['which pandoc'], - shell=True, - stdout=subprocess.PIPE, - universal_newlines=True) - - pandoc_path = process.communicate()[0] - pandoc_path = pandoc_path.strip('\n') - - pandoc.core.PANDOC_PATH = pandoc_path - + print(pandoc.core.PANDOC_PATH) doc = pandoc.Document() - doc.markdown = long_description - long_description = doc.rst - open("README.rst", "w").write(doc.rst) + doc.markdown = long_description.encode('utf-8') + long_description = doc.rst.decode() + open("README.rst", "wb").write(doc.rst) + except: - if os.path.exists("README.rst"): - long_description = open("README.rst").read() - else: - print("Could not find pandoc or convert properly") - print(" make sure you have pandoc (system) and pyandoc (python module) installed") + print("Could not find pandoc or convert properly") + print(" make sure you have pandoc (system) and pyandoc (python module) installed") return long_description From 22cf1a2110a17b53546290a3d31f369c5bf5e28e Mon Sep 17 00:00:00 2001 From: Corey Kline Date: Tue, 23 Jul 2019 11:48:59 -0500 Subject: [PATCH 119/140] Bump version Signed-off-by: Corey Kline --- devicecloud/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devicecloud/version.py b/devicecloud/version.py index ecae618..db2e836 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -5,4 +5,4 @@ # Copyright (c) 2015-2018 Digi International Inc. -__version__ = "0.5.4" +__version__ = "0.5.5" From 5cab8885f32201dd300beaadee1c0d0b1a879f09 Mon Sep 17 00:00:00 2001 From: Corey Kline Date: Wed, 24 Jul 2019 14:33:26 -0500 Subject: [PATCH 120/140] Add long description content type Signed-off-by: Corey Kline --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 8351e24..c76fd01 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,7 @@ import os from setuptools import setup, find_packages +content_type = 'txt/markdown' def get_version(): # In order to get the version safely, we read the version.py file @@ -28,23 +29,22 @@ def get_long_description(): try: import subprocess import pandoc - print(pandoc.core.PANDOC_PATH) doc = pandoc.Document() doc.markdown = long_description.encode('utf-8') long_description = doc.rst.decode() open("README.rst", "wb").write(doc.rst) - + content_type = 'txt/x-rst' except: print("Could not find pandoc or convert properly") print(" make sure you have pandoc (system) and pyandoc (python module) installed") return long_description - setup( name="devicecloud", version=get_version(), description="Python API to the Digi Device Cloud", + long_description_content_type=content_type, long_description=get_long_description(), url="https://github.com/digidotcom/python-devicecloud", author="Digi International Inc.", From 625d6b1e366c57581a2f99584fc002b8a160260f Mon Sep 17 00:00:00 2001 From: Corey Kline Date: Wed, 24 Jul 2019 14:44:38 -0500 Subject: [PATCH 121/140] Change long description content type Signed-off-by: Corey Kline --- setup.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/setup.py b/setup.py index c76fd01..b27c32e 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,6 @@ import os from setuptools import setup, find_packages -content_type = 'txt/markdown' - def get_version(): # In order to get the version safely, we read the version.py file # as text. This is necessary as devicecloud/__init__.py uses @@ -31,9 +29,7 @@ def get_long_description(): import pandoc doc = pandoc.Document() doc.markdown = long_description.encode('utf-8') - long_description = doc.rst.decode() open("README.rst", "wb").write(doc.rst) - content_type = 'txt/x-rst' except: print("Could not find pandoc or convert properly") print(" make sure you have pandoc (system) and pyandoc (python module) installed") @@ -44,7 +40,7 @@ def get_long_description(): name="devicecloud", version=get_version(), description="Python API to the Digi Device Cloud", - long_description_content_type=content_type, + long_description_content_type='text/markdown', long_description=get_long_description(), url="https://github.com/digidotcom/python-devicecloud", author="Digi International Inc.", From 13d3bf89ff4cb1fa4f0e153c2e67e31dc73cf1c0 Mon Sep 17 00:00:00 2001 From: Corey Kline Date: Wed, 24 Jul 2019 14:46:37 -0500 Subject: [PATCH 122/140] Bump version Signed-off-by: Corey Kline --- devicecloud/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devicecloud/version.py b/devicecloud/version.py index db2e836..cb30deb 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -5,4 +5,4 @@ # Copyright (c) 2015-2018 Digi International Inc. -__version__ = "0.5.5" +__version__ = "0.5.6" From afc3d35a41375c21597db5b57ebbd2bc08458b07 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2019 13:45:31 +0000 Subject: [PATCH 123/140] build(deps): bump requests from 2.18.4 to 2.20.0 Bumps [requests](https://github.com/requests/requests) from 2.18.4 to 2.20.0. - [Release notes](https://github.com/requests/requests/releases) - [Changelog](https://github.com/psf/requests/blob/master/HISTORY.md) - [Commits](https://github.com/requests/requests/compare/v2.18.4...v2.20.0) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 147559a..dd0fb79 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,6 @@ certifi==2018.4.16 chardet==3.0.4 idna==2.6 python-dateutil==2.7.3 -requests==2.18.4 +requests==2.20.0 six==1.11.0 urllib3==1.22 From 38adc8c66f07774d29e7f6bbf9570a7d1bc12846 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Tue, 14 Jan 2020 21:44:41 -0600 Subject: [PATCH 124/140] Move version file path to var --- make.sh | 1 + setup.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/make.sh b/make.sh index 25e9c86..3586fce 100644 --- a/make.sh +++ b/make.sh @@ -11,3 +11,4 @@ python setup.py sdist bdist_wheel python -m twine upload dist/* echo "Package uploaded to Pypi" + diff --git a/setup.py b/setup.py index ecb6320..5ea72d9 100644 --- a/setup.py +++ b/setup.py @@ -8,13 +8,14 @@ import os from setuptools import setup, find_packages +VERSIONFILE = "devicecloud/version.py" def get_version(): # In order to get the version safely, we read the version.py file # as text. This is necessary as devicecloud/__init__.py uses # things that won't yet be present when the package is being # installed. - verstrline = open("devicecloud/version.py", "r").read() + verstrline = open(VERSIONFILE, "r").read() version_string_re = re.compile(r"^__version__ = ['\"]([^'\"]*)['\"]", re.MULTILINE) match = version_string_re.search(verstrline) if match: From 7dc31aaebcfaf5fd8de0369e8ae03604f8fc6886 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Wed, 22 Apr 2020 21:31:46 -0500 Subject: [PATCH 125/140] Merge branches, bump requests to 2.20.1 - release 0.5.7 --- devicecloud/version.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/devicecloud/version.py b/devicecloud/version.py index cb30deb..fc1c845 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -5,4 +5,4 @@ # Copyright (c) 2015-2018 Digi International Inc. -__version__ = "0.5.6" +__version__ = "0.5.7" diff --git a/requirements.txt b/requirements.txt index dd0fb79..ae4ade9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,6 @@ certifi==2018.4.16 chardet==3.0.4 idna==2.6 python-dateutil==2.7.3 -requests==2.20.0 +requests==2.20.1 six==1.11.0 urllib3==1.22 From 7800f4870005e7a56f6918a3c3a5f1298b1c5ee0 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Wed, 22 Apr 2020 22:31:23 -0500 Subject: [PATCH 126/140] remove py33 from tox --- tox.ini | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index b4265f9..3b236b2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,12 @@ [tox] -envlist = py27,py33,py34,py35,py36,py37,pypy,pypy3 +envlist = py27,py34,py35,py36,py37,pypy,pypy3 [testenv] passenv = * -deps= - -rtest-requirements.txt -commands=nosetests -m '^(int|unit)?[Tt]est' +deps = +commands = + pip install -r test-requirements.txt + nosetests -m '^(int|unit)?[Tt]est' [testenv:coverage] deps= From f9a96efe033b254f5965d039748438707287cd2a Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Wed, 22 Apr 2020 22:31:35 -0500 Subject: [PATCH 127/140] update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2838413..e118fe4 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ env3/ .toxenv/ venv/ .venv/ +.venv2/ #------------------------------------------------------------------------------- # Setuptools Stuff From 316069c14b3ea9ab5b6a1ffef21b960dd3de0141 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Wed, 22 Apr 2020 22:31:55 -0500 Subject: [PATCH 128/140] update requirements to include compatible versions --- requirements.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index ae4ade9..0c79c04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -arrow==0.12.1 -backports.functools-lru-cache==1.5 -certifi==2018.4.16 -chardet==3.0.4 -idna==2.6 -python-dateutil==2.7.3 -requests==2.20.1 -six==1.11.0 -urllib3==1.22 +arrow~=0.12.1 +backports.functools-lru-cache~=1.5 +certifi~=2018.4.16 +chardet~=3.0.4 +idna~=2.6 +python-dateutil~=2.7.3 +requests~=2.20.1 +six~=1.14.0 +urllib3~=1.22 From 6fb872b1adf6a475dc8737da8262d4b003f2e3e2 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Wed, 22 Apr 2020 22:44:49 -0500 Subject: [PATCH 129/140] update changelog --- CHANGELOG.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75c54ae..08488df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,37 @@ ## Python Devicecloud Library Changelog +### 0.5.7 / 2020-04-22 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.6...0.5.7) + +Enhancement: + +* update requests dependency to 2.20 +* update other dependecies to "compatible" versions on install + + +### 0.5.6 / 2019-07-24 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.5...0.5.6) + +Enhancement: + +* core: update package long description + +### 0.5.5 / 2019-07-23 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.4...0.5.5) + +Enhancement: + +* core: remove subprocess during installation (make installation Windows compatible) + + +### 0.5.4 / 2019-05-22 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.3...0.5.4) + +Enhancement: + +* tests: added test for sci firmware update attributes + + ### 0.5.3 / 2018-07-24 [Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.1...0.5.3) From 65c703f17446edea0c7febb24c917474e13af74c Mon Sep 17 00:00:00 2001 From: Zhiyu Wang Date: Tue, 12 Jan 2021 14:37:45 -0500 Subject: [PATCH 130/140] streams: add geojson type Because the new location data point is wrapped as GEOJSON object and this type has not been supported by devicecloud, add the new type STREAM_TYPE_GEOJSON to read and parse GEOJSON object. DAL-4493 --- devicecloud/streams.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/devicecloud/streams.py b/devicecloud/streams.py index c87c496..83693c1 100644 --- a/devicecloud/streams.py +++ b/devicecloud/streams.py @@ -26,6 +26,7 @@ STREAM_TYPE_STRING = "STRING" STREAM_TYPE_BINARY = "BINARY" STREAM_TYPE_JSON = "JSON" +STREAM_TYPE_GEOJSON = "GEOJSON" STREAM_TYPE_UNKNOWN = "UNKNOWN" ROLLUP_INTERVAL_HALF = "half" @@ -54,7 +55,8 @@ STREAM_TYPE_STRING: (str, str), STREAM_TYPE_BINARY: (str, str), STREAM_TYPE_UNKNOWN: (str, str), - STREAM_TYPE_JSON: (json.loads, json.dumps) + STREAM_TYPE_JSON: (json.loads, json.dumps), + STREAM_TYPE_GEOJSON: (json.loads, json.dumps) } From 6141cd8b033516ad559e7d03365308a7574b6dcf Mon Sep 17 00:00:00 2001 From: Zhiyu Wang Date: Wed, 17 Feb 2021 12:13:22 -0500 Subject: [PATCH 131/140] release: Add GEOJSON data format support Increase the version number by one because a new release is needed to be used by dal_test. DAL-4611 --- devicecloud/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devicecloud/version.py b/devicecloud/version.py index fc1c845..69aea6f 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -5,4 +5,4 @@ # Copyright (c) 2015-2018 Digi International Inc. -__version__ = "0.5.7" +__version__ = "0.5.8" From 8d2c72a778ee14b6241fa4b2c921121f15ca56e8 Mon Sep 17 00:00:00 2001 From: Anubhav Gupta Date: Fri, 18 Feb 2022 10:16:34 -0500 Subject: [PATCH 132/140] updated requirements.txt to the latest release version packages. Bumped up the version to 0.5.9. --- devicecloud/version.py | 2 +- requirements.txt | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/devicecloud/version.py b/devicecloud/version.py index 69aea6f..3a0d43d 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -5,4 +5,4 @@ # Copyright (c) 2015-2018 Digi International Inc. -__version__ = "0.5.8" +__version__ = "0.5.9" diff --git a/requirements.txt b/requirements.txt index 0c79c04..4e25988 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -arrow~=0.12.1 +arrow~=1.2.2 backports.functools-lru-cache~=1.5 -certifi~=2018.4.16 -chardet~=3.0.4 -idna~=2.6 -python-dateutil~=2.7.3 -requests~=2.20.1 -six~=1.14.0 +certifi~=2021.10.8 +chardet~=4.0.0 +idna~=3.3 +python-dateutil~=2.8.2 +requests~=2.27.1 +six~=1.16.0 urllib3~=1.22 From 402bff32974480a44c89cf35c8cb4bd0b82dd29a Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Fri, 18 Feb 2022 09:45:08 -0600 Subject: [PATCH 133/140] git - set execute flag on make scripts, ensure sh scripts have LF line endings --- .gitattributes | 25 +++++++++++++++++++++++++ make.bat | 0 make.sh | 0 3 files changed, 25 insertions(+) create mode 100644 .gitattributes mode change 100644 => 100755 make.bat mode change 100644 => 100755 make.sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..afda6a4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,25 @@ +# Set the default behavior, in case people don't have core.autocrlf set. +* text=auto + +# Explicitly declare text files you want to always be normalized and converted +# to native line endings on checkout. +*.c text +*.h text + +# Declare files that will always have LF line endings on checkout. +*.sh text eol=lf + +# Denote all files that are truly binary and should not be modified. +*.png binary +*.jpg binary + +# Existing attributes +*.java diff +*.js diff +*.pl diff +*.txt diff +*.ts diff +*.html diff +*.sh diff +*.xml diff +*.py diff diff --git a/make.bat b/make.bat old mode 100644 new mode 100755 diff --git a/make.sh b/make.sh old mode 100644 new mode 100755 From 16d060f5f65cf547b715f46ad107674971ea3744 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Fri, 18 Feb 2022 10:04:32 -0600 Subject: [PATCH 134/140] make - use "python -m pip" for pip install --- make.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/make.sh b/make.sh index 3586fce..5534592 100755 --- a/make.sh +++ b/make.sh @@ -1,8 +1,8 @@ #!/bin/sh # Install/Upgrade Build Tools and Dependencies -pip install --upgrade setuptools wheel -pip install --upgrade twine +python -m pip install --upgrade setuptools wheel +python -m pip install --upgrade twine # Build distribution python setup.py sdist bdist_wheel From 261af6e87d24c01ac86479e70930d19da9efcc69 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Fri, 18 Feb 2022 10:04:59 -0600 Subject: [PATCH 135/140] changelog - update log for 0.5.8 and 0.5.9 --- CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08488df..fa95648 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ ## Python Devicecloud Library Changelog + +### 0.5.9 / 2021-02-18 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.8...0.5.9) + +Enhancement: + +* core: update other dependecies to "compatible" versions on install + + +### 0.5.8 / 2021-02-17 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.7...0.5.8) + +Enhancement: + +* datastreams: add "GEOJSON" format type to DataStreams + + ### 0.5.7 / 2020-04-22 [Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.6...0.5.7) From c7f488aef8b0141a187e1ff27aaca584eb6afbe6 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Fri, 18 Feb 2022 10:32:41 -0600 Subject: [PATCH 136/140] make - push to external and internal pypi servers --- make.bat | 5 ++++- make.sh | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/make.bat b/make.bat index 5552dd0..aad76c5 100755 --- a/make.bat +++ b/make.bat @@ -10,4 +10,7 @@ python setup.py sdist bdist_wheel rem Upload to public Pypi Server python -m twine upload dist/* -rem "Package uploaded to pypi" +rem Upload to Internal Pypi Server +python setup.py sdist upload -i http://pypi.digi.com/simple + +rem "Package uploaded to Pypi Servers" diff --git a/make.sh b/make.sh index 5534592..3f9a174 100755 --- a/make.sh +++ b/make.sh @@ -10,5 +10,8 @@ python setup.py sdist bdist_wheel # Upload to public Pypi Server python -m twine upload dist/* -echo "Package uploaded to Pypi" +# Upload to Internal Pypi Server +python setup.py sdist upload -i http://pypi.digi.com/simple + +echo "Package uploaded to Pypi Servers" From a962ae34f2f919eb094c669e383e5a9e4d29a711 Mon Sep 17 00:00:00 2001 From: Brandon Moser Date: Mon, 21 Feb 2022 10:49:56 -0600 Subject: [PATCH 137/140] readme - update all links to https --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 6ebf0c0..62ae59d 100644 --- a/README.md +++ b/README.md @@ -3,20 +3,19 @@ Python Device Cloud Library [![Build Status](https://travis-ci.org/digidotcom/python-devicecloud.svg?branch=master)](https://travis-ci.org/digidotcom/python-devicecloud) [![Coverage Status](https://img.shields.io/coveralls/digidotcom/python-devicecloud.svg)](https://coveralls.io/r/digidotcom/python-devicecloud) -[![Code Climate](https://img.shields.io/codeclimate/github/digidotcom/python-devicecloud.svg)](https://codeclimate.com/github/digidotcom/python-devicecloud) [![Latest Version](https://img.shields.io/pypi/v/devicecloud.svg)](https://pypi.python.org/pypi/devicecloud/) [![License](https://img.shields.io/badge/license-MPL%202.0-blue.svg)](https://github.com/digidotcom/python-devicecloud/blob/master/LICENSE) -Be sure to check out the [full documentation](http://digidotcom.github.io/python-devicecloud). A [Changelog](https://github.com/digidotcom/python-devicecloud/blob/master/CHANGELOG.md) is also available. +Be sure to check out the [full documentation](https://digidotcom.github.io/python-devicecloud). A [Changelog](https://github.com/digidotcom/python-devicecloud/blob/master/CHANGELOG.md) is also available. Overview -------- -Python-devicecloud is a library providing simple, intuitive access to [Digi Device Cloud(sm)](http://www.digi.com/products/cloud/digi-device-cloud) for clients written in Python. +Python-devicecloud is a library providing simple, intuitive access to [Digi Device Cloud(sm)](https://www.digi.com/products/cloud/digi-device-cloud) for clients written in Python. -The library wraps Device Cloud's REST API and hides the details of forming HTTP requests in order to gain access to device information, file data, streams, and other features of Device Cloud. The API can be found [here](http://ftp1.digi.com/support/documentation/90002008_redirect.htm). +The library wraps Device Cloud's REST API and hides the details of forming HTTP requests in order to gain access to device information, file data, streams, and other features of Device Cloud. The API can be found [here](https://ftp1.digi.com/support/documentation/90002008_redirect.htm). -The primary target audience for this library is individuals interfacing with Device Cloud from the server side or developers writing tools to aid device development. For efficient connectivity from devices, we suggest that you first look at using the [Device Cloud Connector](http://www.digi.com/support/productdetail?pid=5575). That being said, this library could also be used on devices if deemed suitable. +The primary target audience for this library is individuals interfacing with Device Cloud from the server side or developers writing tools to aid device development. For efficient connectivity from devices, we suggest that you first look at using the [Device Cloud Connector](https://www.digi.com/support/productdetail?pid=5575). That being said, this library could also be used on devices if deemed suitable. Example ------- @@ -126,7 +125,7 @@ This software is open-source software. Copyright (c) 2015-2018 Digi International Inc. -This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, you can obtain one at http://mozilla.org/MPL/2.0/. +This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, you can obtain one at https://mozilla.org/MPL/2.0/. Digi, Digi International, the Digi logo, the Digi website, Digi Device Cloud, Digi Remote Manager, and Digi Cloud Connector are trademarks or registered trademarks of Digi International Inc. in the United States and other countries worldwide. All other trademarks are the property of their respective owners. From c5e1db30d1597c79b6c45a8a96b4d13f4ba880b3 Mon Sep 17 00:00:00 2001 From: Anubhav Gupta Date: Mon, 4 Mar 2024 14:27:13 -0500 Subject: [PATCH 138/140] loosen all package versions in requirements.txt Updated version to 0.5.10 --- devicecloud/version.py | 2 +- requirements.txt | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/devicecloud/version.py b/devicecloud/version.py index 3a0d43d..aee2b9e 100644 --- a/devicecloud/version.py +++ b/devicecloud/version.py @@ -5,4 +5,4 @@ # Copyright (c) 2015-2018 Digi International Inc. -__version__ = "0.5.9" +__version__ = "0.5.10" diff --git a/requirements.txt b/requirements.txt index 4e25988..02bf492 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -arrow~=1.2.2 -backports.functools-lru-cache~=1.5 -certifi~=2021.10.8 -chardet~=4.0.0 -idna~=3.3 -python-dateutil~=2.8.2 -requests~=2.27.1 -six~=1.16.0 -urllib3~=1.22 +arrow>=1.2.2 +backports.functools-lru-cache>=1.5 +certifi>=2021.10.8 +chardet>=4.0.0 +idna>=3.3 +python-dateutil>=2.8.2 +requests>=2.27.1 +six>=1.16.0 +urllib3>=1.22 From 668af9f8bc6592a889e144740bb521c3def0bf2f Mon Sep 17 00:00:00 2001 From: Ruben Moral Date: Thu, 28 Aug 2025 12:57:00 +0200 Subject: [PATCH 139/140] Add support for authentication using API keys --- devicecloud/__init__.py | 35 +++++++++++++++++++++++++++++++---- docs/index.rst | 3 +++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/devicecloud/__init__.py b/devicecloud/__init__.py index 88060ed..92f5dd1 100644 --- a/devicecloud/__init__.py +++ b/devicecloud/__init__.py @@ -2,14 +2,14 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # -# Copyright (c) 2015-2018 Digi International Inc. +# Copyright (c) 2015-2025 Digi International Inc. import logging import time import json from devicecloud.util import validate_type -from requests.auth import HTTPBasicAuth +from requests.auth import HTTPBasicAuth, AuthBase import requests from devicecloud.version import __version__ import six @@ -344,21 +344,35 @@ class DeviceCloud(object): if dc.has_valid_credentials(): print list(dc.devicecore.get_devices()) + It's also possible to authenticate using API keys:: + + dc = DeviceCloud(api_key_id='my-id', api_key_secret='my-secret') + From there, access to all of Device Clouds features are possible. In some cases, methods for quickly performing selected actions may be provided directly via the ``DeviceCloud`` object while advanced usage requires using functionality exposed through other interfaces. """ - def __init__(self, username, password, base_url=None, + def __init__(self, username=None, password=None, + api_key_id=None, api_key_secret=None, + base_url=None, throttle_retries=DEFAULT_THROTTLE_RETRIES, throttle_delay_init=DEFAULT_THROTTLE_DELAY_INIT, throttle_delay_max=DEFAULT_THROTTLE_DELAY_MAX, throttle_delay_backoff_coefficient=DEFAULT_THROTTLE_DELAY_BACKOFF_COEFFICIENT): if base_url is None: base_url = "https://devicecloud.digi.com" + + if username and password: + auth = HTTPBasicAuth(username, password) + elif api_key_id and api_key_secret: + auth = ApiKeyAuth(api_key_id, api_key_secret) + else: + raise ValueError("Must provide either username/password or api_key_id/api_key_secret") + self._conn = DeviceCloudConnection( - auth=HTTPBasicAuth(username, password), + auth=auth, base_url=base_url, throttle_retries=throttle_retries, throttle_delay_init=throttle_delay_init, @@ -543,3 +557,16 @@ def get_web_service_stub(self): from devicecloud.ws import WebServiceStub return WebServiceStub(self._conn, "/ws") + + +class ApiKeyAuth(AuthBase): + """Custom Auth class for API key authentication.""" + + def __init__(self, api_key_id: str, api_key_secret: str): + self.api_key_id = api_key_id + self.api_key_secret = api_key_secret + + def __call__(self, r): + r.headers["X-API-KEY-ID"] = self.api_key_id + r.headers["X-API-KEY-SECRET"] = self.api_key_secret + return r diff --git a/docs/index.rst b/docs/index.rst index e176e8a..5f9a6d9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -47,6 +47,9 @@ quick example of what the API looks like:: dc = DeviceCloud('user', 'pass') + # You can also use API keys for the authentication + # dc = DeviceCloud(api_key_id='my-id', api_key_secret='my-secret') + # show the MAC address of all devices that are currently connected # # This is done using Device Cloud DeviceCore functionality From 348e7bc08b4f003d1867551f5f12694001b795cb Mon Sep 17 00:00:00 2001 From: Ruben Moral Date: Thu, 28 Aug 2025 17:18:57 +0200 Subject: [PATCH 140/140] Update changelog for version 0.5.10 and clean up README --- CHANGELOG.md | 9 +++++++++ README.md | 12 +++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa95648..fb1664c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ ## Python Devicecloud Library Changelog +### 0.5.10 / 2025-08-28 +[Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.9...0.5.10) + +Enhancement: + +* core: loosen all package versions in requirements.txt +* core: add support for authentication using API keys + + ### 0.5.9 / 2021-02-18 [Full Changelog](https://github.com/digidotcom/python-devicecloud/compare/0.5.8...0.5.9) diff --git a/README.md b/README.md index 62ae59d..841a778 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@ Python Device Cloud Library =========================== -[![Build Status](https://travis-ci.org/digidotcom/python-devicecloud.svg?branch=master)](https://travis-ci.org/digidotcom/python-devicecloud) -[![Coverage Status](https://img.shields.io/coveralls/digidotcom/python-devicecloud.svg)](https://coveralls.io/r/digidotcom/python-devicecloud) [![Latest Version](https://img.shields.io/pypi/v/devicecloud.svg)](https://pypi.python.org/pypi/devicecloud/) [![License](https://img.shields.io/badge/license-MPL%202.0-blue.svg)](https://github.com/digidotcom/python-devicecloud/blob/master/LICENSE) @@ -11,11 +9,11 @@ Be sure to check out the [full documentation](https://digidotcom.github.io/pytho Overview -------- -Python-devicecloud is a library providing simple, intuitive access to [Digi Device Cloud(sm)](https://www.digi.com/products/cloud/digi-device-cloud) for clients written in Python. +Python-devicecloud is a library providing simple, intuitive access to [Digi Remote Manager](https://www.digi.com/products/iot-software-services/digi-remote-manager) for clients written in Python. -The library wraps Device Cloud's REST API and hides the details of forming HTTP requests in order to gain access to device information, file data, streams, and other features of Device Cloud. The API can be found [here](https://ftp1.digi.com/support/documentation/90002008_redirect.htm). +The library wraps Digi Remote Manager's REST API and hides the details of forming HTTP requests in order to gain access to device information, file data, streams, and other features of Digi Remote Manager. The API can be found [here](https://doc-remotemanager.digi.com/). -The primary target audience for this library is individuals interfacing with Device Cloud from the server side or developers writing tools to aid device development. For efficient connectivity from devices, we suggest that you first look at using the [Device Cloud Connector](https://www.digi.com/support/productdetail?pid=5575). That being said, this library could also be used on devices if deemed suitable. +The primary target audience for this library is individuals interfacing with Digi Remote Manager from the server side or developers writing tools to aid device development. Example ------- @@ -44,7 +42,7 @@ for stream in dc.streams.get_streams(): print "%s -> %s" % (stream.get_stream_id(), stream.get_current_value()) ``` -For more examples and detailed documentation, be sure to checkout out the [Full API Documentation](https://digidotcom.github.io/python-devicecloud). +For more examples and detailed documentation, be sure to check out the [Full API Documentation](https://digidotcom.github.io/python-devicecloud). Installation ------------ @@ -123,7 +121,7 @@ License This software is open-source software. -Copyright (c) 2015-2018 Digi International Inc. +Copyright (c) 2015-2025 Digi International Inc. This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, you can obtain one at https://mozilla.org/MPL/2.0/.