From 8cc31c69ceb417dfed3cc0b1cc1df5ceee413fd0 Mon Sep 17 00:00:00 2001 From: Alex Ianchici Date: Tue, 16 Dec 2014 11:48:58 -0800 Subject: [PATCH 001/101] Set the certificate information regardless of whether or not the protocol is set to https in the case of a redirect from http -> https --- src/etcd/client.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index 271d85ed..474b332c 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -102,18 +102,18 @@ def uri(protocol, host, port): # (<1.0) won't be able to connect. kw['ssl_version'] = ssl.PROTOCOL_TLSv1 - if cert: - if isinstance(cert, tuple): - # Key and cert are separate - kw['cert_file'] = cert[0] - kw['key_file'] = cert[1] - else: - # combined certificate - kw['cert_file'] = cert + if cert: + if isinstance(cert, tuple): + # Key and cert are separate + kw['cert_file'] = cert[0] + kw['key_file'] = cert[1] + else: + # combined certificate + kw['cert_file'] = cert - if ca_cert: - kw['ca_certs'] = ca_cert - kw['cert_reqs'] = ssl.CERT_REQUIRED + if ca_cert: + kw['ca_certs'] = ca_cert + kw['cert_reqs'] = ssl.CERT_REQUIRED self.http = urllib3.PoolManager(num_pools=10, **kw) From 19c9882f005fff4e052bbeb38fbd18d36d188f9e Mon Sep 17 00:00:00 2001 From: Christoph Heer Date: Tue, 13 Jan 2015 08:07:11 +0100 Subject: [PATCH 002/101] Moved pyOpenSSL into test requirements --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e0758e66..cabda6e8 100644 --- a/setup.py +++ b/setup.py @@ -9,13 +9,13 @@ version = '0.3.2' install_requires = [ - 'urllib3>=1.7', - 'pyOpenSSL>=0.14', + 'urllib3>=1.7' ] test_requires = [ 'mock', 'nose', + 'pyOpenSSL>=0.14' ] setup(name='python-etcd', From e68572e5297abbeacac98366d5a0601f2f1c27cc Mon Sep 17 00:00:00 2001 From: Samuel Marks Date: Mon, 27 Apr 2015 17:11:37 +1000 Subject: [PATCH 003/101] Implemented `pop` --- src/etcd/__init__.py | 2 ++ src/etcd/client.py | 31 +++++++++++++++++++++++++++++ src/etcd/tests/unit/test_request.py | 20 +++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/src/etcd/__init__.py b/src/etcd/__init__.py index 9367ce9c..18ef0364 100644 --- a/src/etcd/__init__.py +++ b/src/etcd/__init__.py @@ -25,6 +25,8 @@ def __init__(self, action=None, node=None, prevNode=None, **kwdargs): node (dict): The dictionary containing all node information. + prevNode (dict): The dictionary containing previous node information. + """ self.action = action for (key, default) in self._node_props.items(): diff --git a/src/etcd/client.py b/src/etcd/client.py index f50e4dbc..4bd746fe 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -392,6 +392,37 @@ def delete(self, key, recursive=None, dir=None, **kwdargs): self.key_endpoint + key, self._MDELETE, params=kwds) return self._result_from_response(response) + def pop(self, key, recursive=None, dir=None, **kwdargs): + """ + Remove specified key from etcd and return the corresponding value. + + Args: + + key (str): Key. + + recursive (bool): if we want to recursively delete a directory, set + it to true + + dir (bool): if we want to delete a directory, set it to true + + prevValue (str): compare key to this value, and swap only if + corresponding (optional). + + prevIndex (int): modify key only if actual modifiedIndex matches the + provided one (optional). + + Returns: + client.EtcdResult + + Raises: + KeyValue: If the key doesn't exists. + + >>> print client.pop('/key').value + 'value' + + """ + return self.delete(key=key, recursive=recursive, dir=dir, **kwdargs)._prev_node + # Higher-level methods on top of the basic primitives def test_and_set(self, key, value, prev_value, ttl=None): """ diff --git a/src/etcd/tests/unit/test_request.py b/src/etcd/tests/unit/test_request.py index df1ae575..340bece2 100644 --- a/src/etcd/tests/unit/test_request.py +++ b/src/etcd/tests/unit/test_request.py @@ -240,6 +240,26 @@ def test_delete(self): res = self.client.delete('/testKey') self.assertEquals(res, etcd.EtcdResult(**d)) + def test_pop(self): + """ Can pop a value """ + d = { + u'action': u'delete', + u'node': { + u'key': u'/testkey', + u'modifiedIndex': 3, + u'createdIndex': 2 + }, + u'prevNode': {u'newKey': False, u'createdIndex': None, + u'modifiedIndex': 190, u'value': u'test', u'expiration': None, + u'key': u'/testkey', u'ttl': None, u'dir': False} + } + + self._mock_api(200, d) + res = self.client.pop(d['node']['key']) + self.assertEquals({attr: getattr(res, attr) for attr in dir(res) + if attr in etcd.EtcdResult._node_props}, d['prevNode']) + self.assertEqual(res.value, d['prevNode']['value']) + def test_read(self): """ Can get a value """ d = { From 2730adacf82481554b9c03f087a917f25b94ae8a Mon Sep 17 00:00:00 2001 From: Bradley Cicenas Date: Fri, 22 May 2015 11:47:34 -0400 Subject: [PATCH 004/101] add increment to index in eternal watch generator --- src/etcd/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index 2864ae58..a8fe89bf 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -639,7 +639,7 @@ def eternal_watch(self, key, index=None, recursive=None): local_index = index while True: response = self.watch(key, index=local_index, timeout=0, recursive=recursive) - local_index = response.etcd_index + local_index = response.etcd_index + 1 yield response def get_lock(self, *args, **kwargs): From d7069c1cda0e2edc7a1ff2a2481158857978c488 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Tue, 9 Jun 2015 12:58:42 +0200 Subject: [PATCH 005/101] Make eternal_watch use modifiedIndex, not etcd_index Contrary to what advised in the etcd documentation as of now, the correct way to handle a request for eternal watch is to increment on the modifiedIndex and not on X-Etcd-Index. --- src/etcd/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index a8fe89bf..bedfe719 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -473,6 +473,7 @@ def read(self, key, **kwdargs): response = self.api_execute( self.key_endpoint + key, self._MGET, params=params, timeout=timeout) + print response return self._result_from_response(response) def delete(self, key, recursive=None, dir=None, **kwdargs): @@ -639,7 +640,7 @@ def eternal_watch(self, key, index=None, recursive=None): local_index = index while True: response = self.watch(key, index=local_index, timeout=0, recursive=recursive) - local_index = response.etcd_index + 1 + local_index = response.modifiedIndex + 1 yield response def get_lock(self, *args, **kwargs): From 9f676c67812d0bb75eefb13a7336a8947073e137 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Wed, 10 Jun 2015 12:00:26 +0200 Subject: [PATCH 006/101] remove spurious print statement --- src/etcd/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index bedfe719..e4376b2f 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -473,7 +473,6 @@ def read(self, key, **kwdargs): response = self.api_execute( self.key_endpoint + key, self._MGET, params=params, timeout=timeout) - print response return self._result_from_response(response) def delete(self, key, recursive=None, dir=None, **kwdargs): From 1a91a01700ff486e3912bb2576ee5439207284cf Mon Sep 17 00:00:00 2001 From: Michal Witkowski Date: Thu, 11 Jun 2015 14:58:07 +0100 Subject: [PATCH 007/101] Fix Issue #110, parametrising pool size --- src/etcd/client.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index a578add7..d8498867 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -55,6 +55,7 @@ def __init__( allow_reconnect=False, use_proxies=False, expected_cluster_id=None, + per_host_pool_size=10 ): """ Initialize the client. @@ -93,6 +94,9 @@ def __init__( reads will raise EtcdClusterIdChanged if they receive a response with a different cluster ID. + per_host_pool_size (int): specifies maximum number of connections to pool + by host. By default this will use up to 10 + connections. """ _log.info("New etcd client created for %s:%s%s", host, port, version_prefix) @@ -121,7 +125,9 @@ def uri(protocol, host, port): # SSL Client certificate support - kw = {} + kw = { + 'maxsize': per_host_pool_size + } if self._read_timeout > 0: kw['timeout'] = self._read_timeout From 6aae3644c00869f9525bc5a5066c4fb03060ef46 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Sat, 20 Jun 2015 15:59:37 +0200 Subject: [PATCH 008/101] Reduce logging noise Most people will run their applications at loglevel=info, I think we're way too noisy at the moment (I had to set my loglevel to warning in order not to show too many logs to my users). --- src/etcd/client.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index 50dc3b4b..905cd6a5 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -95,10 +95,10 @@ def __init__( if they receive a response with a different cluster ID. per_host_pool_size (int): specifies maximum number of connections to pool - by host. By default this will use up to 10 + by host. By default this will use up to 10 connections. """ - _log.info("New etcd client created for %s:%s%s", + _log.debug("New etcd client created for %s:%s%s", host, port, version_prefix) self._protocol = protocol @@ -374,7 +374,7 @@ def write(self, key, value, ttl=None, dir=False, append=False, **kwdargs): 'newValue' """ - _log.info("Writing %s to key %s ttl=%s dir=%s append=%s", + _log.debug("Writing %s to key %s ttl=%s dir=%s append=%s", value, key, ttl, dir, append) key = self._sanitize_key(key) params = {} @@ -419,7 +419,7 @@ def update(self, obj): obj (etcd.EtcdResult): The object that needs updating. """ - _log.info("Updating %s to %s.", obj.key, obj.value) + _log.debug("Updating %s to %s.", obj.key, obj.value) kwdargs = { 'dir': obj.dir, 'ttl': obj.ttl, @@ -463,7 +463,7 @@ def read(self, key, **kwdargs): 'value' """ - _log.info("Issuing read for key %s with args %s", key, kwdargs) + _log.debug("Issuing read for key %s with args %s", key, kwdargs) key = self._sanitize_key(key) params = {} @@ -510,7 +510,7 @@ def delete(self, key, recursive=None, dir=None, **kwdargs): '/key' """ - _log.info("Deleting %s recursive=%s dir=%s extra args=%s", + _log.debug("Deleting %s recursive=%s dir=%s extra args=%s", key, recursive, dir, kwdargs) key = self._sanitize_key(key) @@ -768,7 +768,7 @@ def api_execute(self, path, method, params=None, timeout=None): self._base_uri = self._next_server() some_request_failed = True else: - _log.info("Reconnection disabled, giving up.") + _log.debug("Reconnection disabled, giving up.") raise etcd.EtcdConnectionFailed( "Connection to etcd failed due to %r" % e) except: From 537b5409aedd2727a32591782b8e4ce5ffe52772 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Sat, 20 Jun 2015 19:42:47 +0200 Subject: [PATCH 009/101] Add condition to the cluster id check Some endpoints, most notably the /stats/ ones, do not return a x-etcd-cluster header. While this gets fixed in etcd, we need to not fail hard in those cases. --- src/etcd/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index 905cd6a5..49934227 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -780,7 +780,8 @@ def api_execute(self, path, method, params=None, timeout=None): # preload_content=False above so we can read the headers # before we wait for the content of a long poll. cluster_id = response.getheader("x-etcd-cluster-id") - id_changed = (self.expected_cluster_id and + id_changed = (self.expected_cluster_id + and cluster_id is not None and cluster_id != self.expected_cluster_id) # Update the ID so we only raise the exception once. self.expected_cluster_id = cluster_id From 300fafb046d9763666d580cbf6dbed255c140a85 Mon Sep 17 00:00:00 2001 From: Alex Chan Date: Mon, 22 Jun 2015 15:08:38 +0100 Subject: [PATCH 010/101] Fix unhelpful log message when cluster ID changes When the cluster ID changes, python-etcd writes a log message of the form The UUID of the cluster changed from (expected) to (actual). Since the expected cluster ID was overwritten with the actual value immediately before this message was logged, the log message would always use the same value in two places. This commit corrects the message to use the correct value for (expected). --- src/etcd/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index 49934227..67cbdf8f 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -784,6 +784,7 @@ def api_execute(self, path, method, params=None, timeout=None): and cluster_id is not None and cluster_id != self.expected_cluster_id) # Update the ID so we only raise the exception once. + old_expected_cluster_id = self.expected_cluster_id self.expected_cluster_id = cluster_id if id_changed: # Defensive: clear the pool so that we connect afresh next @@ -791,7 +792,7 @@ def api_execute(self, path, method, params=None, timeout=None): self.http.clear() raise etcd.EtcdClusterIdChanged( 'The UUID of the cluster changed from {} to ' - '{}.'.format(self.expected_cluster_id, cluster_id)) + '{}.'.format(old_expected_cluster_id, cluster_id)) if some_request_failed: if not self._use_proxies: From 8a35674cc8ef50b63d3d27fbe820d3617dba3796 Mon Sep 17 00:00:00 2001 From: Jose Plana Date: Tue, 30 Jun 2015 00:36:53 +0200 Subject: [PATCH 011/101] Fix tests. Now urllib3 has been wrapped within a etcd.EtcdConenctionFailed --- src/etcd/tests/integration/test_ssl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/etcd/tests/integration/test_ssl.py b/src/etcd/tests/integration/test_ssl.py index 16185c27..6ba6a3ad 100644 --- a/src/etcd/tests/integration/test_ssl.py +++ b/src/etcd/tests/integration/test_ssl.py @@ -81,8 +81,8 @@ def test_get_set_unauthenticated_with_ca(self): client = etcd.Client( protocol='https', port=6001, ca_cert=self.ca2_cert_path) - self.assertRaises(urllib3.exceptions.SSLError, client.set, '/test-set', 'test-key') - self.assertRaises(urllib3.exceptions.SSLError, client.get, '/test-set') + self.assertRaises(etcd.EtcdConnectionFailed, client.set, '/test-set', 'test-key') + self.assertRaises(etcd.EtcdConnectionFailed, client.get, '/test-set') def test_get_set_authenticated(self): """ INTEGRATION: set/get a new value authenticated """ From c545ee4713c76a759113cafb9c78dafa7c6c1a12 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Wed, 1 Jul 2015 11:40:09 +0200 Subject: [PATCH 012/101] Add a client-side locking implementation Since server-side locking has been removed from etcd itself, we implement locks client-side using Zookeeper's recipes as a base. What we still miss is: - Integration tests - Documentation (although the API for the lock is pretty much the same we had in the past) --- src/etcd/__init__.py | 6 + src/etcd/lock.py | 175 ++++++++++++++++++++++++++++ src/etcd/tests/unit/__init__.py | 36 +++++- src/etcd/tests/unit/test_lock.py | 172 +++++++++++++++++++++++++++ src/etcd/tests/unit/test_request.py | 32 +---- 5 files changed, 388 insertions(+), 33 deletions(-) create mode 100644 src/etcd/lock.py create mode 100644 src/etcd/tests/unit/test_lock.py diff --git a/src/etcd/__init__.py b/src/etcd/__init__.py index 370be429..2a5992b4 100644 --- a/src/etcd/__init__.py +++ b/src/etcd/__init__.py @@ -1,5 +1,6 @@ import logging from .client import Client +from .lock import Lock _log = logging.getLogger(__name__) @@ -222,6 +223,11 @@ class EtcdDirNotEmpty(EtcdValueError): """ pass +class EtcdLockExpired(EtcdException): + """ + Our lock apparently expired while we were trying to acquire it. + """ + class EtcdError(object): # See https://github.com/coreos/etcd/blob/master/Documentation/errorcode.md diff --git a/src/etcd/lock.py b/src/etcd/lock.py new file mode 100644 index 00000000..9825c6bb --- /dev/null +++ b/src/etcd/lock.py @@ -0,0 +1,175 @@ +import logging +import etcd +import uuid + +_log = logging.getLogger(__name__) + +class Lock(object): + """ + Locking recipe for etcd, inspired by the kazoo recipe for zookeeper + """ + + def __init__(self, client, lock_name): + self.client = client + self.name = lock_name + # props to Netflix Curator for this trick. It is possible for our + # create request to succeed on the server, but for a failure to + # prevent us from getting back the full path name. We prefix our + # lock name with a uuid and can check for its presence on retry. + self._uuid = uuid.uuid4().hex + self.path = "/_locks/{}".format(lock_name) + self.is_taken = False + self._sequence = None + _log.debug("Initiating lock for %s with uuid %s", self.path, self._uuid) + + @property + def uuid(self): + """ + The unique id of the lock + """ + return self._uuid + + @uuid.setter + def set_uuid(self, value): + old_uuid = self._uuid + self._uuid = value + if not self._find_lock(): + _log.warn("The hand-set uuid was not found, refusing") + self._uuid = old_uuid + raise ValueError("Inexistent UUID") + + @property + def is_acquired(self): + """ + tells us if the lock is acquired + """ + if not self.is_taken: + _log.debug("Lock not taken") + return False + try: + self.client.read(self.lock_key) + return True + except etcd.EtcdKeyNotFound: + _log.warn("Lock was supposedly taken, but we cannot find it") + self.is_taken = False + return False + + def acquire(self, blocking=True, lock_ttl=3600, timeout=None): + """ + Acquire the lock. + + :param blocking Block until the lock is obtained, or timeout is reached + :param lock_ttl The duration of the lock we acquired, set to None for eternal locks + :param timeout The time to wait before giving up on getting a lock + """ + # First of all try to write, if our lock is not present. + if not self._find_lock(): + _log.debug("Lock not found, writing it to %s", self.path) + res = self.client.write(self.path, self.uuid, ttl=lock_ttl, append=True) + self._set_sequence(res.key) + _log.debug("Lock key %s written, sequence is %s", res.key, self._sequence) + elif lock_ttl: + # Renew our lock if already here! + self.client.write(self.lock_key, self.uuid, ttl=lock_ttl) + + # now get the owner of the lock, and the next lowest sequence + return self._acquired(blocking=blocking, timeout=timeout) + + def release(self): + """ + Release the lock + """ + if not self._sequence: + self._find_lock() + try: + _log.debug("Releasing existing lock %s", self.lock_key) + self.client.delete(self.lock_key) + except etcd.EtcdKeyNotFound: + _log.info("Lock %s not found, nothing to release", self.lock_key) + pass + finally: + self.is_taken = False + + def __enter__(self): + """ + You can use the lock as a contextmanager + """ + self.acquire(blocking=True, lock_ttl=0) + + def __exit__(self, type, value, traceback): + self.release() + + def _acquired(self, blocking=True, timeout=None): + locker, nearest = self._get_locker() + self.is_taken = False + if self.lock_key == locker: + _log.debug("Lock acquired!") + # We own the lock, yay! + self.is_taken = True + return True + else: + self.is_taken = False + if not blocking: + return False + # Let's look for the lock + watch_key = nearest + _log.debug("Lock not acquired, now watching %s", watch_key) + t = max(0, timeout) + while True: + try: + r = self.client.watch(watch_key, timeout=t) + _log.debug("Detected variation for %s: %s", r.key, r.action) + return self._acquired(blocking=True, timeout=timeout) + except etcd.EtcdKeyNotFound: + _log.debug("Key %s not present anymore, moving on", watch_key) + return self._acquired(blocking=True, timeout=timeout) + except etcd.EtcdException: + # TODO: log something... + pass + + @property + def lock_key(self): + if not self._sequence: + raise ValueError("No sequence present.") + return self.path + '/' + str(self._sequence) + + def _set_sequence(self, key): + self._sequence = int(key.replace(self.path, '').lstrip('/')) + + def _find_lock(self): + if self._sequence: + try: + res = self.client.read(self.lock_key) + self._uuid = res.value + return True + except etcd.EtcdKeyNotFound: + return False + elif self._uuid: + try: + for r in self.client.read(self.path, recursive=True).leaves: + if r.value == self._uuid: + self._set_sequence(r.key) + return True + except etcd.EtcdKeyNotFound: + pass + return False + + def _get_locker(self): + results = [res for res in + self.client.read(self.path, recursive=True).leaves] + if not self._sequence: + self._find_lock() + l = sorted([r.key for r in results]) + _log.debug("Lock keys found: %s", l) + try: + i = l.index(self.lock_key) + if i == 0: + _log.debug("No key before our one, we are the locker") + return (l[0], None) + else: + _log.debug("Locker: %s, key to watch: %s", l[0], l[i-1]) + return (l[0], l[i-1]) + except ValueError: + # Something very wrong is going on, most probably + # our lock has expired + raise etcd.EtcdLockExpired(u"Lock not found") diff --git a/src/etcd/tests/unit/__init__.py b/src/etcd/tests/unit/__init__.py index 8cd6aa28..1dd44ab9 100644 --- a/src/etcd/tests/unit/__init__.py +++ b/src/etcd/tests/unit/__init__.py @@ -1,2 +1,34 @@ -from . import test_client -from . import test_request +import etcd +import unittest +import urllib3 +import json +try: + import mock +except ImportError: + from unittest import mock + + +class TestClientApiBase(unittest.TestCase): + + def setUp(self): + self.client = etcd.Client() + + def _prepare_response(self, s, d, cluster_id=None): + if isinstance(d, dict): + data = json.dumps(d).encode('utf-8') + else: + data = d.encode('utf-8') + + r = mock.create_autospec(urllib3.response.HTTPResponse)() + r.status = s + r.data = data + r.getheader.return_value = cluster_id or "abcd1234" + return r + + def _mock_api(self, status, d, cluster_id=None): + resp = self._prepare_response(status, d, cluster_id=cluster_id) + self.client.api_execute = mock.create_autospec( + self.client.api_execute, return_value=resp) + + def _mock_exception(self, exc, msg): + self.client.api_execute = mock.Mock(side_effect=exc(msg)) diff --git a/src/etcd/tests/unit/test_lock.py b/src/etcd/tests/unit/test_lock.py new file mode 100644 index 00000000..9d671144 --- /dev/null +++ b/src/etcd/tests/unit/test_lock.py @@ -0,0 +1,172 @@ +import etcd +import mock +from etcd.tests.unit import TestClientApiBase + + +class TestClientLock(TestClientApiBase): + + def recursive_read(self): + nodes = [ + {"key": "/_locks/test_lock/1", "value": "2qwwwq", + "modifiedIndex":33,"createdIndex":33}, + {"key": "/_locks/test_lock/34", "value": self.locker.uuid, + "modifiedIndex":34,"createdIndex":34}, + ] + d = { + "action": "get", + "node": {"dir": True, + "nodes": [{"key":"/_locks/test_lock", "dir": True, + "nodes": nodes}]} + } + self._mock_api(200, d) + + def setUp(self): + super(TestClientLock, self).setUp() + self.locker = etcd.Lock(self.client, 'test_lock') + + def test_initialization(self): + """ + Verify the lock gets initialized correctly + """ + self.assertEquals(self.locker.name, u'test_lock') + self.assertEquals(self.locker.path, u'/_locks/test_lock') + self.assertEquals(self.locker.is_taken, False) + + def test_acquire(self): + """ + Acquiring a precedingly inexistent lock works. + """ + l = etcd.Lock(self.client, 'test_lock') + l._find_lock = mock.create_autospec(l._find_lock, return_value=False) + l._acquired = mock.create_autospec(l._acquired, return_value=True) + # Mock the write + d = { + u'action': u'set', + u'node': { + u'modifiedIndex': 190, + u'key': u'/_locks/test_lock/1', + u'value': l.uuid + } + } + self._mock_api(200, d) + self.assertEquals(l.acquire(), True) + self.assertEquals(l._sequence, 1) + + def test_is_acquired(self): + """ + Test is_acquired + """ + self.locker._sequence = 1 + d = { + u'action': u'get', + u'node': { + u'modifiedIndex': 190, + u'key': u'/_locks/test_lock/1', + u'value': self.locker.uuid + } + } + self._mock_api(200, d) + self.locker.is_taken = True + self.assertEquals(self.locker.is_acquired, True) + + def test_is_not_acquired(self): + """ + Test is_acquired failures + """ + self.locker._sequence = 2 + self.locker.is_taken = False + self.assertEquals(self.locker.is_acquired, False) + self.locker.is_taken = True + self._mock_exception(etcd.EtcdKeyNotFound, self.locker.lock_key) + self.assertEquals(self.locker.is_acquired, False) + self.assertEquals(self.locker.is_taken, False) + + def test_acquired(self): + """ + Test the acquiring primitives + """ + self.locker._sequence = 4 + retval = ('/_locks/test_lock/4', None) + self.locker._get_locker = mock.create_autospec( + self.locker._get_locker, return_value=retval) + self.assertTrue(self.locker._acquired()) + self.assertTrue(self.locker.is_taken) + retval = ('/_locks/test_lock/1', '/_locks/test_lock/4') + self.locker._get_locker = mock.create_autospec( + self.locker._get_locker, return_value=retval) + self.assertFalse(self.locker._acquired(blocking=False)) + self.assertFalse(self.locker.is_taken) + d = { + u'action': u'delete', + u'node': { + u'modifiedIndex': 190, + u'key': u'/_locks/test_lock/1', + u'value': self.locker.uuid + } + } + self._mock_api(200, d) + returns = [('/_locks/test_lock/1', '/_locks/test_lock/4'), ('/_locks/test_lock/4', None)] + + def side_effect(): + return returns.pop() + + self.locker._get_locker = mock.create_autospec( + self.locker._get_locker, side_effect=side_effect) + self.assertTrue(self.locker._acquired()) + + def test_lock_key(self): + """ + Test responses from the lock_key property + """ + with self.assertRaises(ValueError): + self.locker.lock_key + self.locker._sequence = 5 + self.assertEquals(u'/_locks/test_lock/5',self.locker.lock_key) + + def test_set_sequence(self): + self.locker._set_sequence('/_locks/test_lock/10') + self.assertEquals(10, self.locker._sequence) + + def test_find_lock(self): + d = { + u'action': u'get', + u'node': { + u'modifiedIndex': 190, + u'key': u'/_locks/test_lock/1', + u'value': self.locker.uuid + } + } + self._mock_api(200, d) + self.locker._sequence = 1 + self.assertTrue(self.locker._find_lock()) + # Now let's pretend the lock is not there + self._mock_exception(etcd.EtcdKeyNotFound, self.locker.lock_key) + self.assertFalse(self.locker._find_lock()) + self.locker._sequence = None + self.recursive_read() + self.assertTrue(self.locker._find_lock()) + self.assertEquals(self.locker._sequence, 34) + + + def test_get_locker(self): + self.recursive_read() + self.assertEquals((u'/_locks/test_lock/1', u'/_locks/test_lock/1'), + self.locker._get_locker()) + with self.assertRaises(etcd.EtcdLockExpired): + self.locker._sequence = 35 + self.locker._get_locker() + + def test_release(self): + d = { + u'action': u'delete', + u'node': { + u'modifiedIndex': 190, + u'key': u'/_locks/test_lock/1', + u'value': self.locker.uuid + } + } + self._mock_api(200, d) + self.locker._sequence = 1 + self.locker.is_taken = True + self.locker.release() + self.assertFalse(self.locker.is_taken) diff --git a/src/etcd/tests/unit/test_request.py b/src/etcd/tests/unit/test_request.py index beee6ecf..9f540008 100644 --- a/src/etcd/tests/unit/test_request.py +++ b/src/etcd/tests/unit/test_request.py @@ -1,41 +1,11 @@ import etcd -import unittest -import json -import urllib3 +from etcd.tests.unit import TestClientApiBase try: import mock except ImportError: from unittest import mock -from etcd import EtcdException - - -class TestClientApiBase(unittest.TestCase): - - def setUp(self): - self.client = etcd.Client() - - def _prepare_response(self, s, d, cluster_id=None): - if isinstance(d, dict): - data = json.dumps(d).encode('utf-8') - else: - data = d.encode('utf-8') - - r = mock.create_autospec(urllib3.response.HTTPResponse)() - r.status = s - r.data = data - r.getheader.return_value = cluster_id or "abcd1234" - return r - - def _mock_api(self, status, d, cluster_id=None): - resp = self._prepare_response(status, d, cluster_id=cluster_id) - self.client.api_execute = mock.create_autospec( - self.client.api_execute, return_value=resp) - - def _mock_exception(self, exc, msg): - self.client.api_execute = mock.Mock(side_effect=exc(msg)) - class TestClientApiInternals(TestClientApiBase): From 0adaa9e0f477bc2914471a95f7aad13c075c258d Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Tue, 28 Jul 2015 11:07:05 +0200 Subject: [PATCH 013/101] Fix locking docs, bump version --- README.rst | 38 +++++++++++--------------------------- setup.py | 2 +- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/README.rst b/README.rst index 0d9b4654..1d661d8d 100644 --- a/README.rst +++ b/README.rst @@ -108,41 +108,25 @@ Locking module # Initialize the lock object: # NOTE: this does not acquire a lock yet client = etcd.Client() - lock = client.get_lock('/customer1', ttl=60) + lock = etcd.Lock(client, 'my_lock_name') # Use the lock object: - lock.acquire(timeout=30) #returns if lock could not be acquired within 30 seconds - lock.is_locked() # True - lock.renew(60) - lock.release() - lock.is_locked() # False + lock.acquire(blocking=True, # will block until the lock is acquired + lock_ttl=None) # lock will live until we release it + lock.is_acquired() # + lock.acquire(lock_ttl=60) # renew a lock + lock.release() # release an existing lock + lock.is_acquired() # False # The lock object may also be used as a context manager: client = etcd.Client() - lock = client.get_lock('/customer1', ttl=60) - with lock as my_lock: + with etcd.Lock('customer1') as my_lock: do_stuff() - lock.is_locked() # True - lock.renew(60) - lock.is_locked() # False + my_lock.is_acquired() # True + my_lock.acquire(lock_ttl = 60) + my_lock.is_acquired() # False -Leader Election module -~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: python - - # Set a leader object with a name; if no name is given, the local hostname - # is used. - # Zero or no ttl means the leader object is persistent. - client = etcd.Client() - client.election.set('/mysql', name='foo.example.com', ttl=120, timeout=30) # returns the etcd index - - # Get the name - print(client.election.get('/mysql')) # 'foo.example.com' - # Delete it! - print(client.election.delete('/mysql', name='foo.example.com')) - Get machines in the cluster ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/setup.py b/setup.py index ca098408..193a49bb 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ NEWS = open(os.path.join(here, 'NEWS.txt')).read() -version = '0.3.3' +version = '0.4.0' install_requires = [ 'urllib3>=1.7' From 14476e283adb3414558fe2cccc61514ee3364017 Mon Sep 17 00:00:00 2001 From: Jose Plana Date: Sat, 1 Aug 2015 11:16:51 +0200 Subject: [PATCH 014/101] Fixed typo --- src/etcd/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index 67cbdf8f..fe766a5a 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -72,7 +72,7 @@ def __init__( read_timeout (int): max seconds to wait for a read. allow_redirect (bool): allow the client to connect to other nodes. -+ + protocol (str): Protocol used to connect to etcd. cert (mixed): If a string, the whole ssl client certificate; From ecd777f9dda65e5a7a0284e766dccc934814f153 Mon Sep 17 00:00:00 2001 From: Jose Plana Date: Sat, 1 Aug 2015 11:18:23 +0200 Subject: [PATCH 015/101] Release version 0.4.1 --- NEWS.txt | 21 +++++++++++++++++++++ docs-source/conf.py | 4 ++-- setup.py | 2 +- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/NEWS.txt b/NEWS.txt index 2c6da2cb..cc3ae339 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -1,6 +1,27 @@ News ==== +0.4.1 +----- +*Release date: 1-Aug-2015* + +* Added client-side leader election +* Added stats endpoints +* Added logging +* Better exception handling +* Check for cluster ID on each request +* Added etcd.Client.members and fixed etcd.Client.leader +* Removed locking and election etcd support +* Allow the use of etcd proxies with reconnections +* Implement pop: Remove key from etc and return the corresponding value. +* Eternal watcher can be now recursive +* Fix etcd.Client machines +* Do not send parameters with `None` value to etcd +* Support ttl=0 in write. +* Moved pyOpenSSL into test requirements. +* Always set certificate information so redirects from http to https work. + + 0.3.3 ----- *Release date: 12-Apr-2015* diff --git a/docs-source/conf.py b/docs-source/conf.py index f18d4a06..8e4218d3 100644 --- a/docs-source/conf.py +++ b/docs-source/conf.py @@ -59,9 +59,9 @@ def __getattr__(cls, name): # built documents. # # The short X.Y version. -version = '0.3' +version = '0.4' # The full version, including alpha/beta/rc tags. -release = '0.3.3' +release = '0.4.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 193a49bb..0766c609 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ NEWS = open(os.path.join(here, 'NEWS.txt')).read() -version = '0.4.0' +version = '0.4.1' install_requires = [ 'urllib3>=1.7' From f851e2665dfa9859f5db203cb31d287a6099d6e7 Mon Sep 17 00:00:00 2001 From: Simon Gomizelj Date: Tue, 18 Aug 2015 20:17:40 -0400 Subject: [PATCH 016/101] Fix Lock documentation in README The client parameter is missing. --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 1d661d8d..163d2578 100644 --- a/README.rst +++ b/README.rst @@ -120,7 +120,7 @@ Locking module # The lock object may also be used as a context manager: client = etcd.Client() - with etcd.Lock('customer1') as my_lock: + with etcd.Lock(client, 'customer1') as my_lock: do_stuff() my_lock.is_acquired() # True my_lock.acquire(lock_ttl = 60) From c236e2e583ce4e8646180aba9b6842d484a571f5 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Sat, 19 Sep 2015 12:02:46 +0200 Subject: [PATCH 017/101] Do not cast the lock sequence to int Because the etcd devs changed the format of sequence keys in etcd 2.2 (MEH), we will better not make assumptions about the nature of the sequence - we will just assume they are still ordered so that sort will work. Closes: #123 --- src/etcd/lock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etcd/lock.py b/src/etcd/lock.py index 9825c6bb..582e6702 100644 --- a/src/etcd/lock.py +++ b/src/etcd/lock.py @@ -134,7 +134,7 @@ def lock_key(self): return self.path + '/' + str(self._sequence) def _set_sequence(self, key): - self._sequence = int(key.replace(self.path, '').lstrip('/')) + self._sequence = key.replace(self.path, '').lstrip('/') def _find_lock(self): if self._sequence: From 04d25b166e9d45b1e0025b4dd5a6b314c15dec3c Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Sun, 20 Sep 2015 12:07:58 +0200 Subject: [PATCH 018/101] Fix unit tests --- src/etcd/tests/unit/test_lock.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/etcd/tests/unit/test_lock.py b/src/etcd/tests/unit/test_lock.py index 9d671144..75ae6761 100644 --- a/src/etcd/tests/unit/test_lock.py +++ b/src/etcd/tests/unit/test_lock.py @@ -50,13 +50,13 @@ def test_acquire(self): } self._mock_api(200, d) self.assertEquals(l.acquire(), True) - self.assertEquals(l._sequence, 1) + self.assertEquals(l._sequence, '1') def test_is_acquired(self): """ Test is_acquired """ - self.locker._sequence = 1 + self.locker._sequence = '1' d = { u'action': u'get', u'node': { @@ -73,7 +73,7 @@ def test_is_not_acquired(self): """ Test is_acquired failures """ - self.locker._sequence = 2 + self.locker._sequence = '2' self.locker.is_taken = False self.assertEquals(self.locker.is_acquired, False) self.locker.is_taken = True @@ -85,7 +85,7 @@ def test_acquired(self): """ Test the acquiring primitives """ - self.locker._sequence = 4 + self.locker._sequence = '4' retval = ('/_locks/test_lock/4', None) self.locker._get_locker = mock.create_autospec( self.locker._get_locker, return_value=retval) @@ -120,12 +120,12 @@ def test_lock_key(self): """ with self.assertRaises(ValueError): self.locker.lock_key - self.locker._sequence = 5 + self.locker._sequence = '5' self.assertEquals(u'/_locks/test_lock/5',self.locker.lock_key) def test_set_sequence(self): self.locker._set_sequence('/_locks/test_lock/10') - self.assertEquals(10, self.locker._sequence) + self.assertEquals('10', self.locker._sequence) def test_find_lock(self): d = { @@ -137,7 +137,7 @@ def test_find_lock(self): } } self._mock_api(200, d) - self.locker._sequence = 1 + self.locker._sequence = '1' self.assertTrue(self.locker._find_lock()) # Now let's pretend the lock is not there self._mock_exception(etcd.EtcdKeyNotFound, self.locker.lock_key) @@ -145,7 +145,7 @@ def test_find_lock(self): self.locker._sequence = None self.recursive_read() self.assertTrue(self.locker._find_lock()) - self.assertEquals(self.locker._sequence, 34) + self.assertEquals(self.locker._sequence, '34') def test_get_locker(self): @@ -153,7 +153,7 @@ def test_get_locker(self): self.assertEquals((u'/_locks/test_lock/1', u'/_locks/test_lock/1'), self.locker._get_locker()) with self.assertRaises(etcd.EtcdLockExpired): - self.locker._sequence = 35 + self.locker._sequence = '35' self.locker._get_locker() def test_release(self): From 87c055e612320251072edd916b3b8e640fa85773 Mon Sep 17 00:00:00 2001 From: Peter Wagner Date: Mon, 21 Sep 2015 07:10:02 -0400 Subject: [PATCH 019/101] Python3 fix when blocking on contented lock The default of "None" triggers: TypeError: unorderable types: NoneType() > int() Introducing failing test case + fix; mocked API response may be wonky but it demonstrates issue. --- src/etcd/lock.py | 2 +- src/etcd/tests/unit/test_lock.py | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/etcd/lock.py b/src/etcd/lock.py index 9825c6bb..9c9d015a 100644 --- a/src/etcd/lock.py +++ b/src/etcd/lock.py @@ -99,7 +99,7 @@ def __enter__(self): def __exit__(self, type, value, traceback): self.release() - def _acquired(self, blocking=True, timeout=None): + def _acquired(self, blocking=True, timeout=0): locker, nearest = self._get_locker() self.is_taken = False if self.lock_key == locker: diff --git a/src/etcd/tests/unit/test_lock.py b/src/etcd/tests/unit/test_lock.py index 9d671144..3855b115 100644 --- a/src/etcd/tests/unit/test_lock.py +++ b/src/etcd/tests/unit/test_lock.py @@ -114,6 +114,27 @@ def side_effect(): self.locker._get_locker, side_effect=side_effect) self.assertTrue(self.locker._acquired()) + def test_acquired_no_timeout(self): + self.locker._sequence = 4 + returns = [('/_locks/test_lock/4', None), ('/_locks/test_lock/1', '/_locks/test_lock/4')] + + def side_effect(): + return returns.pop() + + d = { + u'action': u'get', + u'node': { + u'modifiedIndex': 190, + u'key': u'/_locks/test_lock/4', + u'value': self.locker.uuid + } + } + self._mock_api(200, d) + + self.locker._get_locker = mock.create_autospec( + self.locker._get_locker, side_effect=side_effect) + self.assertTrue(self.locker._acquired()) + def test_lock_key(self): """ Test responses from the lock_key property @@ -147,7 +168,6 @@ def test_find_lock(self): self.assertTrue(self.locker._find_lock()) self.assertEquals(self.locker._sequence, 34) - def test_get_locker(self): self.recursive_read() self.assertEquals((u'/_locks/test_lock/1', u'/_locks/test_lock/1'), From 8cf8b5158fc31f38b759daad134e86aa917a84e9 Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Fri, 5 Jun 2015 11:17:09 -0700 Subject: [PATCH 020/101] Fix #6: reduce scope of exception handler, preload data. --- src/etcd/client.py | 54 ++++++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index fe766a5a..f485ac40 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -688,8 +688,13 @@ def election(self): def _result_from_response(self, response): """ Creates an EtcdResult from json dictionary """ + raw_response = response.data + try: + res = json.loads(raw_response.decode('utf-8')) + except (TypeError, ValueError, UnicodeError) as e: + raise etcd.EtcdException( + 'Server response was not valid JSON: %r' % e) try: - res = json.loads(response.data.decode('utf-8')) r = etcd.EtcdResult(**res) if response.status == 201: r.newKey = True @@ -697,7 +702,7 @@ def _result_from_response(self, response): return r except Exception as e: raise etcd.EtcdException( - 'Unable to decode server response: %s' % e) + 'Unable to decode server response: %r' % e) def _next_server(self): """ Selects the next server in the list, refreshes the server list. """ @@ -752,7 +757,15 @@ def api_execute(self, path, method, params=None, timeout=None): else: raise etcd.EtcdException( 'HTTP method {} not supported'.format(method)) - + + # Check the cluster ID hasn't changed under us. We use + # preload_content=False above so we can read the headers + # before we wait for the content of a watch. + self._check_cluster_id(response) + # Now force the data to be preloaded in order to trigger any + # IO-related errors in this method rather than when we try to + # access it later. + _ = response.data # urllib3 doesn't wrap all httplib exceptions and earlier versions # don't wrap socket errors either. except (urllib3.exceptions.HTTPError, @@ -774,26 +787,7 @@ def api_execute(self, path, method, params=None, timeout=None): except: _log.exception("Unexpected request failure, re-raising.") raise - - else: - # Check the cluster ID hasn't changed under us. We use - # preload_content=False above so we can read the headers - # before we wait for the content of a long poll. - cluster_id = response.getheader("x-etcd-cluster-id") - id_changed = (self.expected_cluster_id - and cluster_id is not None and - cluster_id != self.expected_cluster_id) - # Update the ID so we only raise the exception once. - old_expected_cluster_id = self.expected_cluster_id - self.expected_cluster_id = cluster_id - if id_changed: - # Defensive: clear the pool so that we connect afresh next - # time. - self.http.clear() - raise etcd.EtcdClusterIdChanged( - 'The UUID of the cluster changed from {} to ' - '{}.'.format(old_expected_cluster_id, cluster_id)) - + if some_request_failed: if not self._use_proxies: # The cluster may have changed since last invocation @@ -801,6 +795,20 @@ def api_execute(self, path, method, params=None, timeout=None): self._machines_cache.remove(self._base_uri) return self._handle_server_response(response) + def _check_cluster_id(self, response): + cluster_id = response.getheader("x-etcd-cluster-id") + id_changed = (self.expected_cluster_id and + cluster_id != self.expected_cluster_id) + # Update the ID so we only raise the exception once. + self.expected_cluster_id = cluster_id + if id_changed: + # Defensive: clear the pool so that we connect afresh next + # time. + self.http.clear() + raise etcd.EtcdClusterIdChanged( + 'The UUID of the cluster changed from {} to ' + '{}.'.format(self.expected_cluster_id, cluster_id)) + def _handle_server_response(self, response): """ Handles the server response """ if response.status in [200, 201]: From d07c027a68905228f6ce7210e3980ca24adb895e Mon Sep 17 00:00:00 2001 From: Alex Chan Date: Mon, 22 Jun 2015 17:09:40 +0100 Subject: [PATCH 021/101] Fix unhelpful log message when cluster ID changes When the cluster ID changes, python-etcd writes a log message of the form The UUID of the cluster changed from (expected) to (actual). Since the expected cluster ID was overwritten with the actual value immediately before this message was logged, the log message would always use the same value in two places. This commit corrects the message to use the correct value for (expected). --- src/etcd/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index f485ac40..e8293b3e 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -800,6 +800,7 @@ def _check_cluster_id(self, response): id_changed = (self.expected_cluster_id and cluster_id != self.expected_cluster_id) # Update the ID so we only raise the exception once. + old_expected_cluster_id = self.expected_cluster_id self.expected_cluster_id = cluster_id if id_changed: # Defensive: clear the pool so that we connect afresh next @@ -807,7 +808,7 @@ def _check_cluster_id(self, response): self.http.clear() raise etcd.EtcdClusterIdChanged( 'The UUID of the cluster changed from {} to ' - '{}.'.format(self.expected_cluster_id, cluster_id)) + '{}.'.format(old_expected_cluster_id, cluster_id)) def _handle_server_response(self, response): """ Handles the server response """ From ebe07f2a9eed4214ad6b71090034d5306c06465f Mon Sep 17 00:00:00 2001 From: Spike Curtis Date: Thu, 16 Jul 2015 17:36:05 -0700 Subject: [PATCH 022/101] Fix EtcdResult.get_subtree(leaves_only=False) EtcdResult.get_subtree(leaves_only=False) was returning each leaf node twice, and didn't return the top level node at all. This commit fixes it and provides a UT. --- src/etcd/__init__.py | 13 +-- src/etcd/tests/unit/test_result.py | 144 +++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 6 deletions(-) create mode 100644 src/etcd/tests/unit/test_result.py diff --git a/src/etcd/__init__.py b/src/etcd/__init__.py index 2a5992b4..3fa5747c 100644 --- a/src/etcd/__init__.py +++ b/src/etcd/__init__.py @@ -76,13 +76,14 @@ def get_subtree(self, leaves_only=False): #if the current result is a leaf, return itself yield self return - for n in self._children: - node = EtcdResult(None, n) + else: + # node is not a leaf if not leaves_only: - #Return also dirs, not just value nodes - yield node - for child in node.get_subtree(leaves_only=leaves_only): - yield child + yield self + for n in self._children: + node = EtcdResult(None, n) + for child in node.get_subtree(leaves_only=leaves_only): + yield child return @property diff --git a/src/etcd/tests/unit/test_result.py b/src/etcd/tests/unit/test_result.py new file mode 100644 index 00000000..cb1414b1 --- /dev/null +++ b/src/etcd/tests/unit/test_result.py @@ -0,0 +1,144 @@ +import etcd +import unittest +import json +import urllib3 + +try: + import mock +except ImportError: + from unittest import mock + +class TestEtcdResult(unittest.TestCase): + + def test_get_subtree_1_level(self): + """ + Test get_subtree() for a read with tree 1 level deep. + """ + response = {"node": { + 'key': "/test", + 'value': "hello", + 'expiration': None, + 'ttl': None, + 'modifiedIndex': 5, + 'createdIndex': 1, + 'newKey': False, + 'dir': False, + }} + result = etcd.EtcdResult(**response) + self.assertEqual(result.key, response["node"]["key"]) + self.assertEqual(result.value, response["node"]["value"]) + + # Get subtree returns itself, whether or not leaves_only + subtree = list(result.get_subtree(leaves_only=True)) + self.assertListEqual([result], subtree) + subtree = list(result.get_subtree(leaves_only=False)) + self.assertListEqual([result], subtree) + + def test_get_subtree_2_level(self): + """ + Test get_subtree() for a read with tree 2 levels deep. + """ + leaf0 = { + 'key': "/test/leaf0", + 'value': "hello1", + 'expiration': None, + 'ttl': None, + 'modifiedIndex': 5, + 'createdIndex': 1, + 'newKey': False, + 'dir': False, + } + leaf1 = { + 'key': "/test/leaf1", + 'value': "hello2", + 'expiration': None, + 'ttl': None, + 'modifiedIndex': 6, + 'createdIndex': 2, + 'newKey': False, + 'dir': False, + } + testnode = {"node": { + 'key': "/test/", + 'expiration': None, + 'ttl': None, + 'modifiedIndex': 6, + 'createdIndex': 2, + 'newKey': False, + 'dir': True, + 'nodes': [leaf0, leaf1] + }} + result = etcd.EtcdResult(**testnode) + self.assertEqual(result.key, "/test/") + self.assertTrue(result.dir) + + # Get subtree returns just two leaves for leaves only. + subtree = list(result.get_subtree(leaves_only=True)) + self.assertEqual(subtree[0].key, "/test/leaf0") + self.assertEqual(subtree[1].key, "/test/leaf1") + self.assertEqual(len(subtree), 2) + + # Get subtree returns leaves and directory. + subtree = list(result.get_subtree(leaves_only=False)) + self.assertEqual(subtree[0].key, "/test/") + self.assertEqual(subtree[1].key, "/test/leaf0") + self.assertEqual(subtree[2].key, "/test/leaf1") + self.assertEqual(len(subtree), 3) + + def test_get_subtree_3_level(self): + """ + Test get_subtree() for a read with tree 3 levels deep. + """ + leaf0 = { + 'key': "/test/mid0/leaf0", + 'value': "hello1", + } + leaf1 = { + 'key': "/test/mid0/leaf1", + 'value': "hello2", + } + leaf2 = { + 'key': "/test/mid1/leaf2", + 'value': "hello1", + } + leaf3 = { + 'key': "/test/mid1/leaf3", + 'value': "hello2", + } + mid0 = { + 'key': "/test/mid0/", + 'dir': True, + 'nodes': [leaf0, leaf1] + } + mid1 = { + 'key': "/test/mid1/", + 'dir': True, + 'nodes': [leaf2, leaf3] + } + testnode = {"node": { + 'key': "/test/", + 'dir': True, + 'nodes': [mid0, mid1] + }} + result = etcd.EtcdResult(**testnode) + self.assertEqual(result.key, "/test/") + self.assertTrue(result.dir) + + # Get subtree returns just two leaves for leaves only. + subtree = list(result.get_subtree(leaves_only=True)) + self.assertEqual(subtree[0].key, "/test/mid0/leaf0") + self.assertEqual(subtree[1].key, "/test/mid0/leaf1") + self.assertEqual(subtree[2].key, "/test/mid1/leaf2") + self.assertEqual(subtree[3].key, "/test/mid1/leaf3") + self.assertEqual(len(subtree), 4) + + # Get subtree returns leaves and directory. + subtree = list(result.get_subtree(leaves_only=False)) + self.assertEqual(subtree[0].key, "/test/") + self.assertEqual(subtree[1].key, "/test/mid0/") + self.assertEqual(subtree[2].key, "/test/mid0/leaf0") + self.assertEqual(subtree[3].key, "/test/mid0/leaf1") + self.assertEqual(subtree[4].key, "/test/mid1/") + self.assertEqual(subtree[5].key, "/test/mid1/leaf2") + self.assertEqual(subtree[6].key, "/test/mid1/leaf3") + self.assertEqual(len(subtree), 7) From bcdb84ece3d2c1cb91aa6f9883db5ced357dd9d6 Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Mon, 21 Sep 2015 15:19:27 +0100 Subject: [PATCH 023/101] Ignore responses with no cluster ID when checking the ID. --- src/etcd/client.py | 3 +++ src/etcd/tests/integration/test_simple.py | 1 + 2 files changed, 4 insertions(+) diff --git a/src/etcd/client.py b/src/etcd/client.py index e8293b3e..10395b7e 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -797,6 +797,9 @@ def api_execute(self, path, method, params=None, timeout=None): def _check_cluster_id(self, response): cluster_id = response.getheader("x-etcd-cluster-id") + if not cluster_id: + _log.warning("etcd response did not contain a cluster ID") + return id_changed = (self.expected_cluster_id and cluster_id != self.expected_cluster_id) # Update the ID so we only raise the exception once. diff --git a/src/etcd/tests/integration/test_simple.py b/src/etcd/tests/integration/test_simple.py index c275d6ef..da0954dc 100644 --- a/src/etcd/tests/integration/test_simple.py +++ b/src/etcd/tests/integration/test_simple.py @@ -64,6 +64,7 @@ class TestSimple(EtcdIntegrationTest): def test_machines(self): """ INTEGRATION: retrieve machines """ self.assertEquals(self.client.machines[0], 'http://127.0.0.1:6001') + def test_leader(self): """ INTEGRATION: retrieve leader """ self.assertEquals(self.client.leader['clientURLs'], ['http://127.0.0.1:6001']) From 5f40c6b2d02554c9ed96779308db5876144cec35 Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Mon, 21 Sep 2015 16:41:49 +0100 Subject: [PATCH 024/101] Add cause to EtcdConnectionFailed. --- setup.py | 4 ++-- src/etcd/__init__.py | 7 +++++-- src/etcd/client.py | 11 +++++++---- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 0766c609..f2401e87 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,8 @@ 'pyOpenSSL>=0.14' ] -setup(name='python-etcd', +setup( + name='python-etcd', version=version, description="A python client for etcd", long_description=README + '\n\n' + NEWS, @@ -42,5 +43,4 @@ install_requires=install_requires, tests_require=test_requires, test_suite='nose.collector', - ) diff --git a/src/etcd/__init__.py b/src/etcd/__init__.py index 3fa5747c..b532be6a 100644 --- a/src/etcd/__init__.py +++ b/src/etcd/__init__.py @@ -121,7 +121,7 @@ class EtcdException(Exception): Generic Etcd Exception. """ def __init__(self, message=None, payload=None): - super(Exception, self).__init__(message) + super(EtcdException, self).__init__(message) self.payload = payload @@ -194,7 +194,10 @@ class EtcdConnectionFailed(EtcdException): """ Connection to etcd failed. """ - pass + def __init__(self, message=None, payload=None, cause=None): + super(EtcdConnectionFailed, self).__init__(message=message, + payload=payload) + self.cause = cause class EtcdWatcherCleared(EtcdException): diff --git a/src/etcd/client.py b/src/etcd/client.py index 10395b7e..03b04511 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -704,7 +704,7 @@ def _result_from_response(self, response): raise etcd.EtcdException( 'Unable to decode server response: %r' % e) - def _next_server(self): + def _next_server(self, cause=None): """ Selects the next server in the list, refreshes the server list. """ _log.debug("Selection next machine in cache. Available machines: %s", self._machines_cache) @@ -712,7 +712,8 @@ def _next_server(self): mach = self._machines_cache.pop() except IndexError: _log.error("Machines cache is empty, no machines to try.") - raise etcd.EtcdConnectionFailed('No more machines in the cluster') + raise etcd.EtcdConnectionFailed('No more machines in the cluster', + cause=cause) else: _log.info("Selected new etcd server %s", mach) return mach @@ -778,12 +779,14 @@ def api_execute(self, path, method, params=None, timeout=None): "server.") # _next_server() raises EtcdException if there are no # machines left to try, breaking out of the loop. - self._base_uri = self._next_server() + self._base_uri = self._next_server(cause=e) some_request_failed = True else: _log.debug("Reconnection disabled, giving up.") raise etcd.EtcdConnectionFailed( - "Connection to etcd failed due to %r" % e) + "Connection to etcd failed due to %r" % e, + cause=e + ) except: _log.exception("Unexpected request failure, re-raising.") raise From 1b40e52ce0c9698cb8435004188e00ef6f026f92 Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Tue, 22 Sep 2015 08:58:07 +0100 Subject: [PATCH 025/101] Update .gitignore. --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 00bb6bfc..765321b4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,9 +4,13 @@ bin develop-eggs eggs +.eggs +.idea *.egg-info tmp build dist +docs +etcd .coverage From df6caa92770eb49429bf6ee23f6cebebfe44b9e5 Mon Sep 17 00:00:00 2001 From: Jose Plana Date: Thu, 8 Oct 2015 23:23:51 +0200 Subject: [PATCH 026/101] Prepare release 0.4.2 --- NEWS.txt | 13 +++++++++++++ docs-source/conf.py | 2 +- setup.py | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/NEWS.txt b/NEWS.txt index cc3ae339..2158c21f 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -1,6 +1,19 @@ News ==== +0.4.2 +----- +*Release data: 8-Oct-2015* + +* Fixed lock documentation +* Fixed lock sequences due to etcd 2.2 change +* Better exception management during response processing +* Fixed logging of cluster ID changes +* Fixed subtree results +* Do not check cluster ID if etcd responses don't contain the ID +* Added a cause to EtcdConnectionFailed + + 0.4.1 ----- *Release date: 1-Aug-2015* diff --git a/docs-source/conf.py b/docs-source/conf.py index 8e4218d3..996cb61b 100644 --- a/docs-source/conf.py +++ b/docs-source/conf.py @@ -61,7 +61,7 @@ def __getattr__(cls, name): # The short X.Y version. version = '0.4' # The full version, including alpha/beta/rc tags. -release = '0.4.1' +release = '0.4.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index f2401e87..011f798c 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ NEWS = open(os.path.join(here, 'NEWS.txt')).read() -version = '0.4.1' +version = '0.4.2' install_requires = [ 'urllib3>=1.7' From ee601e99107d1ee97933aeaeebf9d0aee24f9778 Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Wed, 14 Oct 2015 14:38:39 +0100 Subject: [PATCH 027/101] Introduce EtcdWatchTimedOut exception. Suppress spammy error log when a watch times out and raise a dedicated exception instead. EtcdWatchTimedOut subclasses EtcdConnectionFailed for back-compatibility. Revs urllib3 dependency to 1.7.1, which split TimeoutError into ReadTimeoutError and ConnectionTimeoutError. --- buildout.cfg | 2 +- setup.py | 2 +- src/etcd/__init__.py | 7 +++++++ src/etcd/client.py | 8 ++++++++ src/etcd/tests/unit/test_request.py | 16 ++++++++++++++++ 5 files changed, 33 insertions(+), 2 deletions(-) diff --git a/buildout.cfg b/buildout.cfg index 9aaf66eb..bd498e5e 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -4,7 +4,7 @@ parts = python test develop = . eggs = - urllib3==1.7 + urllib3==1.7.1 pyOpenSSL==0.13.1 [python] diff --git a/setup.py b/setup.py index 011f798c..b496fe26 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ version = '0.4.2' install_requires = [ - 'urllib3>=1.7' + 'urllib3>=1.7.1' ] test_requires = [ diff --git a/src/etcd/__init__.py b/src/etcd/__init__.py index b532be6a..2032be3a 100644 --- a/src/etcd/__init__.py +++ b/src/etcd/__init__.py @@ -200,6 +200,13 @@ def __init__(self, message=None, payload=None, cause=None): self.cause = cause +class EtcdWatchTimedOut(EtcdConnectionFailed): + """ + A watch timed out without returning a result. + """ + pass + + class EtcdWatcherCleared(EtcdException): """ Watcher is cleared due to etcd recovery. diff --git a/src/etcd/client.py b/src/etcd/client.py index 03b04511..7d32ccf0 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -772,6 +772,14 @@ def api_execute(self, path, method, params=None, timeout=None): except (urllib3.exceptions.HTTPError, HTTPException, socket.error) as e: + if (params.get("wait") == "true" and + isinstance(e, urllib3.exceptions.ReadTimeoutError)): + _log.debug("Watch timed out.") + raise etcd.EtcdWatchTimedOut( + "Watch timed out: %r" % e, + cause=e + ) + _log.error("Request to server %s failed: %r", self._base_uri, e) if self._allow_reconnect: diff --git a/src/etcd/tests/unit/test_request.py b/src/etcd/tests/unit/test_request.py index 9f540008..177dd064 100644 --- a/src/etcd/tests/unit/test_request.py +++ b/src/etcd/tests/unit/test_request.py @@ -1,3 +1,5 @@ +import urllib3 + import etcd from etcd.tests.unit import TestClientApiBase @@ -428,6 +430,20 @@ def test_compare_and_swap_failure(self): prevValue='oldbog' ) + def test_watch_timeout(self): + """ Exception will be raised if prevValue != value in test_set """ + self.client.http.request = mock.create_autospec( + self.client.http.request, + side_effect=urllib3.exceptions.ReadTimeoutError(self.client.http, + "foo", + "Read timed out") + ) + self.assertRaises( + etcd.EtcdWatchTimedOut, + self.client.watch, + '/testKey', + ) + def test_path_without_trailing_slash(self): """ Exception will be raised if a path without a trailing slash is used """ self.assertRaises(ValueError, self.client.api_execute, From fe684098037ca09db0c8e3dfb989ee400fddbd24 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Sat, 20 Jun 2015 13:44:52 +0200 Subject: [PATCH 028/101] Add srv record-based DNS discovery. We use the same keys used by confd (https://github.com/kelseyhightower/confd) to allow service discovery via DNS. --- .travis.yml | 4 ++-- README.rst | 2 ++ buildout.cfg | 10 ++++++++++ setup.py | 9 ++++++++- src/etcd/client.py | 29 +++++++++++++++++++++++++++-- src/etcd/tests/unit/test_client.py | 25 +++++++++++++++++++++++++ 6 files changed, 74 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 46a25767..15380592 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,10 @@ language: python python: - "2.7" - - "3.3" + - "3.5" before_install: - - ./build_etcd.sh v2.0.10 + - ./build_etcd.sh v2.2.0 - pip install --upgrade setuptools # command to install dependencies diff --git a/README.rst b/README.rst index 163d2578..ee5b70c8 100644 --- a/README.rst +++ b/README.rst @@ -41,6 +41,8 @@ Create a client object client = etcd.Client(port=4002) client = etcd.Client(host='127.0.0.1', port=4003) client = etcd.Client(host='127.0.0.1', port=4003, allow_redirect=False) # wont let you run sensitive commands on non-leader machines, default is true + # If you have defined a SRV record for _etcd._tcp.example.com pointing to the clients + client = etcd.Client(srv_domain='example.com', protocol="https") # create a client against https://api.example.com:443/etcd client = etcd.Client(host='api.example.com', protocol='https', port=443, version_prefix='/etcd') Write a key diff --git a/buildout.cfg b/buildout.cfg index bd498e5e..cba64c58 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -6,6 +6,7 @@ develop = . eggs = urllib3==1.7.1 pyOpenSSL==0.13.1 + ${deps:extraeggs} [python] recipe = zc.recipe.egg @@ -21,3 +22,12 @@ eggs = ${python:eggs} recipe = collective.recipe.sphinxbuilder source = ${buildout:directory}/docs-source build = ${buildout:directory}/docs + + +[deps:python2] +extraeggs = + dnspython==1.12.0 + +[deps:python3] +extraeggs = + dnspython3==1.12.0 diff --git a/setup.py b/setup.py index b496fe26..3d0d4509 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,15 @@ version = '0.4.2' +# Dnspython is two different packages depending on python version +if sys.version_info.major == 2: + dns = 'dnspython' +else: + dns = 'dnspython3' + install_requires = [ - 'urllib3>=1.7.1' + 'urllib3>=1.7.1', + dns ] test_requires = [ diff --git a/src/etcd/client.py b/src/etcd/client.py index 7d32ccf0..c0cae843 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -18,6 +18,7 @@ import urllib3.util import json import ssl +import dns.resolver import etcd try: @@ -46,6 +47,7 @@ def __init__( self, host='127.0.0.1', port=4001, + srv_domain=None, version_prefix='/v2', read_timeout=60, allow_redirect=True, @@ -67,6 +69,8 @@ def __init__( port (int): Port used to connect to etcd. + srv_domain (str): Domain to search the SRV record for cluster autodiscovery. + version_prefix (str): Url or version prefix in etcd url (default=/v2). read_timeout (int): max seconds to wait for a read. @@ -98,8 +102,15 @@ def __init__( by host. By default this will use up to 10 connections. """ - _log.debug("New etcd client created for %s:%s%s", - host, port, version_prefix) + + # If a DNS record is provided, use it to get the hosts list + if srv_domain is not None: + try: + host = self._discover(srv_domain) + except Exception as e: + _log.error("Could not discover the etcd hosts from %s: %s", + srv_domain, e) + self._protocol = protocol def uri(protocol, host, port): @@ -153,6 +164,8 @@ def uri(protocol, host, port): self.http = urllib3.PoolManager(num_pools=10, **kw) + _log.debug("New etcd client created for %s", self.base_uri) + if self._allow_reconnect: # we need the set of servers in the cluster in order to try # reconnecting upon error. The cluster members will be @@ -174,6 +187,18 @@ def uri(protocol, host, port): _log.debug("Machines cache initialised to %s", self._machines_cache) + def _discover(self, domain): + srv_name = "_etcd._tcp.{}".format(domain) + answers = dns.resolver.query(srv_name, 'SRV') + hosts = [] + for answer in answers: + hosts.append( + (answer.target.to_text(omit_final_dot=True), answer.port)) + _log.debug("Found %s", hosts) + if not len(hosts): + raise ValueError("The SRV record is present but no host were found") + return tuple(hosts) + @property def base_uri(self): """URI used by the client to connect to etcd.""" diff --git a/src/etcd/tests/unit/test_client.py b/src/etcd/tests/unit/test_client.py index 2e09d7cb..e5d10998 100644 --- a/src/etcd/tests/unit/test_client.py +++ b/src/etcd/tests/unit/test_client.py @@ -1,5 +1,12 @@ import unittest import etcd +import dns.name +import dns.rdtypes.IN.SRV +import dns.resolver +try: + import mock +except ImportError: + from unittest import mock class TestClient(unittest.TestCase): @@ -97,3 +104,21 @@ def test_allow_reconnect(self): allow_reconnect=True, use_proxies=True, ) + + def test_discover(self): + """Tests discovery.""" + answers = [] + for i in range(1,3): + r = mock.create_autospec(dns.rdtypes.IN.SRV.SRV) + r.port = 2379 + r.target = dns.name.from_unicode(u'etcd{}.example.com'.format(i)) + answers.append(r) + dns.resolver.query = mock.create_autospec(dns.resolver.query, return_value=answers) + self.machines = etcd.Client.machines + etcd.Client.machines = mock.create_autospec(etcd.Client.machines, return_value=[u'https://etcd2.example.com:2379']) + c = etcd.Client(srv_domain="example.com", allow_reconnect=True, protocol="https") + etcd.Client.machines = self.machines + self.assertEquals(c.host, u'etcd1.example.com') + self.assertEquals(c.port, 2379) + self.assertEquals(c._machines_cache, + [u'https://etcd2.example.com:2379']) From daf83098669ea6d2627839ec8612291ea26bdfa3 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Sun, 1 Nov 2015 13:01:18 +0100 Subject: [PATCH 029/101] Fix tests for python 3.5 --- src/etcd/tests/unit/__init__.py | 3 +-- src/etcd/tests/unit/test_client.py | 6 +++++- src/etcd/tests/unit/test_lock.py | 8 +++++--- src/etcd/tests/unit/test_request.py | 9 +++------ 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/etcd/tests/unit/__init__.py b/src/etcd/tests/unit/__init__.py index 1dd44ab9..9360b6bb 100644 --- a/src/etcd/tests/unit/__init__.py +++ b/src/etcd/tests/unit/__init__.py @@ -27,8 +27,7 @@ def _prepare_response(self, s, d, cluster_id=None): def _mock_api(self, status, d, cluster_id=None): resp = self._prepare_response(status, d, cluster_id=cluster_id) - self.client.api_execute = mock.create_autospec( - self.client.api_execute, return_value=resp) + self.client.api_execute = mock.MagicMock(return_value=resp) def _mock_exception(self, exc, msg): self.client.api_execute = mock.Mock(side_effect=exc(msg)) diff --git a/src/etcd/tests/unit/test_client.py b/src/etcd/tests/unit/test_client.py index e5d10998..43017329 100644 --- a/src/etcd/tests/unit/test_client.py +++ b/src/etcd/tests/unit/test_client.py @@ -111,7 +111,11 @@ def test_discover(self): for i in range(1,3): r = mock.create_autospec(dns.rdtypes.IN.SRV.SRV) r.port = 2379 - r.target = dns.name.from_unicode(u'etcd{}.example.com'.format(i)) + try: + method = dns.name.from_unicode + except AttributeError: + method = dns.name.from_text + r.target = method(u'etcd{}.example.com'.format(i)) answers.append(r) dns.resolver.query = mock.create_autospec(dns.resolver.query, return_value=answers) self.machines = etcd.Client.machines diff --git a/src/etcd/tests/unit/test_lock.py b/src/etcd/tests/unit/test_lock.py index 75ae6761..6a41f133 100644 --- a/src/etcd/tests/unit/test_lock.py +++ b/src/etcd/tests/unit/test_lock.py @@ -1,5 +1,8 @@ import etcd -import mock +try: + import mock +except ImportError: + from unittest import mock from etcd.tests.unit import TestClientApiBase @@ -92,8 +95,7 @@ def test_acquired(self): self.assertTrue(self.locker._acquired()) self.assertTrue(self.locker.is_taken) retval = ('/_locks/test_lock/1', '/_locks/test_lock/4') - self.locker._get_locker = mock.create_autospec( - self.locker._get_locker, return_value=retval) + self.locker._get_locker = mock.MagicMock(return_value=retval) self.assertFalse(self.locker._acquired(blocking=False)) self.assertFalse(self.locker.is_taken) d = { diff --git a/src/etcd/tests/unit/test_request.py b/src/etcd/tests/unit/test_request.py index 177dd064..ea837b69 100644 --- a/src/etcd/tests/unit/test_request.py +++ b/src/etcd/tests/unit/test_request.py @@ -397,12 +397,9 @@ def setUp(self): def _mock_api(self, status, d, cluster_id=None): resp = self._prepare_response(status, d) resp.getheader.return_value = cluster_id or "abcdef1234" - self.client.http.request_encode_body = mock.create_autospec( - self.client.http.request_encode_body, return_value=resp - ) - self.client.http.request = mock.create_autospec( - self.client.http.request, return_value=resp - ) + self.client.http.request_encode_body = mock.MagicMock( + return_value=resp) + self.client.http.request = mock.MagicMock(return_value=resp) def _mock_error(self, error_code, msg, cause, method='PUT', fields=None, cluster_id=None): From a56b502d648423f75c8885cbe9e6ce601bec3cee Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Sun, 1 Nov 2015 13:12:04 +0100 Subject: [PATCH 030/101] Add coveralls badge to the readme So that we can be properly ashamed of ourselves --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index ee5b70c8..c9d6dabc 100644 --- a/README.rst +++ b/README.rst @@ -8,6 +8,9 @@ Official documentation: http://python-etcd.readthedocs.org/ .. image:: https://travis-ci.org/jplana/python-etcd.png?branch=master :target: https://travis-ci.org/jplana/python-etcd +.. image:: https://coveralls.io/repos/jplana/python-etcd/badge.svg?branch=master&service=github + :target: https://coveralls.io/github/jplana/python-etcd?branch=master + Installation ------------ From 538ee04f91af927a1c7015dce8d1310299996e18 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Sun, 1 Nov 2015 14:29:06 +0100 Subject: [PATCH 031/101] Add coveralls support --- .travis.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 15380592..2c3ba505 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,13 +8,16 @@ before_install: - pip install --upgrade setuptools # command to install dependencies -install: +install: + - pip install coveralls + - pip install coverage - python bootstrap.py - bin/buildout # command to run tests script: - PATH=$PATH:./etcd/bin bin/test + PATH=$PATH:./etcd/bin coverage run --source=src/etcd --omit="src/etcd/tests/*" bin/test +after_success: coveralls # Add env var to detect it during build env: TRAVIS=True From 7c35b2df3ce7318d65635ee677c6070f4b85603d Mon Sep 17 00:00:00 2001 From: Matthias Urlichs Date: Sun, 11 Oct 2015 04:15:43 +0200 Subject: [PATCH 032/101] Client: clean up open connections when deleting Otherwise you get some warnings about still-open connections from Python3. --- src/etcd/client.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index c0cae843..62b8106e 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -43,6 +43,9 @@ class Client(object): _comparison_conditions = set(('prevValue', 'prevIndex', 'prevExist')) _read_options = set(('recursive', 'wait', 'waitIndex', 'sorted', 'quorum')) _del_conditions = set(('prevValue', 'prevIndex')) + + http = None + def __init__( self, host='127.0.0.1', @@ -199,6 +202,11 @@ def _discover(self, domain): raise ValueError("The SRV record is present but no host were found") return tuple(hosts) + def __del__(self): + """Clean up open connections""" + if self.http is not None: + self.http.clear() + @property def base_uri(self): """URI used by the client to connect to etcd.""" @@ -783,7 +791,7 @@ def api_execute(self, path, method, params=None, timeout=None): else: raise etcd.EtcdException( 'HTTP method {} not supported'.format(method)) - + # Check the cluster ID hasn't changed under us. We use # preload_content=False above so we can read the headers # before we wait for the content of a watch. @@ -823,7 +831,7 @@ def api_execute(self, path, method, params=None, timeout=None): except: _log.exception("Unexpected request failure, re-raising.") raise - + if some_request_failed: if not self._use_proxies: # The cluster may have changed since last invocation From 186215f385ac87cf3136c57f05be4cacca480033 Mon Sep 17 00:00:00 2001 From: Matthias Urlichs Date: Sun, 11 Oct 2015 06:27:02 +0200 Subject: [PATCH 033/101] Suppress ReferenceError when cleaning up since we can't do anything about it at this point --- src/etcd/client.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index 62b8106e..e9ec02ce 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -205,7 +205,11 @@ def _discover(self, domain): def __del__(self): """Clean up open connections""" if self.http is not None: - self.http.clear() + try: + self.http.clear() + except ReferenceError: + # this may hit an already-cleared weakref + pass @property def base_uri(self): From 6d231cf0f9bf9d81c4e1037c3c6606f98ed1daf8 Mon Sep 17 00:00:00 2001 From: Matthias Urlichs Date: Wed, 11 Nov 2015 11:20:32 +0100 Subject: [PATCH 034/101] Fix leader lookup /stats/leader only works when talking to the leader, which is not helpful when checking which node it _is_. --- src/etcd/client.py | 4 ++-- src/etcd/tests/unit/test_request.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index e9ec02ce..ab17acd4 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -315,9 +315,9 @@ def leader(self): try: leader = json.loads( - self.api_execute(self.version_prefix + '/stats/leader', + self.api_execute(self.version_prefix + '/stats/self', self._MGET).data.decode('utf-8')) - return self.members[leader['leader']] + return self.members[leader['leaderInfo']['leader']] except Exception as e: raise etcd.EtcdException("Cannot get leader data: %s" % e) diff --git a/src/etcd/tests/unit/test_request.py b/src/etcd/tests/unit/test_request.py index ea837b69..2456ae1d 100644 --- a/src/etcd/tests/unit/test_request.py +++ b/src/etcd/tests/unit/test_request.py @@ -159,7 +159,7 @@ def test_leader(self, mocker): """ Can request the leader """ members = {"ce2a822cea30bfca": {"id": "ce2a822cea30bfca", "name": "default"}} mocker.return_value = members - self._mock_api(200, {"leader": "ce2a822cea30bfca", "followers": {}}) + self._mock_api(200, {"leaderInfo":{"leader": "ce2a822cea30bfca", "followers": {}}}) self.assertEquals(self.client.leader, members["ce2a822cea30bfca"]) def test_set_plain(self): From 817adc5348a798d2981e6cc5b988373a0985cf54 Mon Sep 17 00:00:00 2001 From: Peter Wagner Date: Fri, 11 Sep 2015 08:51:34 -0400 Subject: [PATCH 035/101] User authentication initial * Initial BASIC auth implementation + unit tests Checkpoint before integration tests. --- src/etcd/client.py | 26 +++++++++++++++++++++ src/etcd/tests/integration/helpers.py | 6 ++--- src/etcd/tests/unit/test_client.py | 33 +++++++++++++++++++++++++++ src/etcd/tests/unit/test_lock.py | 12 +++++----- 4 files changed, 68 insertions(+), 9 deletions(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index ab17acd4..5bb69f49 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -57,6 +57,8 @@ def __init__( protocol='http', cert=None, ca_cert=None, + username=None, + password=None, allow_reconnect=False, use_proxies=False, expected_cluster_id=None, @@ -88,6 +90,10 @@ def __init__( ca_cert (str): The ca certificate. If pressent it will enable validation. + username (str): username for etcd authentication. + + password (str): password for etcd authentication. + allow_reconnect (bool): allow the client to reconnect to another etcd server in the cluster in the case the default one does not respond. @@ -165,6 +171,16 @@ def uri(protocol, host, port): kw['ca_certs'] = ca_cert kw['cert_reqs'] = ssl.CERT_REQUIRED + self.username = None + self.password = None + if username and password: + self.username = username + self.password = password + elif username: + _log.warning('Username provided without password, both are required for authentication') + elif password: + _log.warning('Password provided without username, both are required for authentication') + self.http = urllib3.PoolManager(num_pools=10, **kw) _log.debug("New etcd client created for %s", self.base_uri) @@ -258,6 +274,7 @@ def machines(self): response = self.http.request( self._MGET, uri, + headers=self._get_headers(), timeout=self.read_timeout, redirect=self.allow_redirect) @@ -781,6 +798,7 @@ def api_execute(self, path, method, params=None, timeout=None): timeout=timeout, fields=params, redirect=self.allow_redirect, + headers=self._get_headers(), preload_content=False) elif (method == self._MPUT) or (method == self._MPOST): @@ -791,6 +809,7 @@ def api_execute(self, path, method, params=None, timeout=None): timeout=timeout, encode_multipart=False, redirect=self.allow_redirect, + headers=self._get_headers(), preload_content=False) else: raise etcd.EtcdException( @@ -877,3 +896,10 @@ def _handle_server_response(self, response): r = {"message": "Bad response", "cause": str(resp)} etcd.EtcdError.handle(r) + + def _get_headers(self): + if self.username and self.password: + credentials = ':'.join((self.username, self.password)) + return urllib3.make_headers(basic_auth=credentials) + return {} + diff --git a/src/etcd/tests/integration/helpers.py b/src/etcd/tests/integration/helpers.py index 6c7e21ca..3314be9e 100644 --- a/src/etcd/tests/integration/helpers.py +++ b/src/etcd/tests/integration/helpers.py @@ -77,12 +77,12 @@ def add_one(self, slot, proc_args=None): def kill_one(self, slot): log = logging.getLogger() - dir, process = self.processes.pop(slot) + data_dir, process = self.processes.pop(slot) process.kill() time.sleep(2) log.debug('Killed etcd pid:%d', process.pid) - shutil.rmtree(dir) - log.debug('Removed directory %s' % dir) + shutil.rmtree(data_dir) + log.debug('Removed directory %s' % data_dir) class TestingCA(object): diff --git a/src/etcd/tests/unit/test_client.py b/src/etcd/tests/unit/test_client.py index 43017329..bb05a66a 100644 --- a/src/etcd/tests/unit/test_client.py +++ b/src/etcd/tests/unit/test_client.py @@ -45,6 +45,16 @@ def test_default_allow_redirect(self): client = etcd.Client() assert client.allow_redirect + def test_default_username(self): + """ default username is None""" + client = etcd.Client() + assert client.username is None + + def test_default_password(self): + """ default username is None""" + client = etcd.Client() + assert client.password is None + def test_set_host(self): """ can change host """ client = etcd.Client(host='192.168.1.1') @@ -92,6 +102,29 @@ def test_set_use_proxies(self): client = etcd.Client(use_proxies = True) assert client._use_proxies + def test_set_username_only(self): + client = etcd.Client(username='username') + assert client.username is None + + def test_set_password_only(self): + client = etcd.Client(password='password') + assert client.password is None + + def test_set_username_password(self): + client = etcd.Client(username='username', password='password') + assert client.username == 'username' + assert client.password == 'password' + + def test_get_headers_with_auth(self): + client = etcd.Client(username='username', password='password') + assert client._get_headers() == { + 'authorization': 'Basic dXNlcm5hbWU6cGFzc3dvcmQ=' + } + + def test_get_headers_without_auth(self): + client = etcd.Client() + assert client._get_headers() == {} + def test_allow_reconnect(self): """ Fails if allow_reconnect is false and a list of hosts is given""" with self.assertRaises(etcd.EtcdException): diff --git a/src/etcd/tests/unit/test_lock.py b/src/etcd/tests/unit/test_lock.py index 6a41f133..1b374c8b 100644 --- a/src/etcd/tests/unit/test_lock.py +++ b/src/etcd/tests/unit/test_lock.py @@ -40,8 +40,8 @@ def test_acquire(self): Acquiring a precedingly inexistent lock works. """ l = etcd.Lock(self.client, 'test_lock') - l._find_lock = mock.create_autospec(l._find_lock, return_value=False) - l._acquired = mock.create_autospec(l._acquired, return_value=True) + l._find_lock = mock.MagicMock(spec=l._find_lock, return_value=False) + l._acquired = mock.MagicMock(spec=l._acquired, return_value=True) # Mock the write d = { u'action': u'set', @@ -90,8 +90,8 @@ def test_acquired(self): """ self.locker._sequence = '4' retval = ('/_locks/test_lock/4', None) - self.locker._get_locker = mock.create_autospec( - self.locker._get_locker, return_value=retval) + self.locker._get_locker = mock.MagicMock( + spec=self.locker._get_locker, return_value=retval) self.assertTrue(self.locker._acquired()) self.assertTrue(self.locker.is_taken) retval = ('/_locks/test_lock/1', '/_locks/test_lock/4') @@ -112,8 +112,8 @@ def test_acquired(self): def side_effect(): return returns.pop() - self.locker._get_locker = mock.create_autospec( - self.locker._get_locker, side_effect=side_effect) + self.locker._get_locker = mock.MagicMock( + spec=self.locker._get_locker, side_effect=side_effect) self.assertTrue(self.locker._acquired()) def test_lock_key(self): From f30c873af8b700d18c0b664422252f7f3fce91e3 Mon Sep 17 00:00:00 2001 From: Peter Wagner Date: Thu, 24 Sep 2015 14:45:58 -0400 Subject: [PATCH 036/101] etcd.auth.AuthClient This extension affords create/read/update without cluttering the basic etcd.Client implementation. The model is reworked for a cleaner API: user's roles can be assigned via list/tuple, permissions are moddeled like a dictionary. Adding coverage goal to buildout to verify testing progress. --- buildout.cfg | 8 + src/etcd/auth.py | 369 ++++++++++++++++++ src/etcd/tests/integration/helpers.py | 6 + .../tests/integration/test_authentication.py | 195 +++++++++ 4 files changed, 578 insertions(+) create mode 100644 src/etcd/auth.py create mode 100644 src/etcd/tests/integration/test_authentication.py diff --git a/buildout.cfg b/buildout.cfg index cba64c58..4de90366 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -2,6 +2,7 @@ parts = python sphinxbuilder test + coverage develop = . eggs = urllib3==1.7.1 @@ -18,6 +19,13 @@ recipe = pbp.recipe.noserunner eggs = ${python:eggs} mock +[coverage] +recipe = pbp.recipe.noserunner +eggs = ${test:eggs} + coverage +defaults = --with-coverage + --cover-package=etcd + [sphinxbuilder] recipe = collective.recipe.sphinxbuilder source = ${buildout:directory}/docs-source diff --git a/src/etcd/auth.py b/src/etcd/auth.py new file mode 100644 index 00000000..0f8a2dd8 --- /dev/null +++ b/src/etcd/auth.py @@ -0,0 +1,369 @@ +import json + +import logging + +try: + # Python 3 + from http.client import HTTPException +except ImportError: + # Python 2 + from httplib import HTTPException +import socket +import urllib3 + +from .client import Client +import etcd + +_log = logging.getLogger(__name__) + + +class AuthClient(Client): + """ + Extended etcd client that supports authentication primitives added in 2.1. + """ + + def __init__(self, *args, **kwargs): + super(AuthClient, self).__init__(*args, **kwargs) + + def create_user(self, username, password, roles=[], role_action='roles'): + """ + Add a user. + + Args: + username (str): Username to create. + password (str): Password for username. + roles (list): List of roles as strings. + + Returns: + EtcdUser + + Raises: + etcd.EtcdException: If user can't be created. + """ + try: + uri = self.version_prefix + '/auth/users/' + username + params = {'user': username} + if password: + params['password'] = password + if roles: + params[role_action] = roles + + response = self.json_api_execute(uri, self._MPUT, params=params) + res = json.loads(response.data.decode('utf-8')) + return EtcdUser(self, res) + except Exception as e: + _log.error("Failed to create user in %s%s: %r", + self._base_uri, self.version_prefix, e) + raise etcd.EtcdException("Could not create user") + + def get_user(self, username): + """ + Look up a user. + + Args: + username (str): Username to lookup. + + Returns: + EtcdUser + + Raises: + etcd.EtcdException: If user can't be found. + """ + try: + uri = self.version_prefix + '/auth/users/' + username + response = self.api_execute(uri, self._MGET) + res = json.loads(response.data.decode('utf-8')) + return EtcdUser(self, res) + except Exception as e: + _log.error("Failed to fetch user in %s%s: %r", + self._base_uri, self.version_prefix, e) + raise etcd.EtcdException("Could not fetch user") + + @property + def usernames(self): + """List user names.""" + try: + uri = self.version_prefix + '/auth/users' + response = self.api_execute(uri, self._MGET) + res = json.loads(response.data.decode('utf-8')) + return res['users'] + except Exception as e: + _log.error("Failed to list users in %s%s: %r", + self._base_uri, self.version_prefix, e) + raise etcd.EtcdException("Could not list users") + + @property + def users(self): + """List users in detail.""" + return [self.get_user(x) for x in self.usernames] + + def create_role(self, role_name): + """ + Create a role. + + Args: + role_name (str): Name of role + + Returns: + EtcdRole + """ + return self.modify_role(role_name) + + def get_role(self, role_name): + """ + Look up a role. + + Args: + role_name (str): Name of role. + + Returns: + EtcdRole + """ + try: + uri = self.version_prefix + '/auth/roles/' + role_name + response = self.api_execute(uri, self._MGET) + res = json.loads(response.data.decode('utf-8')) + return EtcdRole(self, res) + except Exception as e: + _log.error("Failed to fetch user in %s%s: %r", + self._base_uri, self.version_prefix, e) + raise etcd.EtcdException("Could not fetch users") + + @property + def role_names(self): + """List role names.""" + try: + uri = self.version_prefix + '/auth/roles' + response = self.api_execute(uri, self._MGET) + res = json.loads(response.data.decode('utf-8')) + return res['roles'] + except Exception as e: + _log.error("Failed to list roles in %s%s: %r", + self._base_uri, self.version_prefix, e) + raise etcd.EtcdException("Could not list roles") + + @property + def roles(self): + """List roles in detail.""" + return [self.get_role(x) for x in self.role_names] + + def toggle_auth(self, auth_enabled=True): + """ + Toggle authentication. + + Args: + auth_enabled (bool): Should auth be enabled or disabled + """ + try: + uri = self.version_prefix + '/auth/enable' + action = auth_enabled and self._MPUT or self._MDELETE + + self.api_execute(uri, action) + except Exception as e: + _log.error("Failed enable authentication in %s%s: %r", + self._base_uri, self.version_prefix, e) + raise etcd.EtcdException("Could not toggle authentication") + + def modify_role(self, role_name, permissions=None, perm_key=None): + """Modifies role.""" + try: + uri = self.version_prefix + '/auth/roles/' + role_name + params = { + 'role': role_name, + } + if permissions: + params[perm_key] = { + 'kv': { + 'read': [k for k, v in permissions.items() if + 'R' in v.upper()], + 'write': [k for k, v in permissions.items() if + 'W' in v.upper()] + } + } + response = self.json_api_execute(uri, self._MPUT, params=params) + res = json.loads(response.data.decode('utf-8')) + return EtcdRole(self, res) + except Exception as e: + _log.error("Failed to modify role in %s%s: %r", + self._base_uri, self.version_prefix, e) + raise etcd.EtcdException("Could not modify role") + + def json_api_execute(self, path, method, params=None, timeout=None): + """ Executes the query. """ + + some_request_failed = False + response = False + + if timeout is None: + timeout = self.read_timeout + + if timeout == 0: + timeout = None + + if not path.startswith('/'): + raise ValueError('Path does not start with /') + + while not response: + try: + url = self._base_uri + path + json_payload = json.dumps(params) + headers = self._get_headers() + headers['Content-Type'] = 'application/json' + response = self.http.urlopen(method, + url, + body=json_payload, + timeout=timeout, + redirect=self.allow_redirect, + headers=headers, + preload_content=False) + # urllib3 doesn't wrap all httplib exceptions and earlier versions + # don't wrap socket errors either. + except (urllib3.exceptions.HTTPError, + HTTPException, + socket.error) as e: + _log.error("Request to server %s failed: %r", + self._base_uri, e) + if self._allow_reconnect: + _log.info("Reconnection allowed, looking for another " + "server.") + # _next_server() raises EtcdException if there are no + # machines left to try, breaking out of the loop. + self._base_uri = self._next_server() + some_request_failed = True + else: + _log.debug("Reconnection disabled, giving up.") + raise etcd.EtcdConnectionFailed( + "Connection to etcd failed due to %r" % e) + except: + _log.exception("Unexpected request failure, re-raising.") + raise + + else: + # Check the cluster ID hasn't changed under us. We use + # preload_content=False above so we can read the headers + # before we wait for the content of a long poll. + cluster_id = response.getheader("x-etcd-cluster-id") + id_changed = (self.expected_cluster_id + and cluster_id is not None and + cluster_id != self.expected_cluster_id) + # Update the ID so we only raise the exception once. + old_expected_cluster_id = self.expected_cluster_id + self.expected_cluster_id = cluster_id + if id_changed: + # Defensive: clear the pool so that we connect afresh next + # time. + self.http.clear() + raise etcd.EtcdClusterIdChanged( + 'The UUID of the cluster changed from {} to ' + '{}.'.format(old_expected_cluster_id, cluster_id)) + + if some_request_failed: + if not self._use_proxies: + # The cluster may have changed since last invocation + self._machines_cache = self.machines + self._machines_cache.remove(self._base_uri) + return self._handle_server_response(response) + + +class EtcdUser(object): + def __init__(self, auth_client, json_user): + self.client = auth_client + self.name = json_user.get('user') + self._roles = json_user.get('roles') or [] + + @property + def password(self): + """Empty property for password.""" + return None + + @password.setter + def password(self, new_password): + """Change user's password.""" + self.client.create_user(self.name, new_password) + + @property + def roles(self): + return tuple(self._roles) + + @roles.setter + def roles(self, roles): + existing_roles = set(self._roles) + new_roles = set(roles) + + if existing_roles == new_roles: + _log.debug('User %s already belongs to %s', self.name, self._roles) + return + + to_revoke = existing_roles - new_roles + to_grant = new_roles - existing_roles + + if to_revoke: + self.client.create_user(self.name, None, roles=list(to_revoke), + role_action='revoke') + if to_grant: + self.client.create_user(self.name, None, roles=list(to_grant), + role_action='grant') + self._roles = new_roles + + +class EtcdRole(object): + def __init__(self, auth_client, role_json): + self.client = auth_client + self.name = role_json.get('role') + self.permissions = RolePermissionsDict(self, role_json) + + +class RolePermissionsDict(dict): + _PERMISSIONS = {'R', 'W'} + + def __init__(self, etcd_role, role_json, *args, **kwargs): + super(RolePermissionsDict, self).__init__(*args, **kwargs) + self.role = etcd_role + permissions = role_json.get('permissions') + if permissions and 'kv' in permissions: + self.__add_permissions(permissions, 'read', 'R') + self.__add_permissions(permissions, 'write', 'W') + + def __add_permissions(self, permissions, label, symbol): + if label in permissions['kv'] and permissions['kv'][label]: + for path in permissions['kv'][label]: + existing_perms = dict.get(self, path) + if existing_perms: + dict.__setitem__(self, path, + existing_perms + symbol) + else: + dict.__setitem__(self, path, symbol) + + def __setitem__(self, key, value): + if not value: + raise ValueError('Permissions may only be (R)ead or (W)ite') + perms = set(x.upper() for x in value) + if not perms <= RolePermissionsDict._PERMISSIONS: + raise ValueError('Permissions may only be (R)ead or (W)ite') + + role_name = self.role.name + perm_dict = {key: value} + existing_value = dict.get(self, key) + + if existing_value: + existing_perms = set(x.upper() for x in existing_value) + if perms != existing_perms: + to_grant = perms - existing_perms + to_revoke = existing_perms - perms + + if to_revoke: + perm_dict = {key: ''.join(to_revoke)} + self.role.client.modify_role(role_name, perm_dict, 'revoke') + if to_grant: + perm_dict = {key: ''.join(to_grant)} + self.role.client.modify_role(role_name, perm_dict, 'grant') + else: + _log.debug('Permission %s=%s already granted', key, value) + else: + self.role.client.modify_role(role_name, perm_dict, 'grant') + + dict.__setitem__(self, key, value) + + def __delitem__(self, key): + self.role.client.modify_role(self.role.name, {key: 'RW'}, 'revoke') + dict.__delitem__(self, key) diff --git a/src/etcd/tests/integration/helpers.py b/src/etcd/tests/integration/helpers.py index 3314be9e..1f1d22bf 100644 --- a/src/etcd/tests/integration/helpers.py +++ b/src/etcd/tests/integration/helpers.py @@ -38,6 +38,12 @@ def run(self, number=1, proc_args=[]): '-initial-cluster', initial_cluster, '-initial-cluster-state', 'new' ]) + else: + proc_args.extend([ + '-initial-cluster', 'test-node-0=http://127.0.0.1:{}'.format(self.internal_port_range_start), + '-initial-cluster-state', 'new' + ]) + for i in range(0, number): self.add_one(i, proc_args) diff --git a/src/etcd/tests/integration/test_authentication.py b/src/etcd/tests/integration/test_authentication.py new file mode 100644 index 00000000..52ba001b --- /dev/null +++ b/src/etcd/tests/integration/test_authentication.py @@ -0,0 +1,195 @@ +import unittest +import shutil +import tempfile + +import time + +import etcd +import etcd.auth +from etcd.tests.integration.test_simple import EtcdIntegrationTest +from etcd.tests.integration import helpers + + +class TestAuthentication(unittest.TestCase): + def setUp(self): + # Restart etcd for each test (since some tests will lock others out) + program = EtcdIntegrationTest._get_exe() + self.directory = tempfile.mkdtemp(prefix='python-etcd') + self.processHelper = helpers.EtcdProcessHelper( + self.directory, + proc_name=program, + port_range_start=6001, + internal_port_range_start=8001) + self.processHelper.run(number=1) + self.client = etcd.auth.AuthClient(port=6001) + + # Wait for sync, to avoid: + # "Not capable of accessing auth feature during rolling upgrades." + time.sleep(0.5) + + def tearDown(self): + self.processHelper.stop() + shutil.rmtree(self.directory) + + def test_create_user(self): + user = self.client.create_user('username', 'password') + assert user.name == 'username' + assert len(user.roles) == 0 + + def test_create_user_with_role(self): + user = self.client.create_user('username', 'password', roles=['root']) + assert user.name == 'username' + assert user.roles == ('root',) + + def test_create_user_add_role(self): + user = self.client.create_user('username', 'password') + self.client.create_role('role') + + # Empty to [root] + user.roles = ['root'] + user = self.client.get_user('username') + assert user.roles == ('root',) + + # [root] to [root,role] + user.roles = ['root', 'role'] + user = self.client.get_user('username') + assert user.roles == ('role', 'root') + + # [root,role] to [role] + user.roles = ['role'] + user = self.client.get_user('username') + assert user.roles == ('role',) + + def test_usernames_empty(self): + assert len(self.client.usernames) == 0 + + def test_usernames(self): + self.client.create_user('username', 'password', roles=['root']) + assert self.client.usernames == ['username'] + + def test_users(self): + self.client.create_user('username', 'password', roles=['root']) + users = self.client.users + assert len(users) == 1 + assert users[0].name == 'username' + + def test_get_user(self): + self.client.create_user('username', 'password', roles=['root']) + user = self.client.get_user('username') + assert user.roles == ('root',) + + def test_get_user_not_found(self): + self.assertRaises(etcd.EtcdException, self.client.get_user, 'username') + + def test_set_user_password(self): + self.client.create_user('username', 'password', roles=['root']) + user = self.client.get_user('username') + assert not user.password + user.password = 'new_password' + assert not user.password + + def test_create_role(self): + role = self.client.create_role('role') + assert role.name == 'role' + assert len(role.permissions) == 0 + + def test_grant_role(self): + role = self.client.create_role('role') + + # Read access to keys under /foo + role.permissions['/foo/*'] = 'R' + assert len(role.permissions) == 1 + assert role.permissions['/foo/*'] == 'R' + + # Write access to the key at /foo/bar + role.permissions['/foo/bar'] = 'W' + assert len(role.permissions) == 2 + + # Full access to keys under /pub + role.permissions['/pub/*'] = 'RW' + assert len(role.permissions) == 3 + + # Fresh fetch to bust cache: + role = self.client.get_role('role') + assert len(role.permissions) == 3 + + def test_get_role(self): + role = self.client.create_role('role') + role.permissions['/foo/*'] = 'R' + + role = self.client.get_role('role') + assert len(role.permissions) == 1 + + def test_revoke_role(self): + role = self.client.create_role('role') + role.permissions['/foo/*'] = 'R' + + del role.permissions['/foo/*'] + + role = self.client.get_role('role') + assert len(role.permissions) == 0 + + def test_modify_role_invalid(self): + role = self.client.create_role('role') + self.assertRaises(ValueError, role.permissions.__setitem__, '/foo/*', + '') + + def test_modify_role_permissions(self): + role = self.client.create_role('role') + role.permissions['/foo/*'] = 'R' + + # Replace R with W + role.permissions['/foo/*'] = 'W' + assert role.permissions['/foo/*'] == 'W' + role = self.client.get_role('role') + assert role.permissions['/foo/*'] == 'W' + + # Extend W to RW + role.permissions['/foo/*'] = 'WR' + role = self.client.get_role('role') + assert role.permissions['/foo/*'] == 'RW' + + # NO-OP RW to RW + role.permissions['/foo/*'] = 'RW' + role = self.client.get_role('role') + assert role.permissions['/foo/*'] == 'RW' + + # Reduce RW to W + role.permissions['/foo/*'] = 'W' + role = self.client.get_role('role') + assert role.permissions['/foo/*'] == 'W' + + def test_role_names_empty(self): + assert self.client.role_names == ['root'] + + def test_role_names(self): + self.client.create_role('role') + assert self.client.role_names == ['role', 'root'] + + def test_roles(self): + self.client.create_role('role') + assert len(self.client.roles) == 2 + + def test_enable_auth(self): + # Store a value, lock out guests + self.client.write('/foo', 'bar') + self.client.create_user('root', 'rootpassword') + # Creating role before auth is enabled prevents default permissions + self.client.create_role('guest') + self.client.toggle_auth(True) + + # Now we can't access key: + try: + self.client.get('/foo') + self.fail('Expected exception') + except etcd.EtcdException as e: + assert 'Insufficient credentials' in str(e) + + # But an authenticated client can: + root_client = etcd.Client(port=6001, + username='root', + password='rootpassword') + assert root_client.get('/foo').value == 'bar' + + def test_enable_auth_before_root_created(self): + self.assertRaises(etcd.EtcdException, self.client.toggle_auth, True) From dd38063e371eec384907c8220366fc836bde6a00 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Fri, 27 Nov 2015 18:41:25 +0100 Subject: [PATCH 037/101] Move the boilerplate retry logic to a wrapper, add api_execute_json Since what we have in AuthClient right now is not DRY at all, and that needs to be fixed before it ships in any release. --- src/etcd/client.py | 189 +++++++++++++++++++++++++-------------------- 1 file changed, 104 insertions(+), 85 deletions(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index 5bb69f49..16eb4c8a 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -19,6 +19,7 @@ import json import ssl import dns.resolver +from functools import wraps import etcd try: @@ -772,95 +773,114 @@ def _next_server(self, cause=None): _log.info("Selected new etcd server %s", mach) return mach + def _wrap_request(payload): + @wraps(payload) + def wrapper(self, path, method, params=None, timeout=None): + some_request_failed = False + response = False + + if timeout is None: + timeout = self.read_timeout + + if timeout == 0: + timeout = None + + if not path.startswith('/'): + raise ValueError('Path does not start with /') + + while not response: + try: + response = payload(self, path, method, + params=params, timeout=timeout) + # Check the cluster ID hasn't changed under us. We use + # preload_content=False above so we can read the headers + # before we wait for the content of a watch. + self._check_cluster_id(response) + # Now force the data to be preloaded in order to trigger any + # IO-related errors in this method rather than when we try to + # access it later. + _ = response.data + # urllib3 doesn't wrap all httplib exceptions and earlier versions + # don't wrap socket errors either. + except (urllib3.exceptions.HTTPError, + HTTPException, socket.error) as e: + if (params.get("wait") == "true" and + isinstance(e, + urllib3.exceptions.ReadTimeoutError)): + _log.debug("Watch timed out.") + raise etcd.EtcdWatchTimedOut( + "Watch timed out: %r" % e, + cause=e + ) + _log.error("Request to server %s failed: %r", + self._base_uri, e) + if self._allow_reconnect: + _log.info("Reconnection allowed, looking for another " + "server.") + # _next_server() raises EtcdException if there are no + # machines left to try, breaking out of the loop. + self._base_uri = self._next_server(cause=e) + some_request_failed = True + else: + _log.debug("Reconnection disabled, giving up.") + raise etcd.EtcdConnectionFailed( + "Connection to etcd failed due to %r" % e, + cause=e + ) + except: + _log.exception("Unexpected request failure, re-raising.") + raise + + if some_request_failed: + if not self._use_proxies: + # The cluster may have changed since last invocation + self._machines_cache = self.machines + self._machines_cache.remove(self._base_uri) + return self._handle_server_response(response) + return wrapper + + @_wrap_request def api_execute(self, path, method, params=None, timeout=None): """ Executes the query. """ - - some_request_failed = False - response = False - - if timeout is None: - timeout = self.read_timeout - - if timeout == 0: - timeout = None - - if not path.startswith('/'): - raise ValueError('Path does not start with /') - - while not response: - try: - url = self._base_uri + path - - if (method == self._MGET) or (method == self._MDELETE): - response = self.http.request( - method, - url, - timeout=timeout, - fields=params, - redirect=self.allow_redirect, - headers=self._get_headers(), - preload_content=False) - - elif (method == self._MPUT) or (method == self._MPOST): - response = self.http.request_encode_body( - method, - url, - fields=params, - timeout=timeout, - encode_multipart=False, - redirect=self.allow_redirect, - headers=self._get_headers(), - preload_content=False) - else: + url = self._base_uri + path + + if (method == self._MGET) or (method == self._MDELETE): + return self.http.request( + method, + url, + timeout=timeout, + fields=params, + redirect=self.allow_redirect, + headers=self._get_headers(), + preload_content=False) + + elif (method == self._MPUT) or (method == self._MPOST): + return self.http.request_encode_body( + method, + url, + fields=params, + timeout=timeout, + encode_multipart=False, + redirect=self.allow_redirect, + headers=self._get_headers(), + preload_content=False) + else: raise etcd.EtcdException( 'HTTP method {} not supported'.format(method)) - # Check the cluster ID hasn't changed under us. We use - # preload_content=False above so we can read the headers - # before we wait for the content of a watch. - self._check_cluster_id(response) - # Now force the data to be preloaded in order to trigger any - # IO-related errors in this method rather than when we try to - # access it later. - _ = response.data - # urllib3 doesn't wrap all httplib exceptions and earlier versions - # don't wrap socket errors either. - except (urllib3.exceptions.HTTPError, - HTTPException, - socket.error) as e: - if (params.get("wait") == "true" and - isinstance(e, urllib3.exceptions.ReadTimeoutError)): - _log.debug("Watch timed out.") - raise etcd.EtcdWatchTimedOut( - "Watch timed out: %r" % e, - cause=e - ) - - _log.error("Request to server %s failed: %r", - self._base_uri, e) - if self._allow_reconnect: - _log.info("Reconnection allowed, looking for another " - "server.") - # _next_server() raises EtcdException if there are no - # machines left to try, breaking out of the loop. - self._base_uri = self._next_server(cause=e) - some_request_failed = True - else: - _log.debug("Reconnection disabled, giving up.") - raise etcd.EtcdConnectionFailed( - "Connection to etcd failed due to %r" % e, - cause=e - ) - except: - _log.exception("Unexpected request failure, re-raising.") - raise - - if some_request_failed: - if not self._use_proxies: - # The cluster may have changed since last invocation - self._machines_cache = self.machines - self._machines_cache.remove(self._base_uri) - return self._handle_server_response(response) + @_wrap_request + def api_execute_json(self, path, method, params=None, timeout=None): + url = self._base_uri + path + json_payload = json.dumps(params) + headers = self._get_headers() + headers['Content-Type'] = 'application/json' + return self.http.urlopen(method, + url, + body=json_payload, + timeout=timeout, + redirect=self.allow_redirect, + headers=headers, + preload_content=False) def _check_cluster_id(self, response): cluster_id = response.getheader("x-etcd-cluster-id") @@ -902,4 +922,3 @@ def _get_headers(self): credentials = ':'.join((self.username, self.password)) return urllib3.make_headers(basic_auth=credentials) return {} - From 1857e763de136f296700f07a58524c9f790206ce Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Sat, 28 Nov 2015 16:48:36 +0100 Subject: [PATCH 038/101] Add error handling for ACLs (use and management) Also removed auth.py; in its current form it's wrong and unusable --- src/etcd/__init__.py | 15 + src/etcd/auth.py | 369 ------------------ src/etcd/client.py | 1 + .../tests/integration/test_authentication.py | 195 --------- 4 files changed, 16 insertions(+), 564 deletions(-) delete mode 100644 src/etcd/auth.py delete mode 100644 src/etcd/tests/integration/test_authentication.py diff --git a/src/etcd/__init__.py b/src/etcd/__init__.py index 2032be3a..f52852c0 100644 --- a/src/etcd/__init__.py +++ b/src/etcd/__init__.py @@ -200,6 +200,13 @@ def __init__(self, message=None, payload=None, cause=None): self.cause = cause +class EtcdInsufficientPermissions(EtcdException): + """ + Request failed because of insufficient permissions. + """ + pass + + class EtcdWatchTimedOut(EtcdConnectionFailed): """ A watch timed out without returning a result. @@ -253,6 +260,7 @@ class EtcdError(object): 107: EtcdRootReadOnly, 108: EtcdDirNotEmpty, # 109: Non-public: existing peer addr. + 110: EtcdInsufficientPermissions, 200: EtcdValueError, 201: EtcdValueError, @@ -284,6 +292,13 @@ def handle(cls, payload): message = payload.get("message") cause = payload.get("cause") msg = '{} : {}'.format(message, cause) + status = payload.get("status") + # Some general status handling, as + # not all endpoints return coherent error messages + if status == 404: + error_code = 100 + elif status == 401: + error_code = 110 exc = cls.error_exceptions.get(error_code, EtcdException) if issubclass(exc, EtcdException): raise exc(msg, payload) diff --git a/src/etcd/auth.py b/src/etcd/auth.py deleted file mode 100644 index 0f8a2dd8..00000000 --- a/src/etcd/auth.py +++ /dev/null @@ -1,369 +0,0 @@ -import json - -import logging - -try: - # Python 3 - from http.client import HTTPException -except ImportError: - # Python 2 - from httplib import HTTPException -import socket -import urllib3 - -from .client import Client -import etcd - -_log = logging.getLogger(__name__) - - -class AuthClient(Client): - """ - Extended etcd client that supports authentication primitives added in 2.1. - """ - - def __init__(self, *args, **kwargs): - super(AuthClient, self).__init__(*args, **kwargs) - - def create_user(self, username, password, roles=[], role_action='roles'): - """ - Add a user. - - Args: - username (str): Username to create. - password (str): Password for username. - roles (list): List of roles as strings. - - Returns: - EtcdUser - - Raises: - etcd.EtcdException: If user can't be created. - """ - try: - uri = self.version_prefix + '/auth/users/' + username - params = {'user': username} - if password: - params['password'] = password - if roles: - params[role_action] = roles - - response = self.json_api_execute(uri, self._MPUT, params=params) - res = json.loads(response.data.decode('utf-8')) - return EtcdUser(self, res) - except Exception as e: - _log.error("Failed to create user in %s%s: %r", - self._base_uri, self.version_prefix, e) - raise etcd.EtcdException("Could not create user") - - def get_user(self, username): - """ - Look up a user. - - Args: - username (str): Username to lookup. - - Returns: - EtcdUser - - Raises: - etcd.EtcdException: If user can't be found. - """ - try: - uri = self.version_prefix + '/auth/users/' + username - response = self.api_execute(uri, self._MGET) - res = json.loads(response.data.decode('utf-8')) - return EtcdUser(self, res) - except Exception as e: - _log.error("Failed to fetch user in %s%s: %r", - self._base_uri, self.version_prefix, e) - raise etcd.EtcdException("Could not fetch user") - - @property - def usernames(self): - """List user names.""" - try: - uri = self.version_prefix + '/auth/users' - response = self.api_execute(uri, self._MGET) - res = json.loads(response.data.decode('utf-8')) - return res['users'] - except Exception as e: - _log.error("Failed to list users in %s%s: %r", - self._base_uri, self.version_prefix, e) - raise etcd.EtcdException("Could not list users") - - @property - def users(self): - """List users in detail.""" - return [self.get_user(x) for x in self.usernames] - - def create_role(self, role_name): - """ - Create a role. - - Args: - role_name (str): Name of role - - Returns: - EtcdRole - """ - return self.modify_role(role_name) - - def get_role(self, role_name): - """ - Look up a role. - - Args: - role_name (str): Name of role. - - Returns: - EtcdRole - """ - try: - uri = self.version_prefix + '/auth/roles/' + role_name - response = self.api_execute(uri, self._MGET) - res = json.loads(response.data.decode('utf-8')) - return EtcdRole(self, res) - except Exception as e: - _log.error("Failed to fetch user in %s%s: %r", - self._base_uri, self.version_prefix, e) - raise etcd.EtcdException("Could not fetch users") - - @property - def role_names(self): - """List role names.""" - try: - uri = self.version_prefix + '/auth/roles' - response = self.api_execute(uri, self._MGET) - res = json.loads(response.data.decode('utf-8')) - return res['roles'] - except Exception as e: - _log.error("Failed to list roles in %s%s: %r", - self._base_uri, self.version_prefix, e) - raise etcd.EtcdException("Could not list roles") - - @property - def roles(self): - """List roles in detail.""" - return [self.get_role(x) for x in self.role_names] - - def toggle_auth(self, auth_enabled=True): - """ - Toggle authentication. - - Args: - auth_enabled (bool): Should auth be enabled or disabled - """ - try: - uri = self.version_prefix + '/auth/enable' - action = auth_enabled and self._MPUT or self._MDELETE - - self.api_execute(uri, action) - except Exception as e: - _log.error("Failed enable authentication in %s%s: %r", - self._base_uri, self.version_prefix, e) - raise etcd.EtcdException("Could not toggle authentication") - - def modify_role(self, role_name, permissions=None, perm_key=None): - """Modifies role.""" - try: - uri = self.version_prefix + '/auth/roles/' + role_name - params = { - 'role': role_name, - } - if permissions: - params[perm_key] = { - 'kv': { - 'read': [k for k, v in permissions.items() if - 'R' in v.upper()], - 'write': [k for k, v in permissions.items() if - 'W' in v.upper()] - } - } - response = self.json_api_execute(uri, self._MPUT, params=params) - res = json.loads(response.data.decode('utf-8')) - return EtcdRole(self, res) - except Exception as e: - _log.error("Failed to modify role in %s%s: %r", - self._base_uri, self.version_prefix, e) - raise etcd.EtcdException("Could not modify role") - - def json_api_execute(self, path, method, params=None, timeout=None): - """ Executes the query. """ - - some_request_failed = False - response = False - - if timeout is None: - timeout = self.read_timeout - - if timeout == 0: - timeout = None - - if not path.startswith('/'): - raise ValueError('Path does not start with /') - - while not response: - try: - url = self._base_uri + path - json_payload = json.dumps(params) - headers = self._get_headers() - headers['Content-Type'] = 'application/json' - response = self.http.urlopen(method, - url, - body=json_payload, - timeout=timeout, - redirect=self.allow_redirect, - headers=headers, - preload_content=False) - # urllib3 doesn't wrap all httplib exceptions and earlier versions - # don't wrap socket errors either. - except (urllib3.exceptions.HTTPError, - HTTPException, - socket.error) as e: - _log.error("Request to server %s failed: %r", - self._base_uri, e) - if self._allow_reconnect: - _log.info("Reconnection allowed, looking for another " - "server.") - # _next_server() raises EtcdException if there are no - # machines left to try, breaking out of the loop. - self._base_uri = self._next_server() - some_request_failed = True - else: - _log.debug("Reconnection disabled, giving up.") - raise etcd.EtcdConnectionFailed( - "Connection to etcd failed due to %r" % e) - except: - _log.exception("Unexpected request failure, re-raising.") - raise - - else: - # Check the cluster ID hasn't changed under us. We use - # preload_content=False above so we can read the headers - # before we wait for the content of a long poll. - cluster_id = response.getheader("x-etcd-cluster-id") - id_changed = (self.expected_cluster_id - and cluster_id is not None and - cluster_id != self.expected_cluster_id) - # Update the ID so we only raise the exception once. - old_expected_cluster_id = self.expected_cluster_id - self.expected_cluster_id = cluster_id - if id_changed: - # Defensive: clear the pool so that we connect afresh next - # time. - self.http.clear() - raise etcd.EtcdClusterIdChanged( - 'The UUID of the cluster changed from {} to ' - '{}.'.format(old_expected_cluster_id, cluster_id)) - - if some_request_failed: - if not self._use_proxies: - # The cluster may have changed since last invocation - self._machines_cache = self.machines - self._machines_cache.remove(self._base_uri) - return self._handle_server_response(response) - - -class EtcdUser(object): - def __init__(self, auth_client, json_user): - self.client = auth_client - self.name = json_user.get('user') - self._roles = json_user.get('roles') or [] - - @property - def password(self): - """Empty property for password.""" - return None - - @password.setter - def password(self, new_password): - """Change user's password.""" - self.client.create_user(self.name, new_password) - - @property - def roles(self): - return tuple(self._roles) - - @roles.setter - def roles(self, roles): - existing_roles = set(self._roles) - new_roles = set(roles) - - if existing_roles == new_roles: - _log.debug('User %s already belongs to %s', self.name, self._roles) - return - - to_revoke = existing_roles - new_roles - to_grant = new_roles - existing_roles - - if to_revoke: - self.client.create_user(self.name, None, roles=list(to_revoke), - role_action='revoke') - if to_grant: - self.client.create_user(self.name, None, roles=list(to_grant), - role_action='grant') - self._roles = new_roles - - -class EtcdRole(object): - def __init__(self, auth_client, role_json): - self.client = auth_client - self.name = role_json.get('role') - self.permissions = RolePermissionsDict(self, role_json) - - -class RolePermissionsDict(dict): - _PERMISSIONS = {'R', 'W'} - - def __init__(self, etcd_role, role_json, *args, **kwargs): - super(RolePermissionsDict, self).__init__(*args, **kwargs) - self.role = etcd_role - permissions = role_json.get('permissions') - if permissions and 'kv' in permissions: - self.__add_permissions(permissions, 'read', 'R') - self.__add_permissions(permissions, 'write', 'W') - - def __add_permissions(self, permissions, label, symbol): - if label in permissions['kv'] and permissions['kv'][label]: - for path in permissions['kv'][label]: - existing_perms = dict.get(self, path) - if existing_perms: - dict.__setitem__(self, path, - existing_perms + symbol) - else: - dict.__setitem__(self, path, symbol) - - def __setitem__(self, key, value): - if not value: - raise ValueError('Permissions may only be (R)ead or (W)ite') - perms = set(x.upper() for x in value) - if not perms <= RolePermissionsDict._PERMISSIONS: - raise ValueError('Permissions may only be (R)ead or (W)ite') - - role_name = self.role.name - perm_dict = {key: value} - existing_value = dict.get(self, key) - - if existing_value: - existing_perms = set(x.upper() for x in existing_value) - if perms != existing_perms: - to_grant = perms - existing_perms - to_revoke = existing_perms - perms - - if to_revoke: - perm_dict = {key: ''.join(to_revoke)} - self.role.client.modify_role(role_name, perm_dict, 'revoke') - if to_grant: - perm_dict = {key: ''.join(to_grant)} - self.role.client.modify_role(role_name, perm_dict, 'grant') - else: - _log.debug('Permission %s=%s already granted', key, value) - else: - self.role.client.modify_role(role_name, perm_dict, 'grant') - - dict.__setitem__(self, key, value) - - def __delitem__(self, key): - self.role.client.modify_role(self.role.name, {key: 'RW'}, 'revoke') - dict.__delitem__(self, key) diff --git a/src/etcd/client.py b/src/etcd/client.py index 16eb4c8a..a5c656d9 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -911,6 +911,7 @@ def _handle_server_response(self, response): # throw the appropriate exception try: r = json.loads(resp) + r['status'] = response.status except (TypeError, ValueError): # Bad JSON, make a response locally. r = {"message": "Bad response", diff --git a/src/etcd/tests/integration/test_authentication.py b/src/etcd/tests/integration/test_authentication.py deleted file mode 100644 index 52ba001b..00000000 --- a/src/etcd/tests/integration/test_authentication.py +++ /dev/null @@ -1,195 +0,0 @@ -import unittest -import shutil -import tempfile - -import time - -import etcd -import etcd.auth -from etcd.tests.integration.test_simple import EtcdIntegrationTest -from etcd.tests.integration import helpers - - -class TestAuthentication(unittest.TestCase): - def setUp(self): - # Restart etcd for each test (since some tests will lock others out) - program = EtcdIntegrationTest._get_exe() - self.directory = tempfile.mkdtemp(prefix='python-etcd') - self.processHelper = helpers.EtcdProcessHelper( - self.directory, - proc_name=program, - port_range_start=6001, - internal_port_range_start=8001) - self.processHelper.run(number=1) - self.client = etcd.auth.AuthClient(port=6001) - - # Wait for sync, to avoid: - # "Not capable of accessing auth feature during rolling upgrades." - time.sleep(0.5) - - def tearDown(self): - self.processHelper.stop() - shutil.rmtree(self.directory) - - def test_create_user(self): - user = self.client.create_user('username', 'password') - assert user.name == 'username' - assert len(user.roles) == 0 - - def test_create_user_with_role(self): - user = self.client.create_user('username', 'password', roles=['root']) - assert user.name == 'username' - assert user.roles == ('root',) - - def test_create_user_add_role(self): - user = self.client.create_user('username', 'password') - self.client.create_role('role') - - # Empty to [root] - user.roles = ['root'] - user = self.client.get_user('username') - assert user.roles == ('root',) - - # [root] to [root,role] - user.roles = ['root', 'role'] - user = self.client.get_user('username') - assert user.roles == ('role', 'root') - - # [root,role] to [role] - user.roles = ['role'] - user = self.client.get_user('username') - assert user.roles == ('role',) - - def test_usernames_empty(self): - assert len(self.client.usernames) == 0 - - def test_usernames(self): - self.client.create_user('username', 'password', roles=['root']) - assert self.client.usernames == ['username'] - - def test_users(self): - self.client.create_user('username', 'password', roles=['root']) - users = self.client.users - assert len(users) == 1 - assert users[0].name == 'username' - - def test_get_user(self): - self.client.create_user('username', 'password', roles=['root']) - user = self.client.get_user('username') - assert user.roles == ('root',) - - def test_get_user_not_found(self): - self.assertRaises(etcd.EtcdException, self.client.get_user, 'username') - - def test_set_user_password(self): - self.client.create_user('username', 'password', roles=['root']) - user = self.client.get_user('username') - assert not user.password - user.password = 'new_password' - assert not user.password - - def test_create_role(self): - role = self.client.create_role('role') - assert role.name == 'role' - assert len(role.permissions) == 0 - - def test_grant_role(self): - role = self.client.create_role('role') - - # Read access to keys under /foo - role.permissions['/foo/*'] = 'R' - assert len(role.permissions) == 1 - assert role.permissions['/foo/*'] == 'R' - - # Write access to the key at /foo/bar - role.permissions['/foo/bar'] = 'W' - assert len(role.permissions) == 2 - - # Full access to keys under /pub - role.permissions['/pub/*'] = 'RW' - assert len(role.permissions) == 3 - - # Fresh fetch to bust cache: - role = self.client.get_role('role') - assert len(role.permissions) == 3 - - def test_get_role(self): - role = self.client.create_role('role') - role.permissions['/foo/*'] = 'R' - - role = self.client.get_role('role') - assert len(role.permissions) == 1 - - def test_revoke_role(self): - role = self.client.create_role('role') - role.permissions['/foo/*'] = 'R' - - del role.permissions['/foo/*'] - - role = self.client.get_role('role') - assert len(role.permissions) == 0 - - def test_modify_role_invalid(self): - role = self.client.create_role('role') - self.assertRaises(ValueError, role.permissions.__setitem__, '/foo/*', - '') - - def test_modify_role_permissions(self): - role = self.client.create_role('role') - role.permissions['/foo/*'] = 'R' - - # Replace R with W - role.permissions['/foo/*'] = 'W' - assert role.permissions['/foo/*'] == 'W' - role = self.client.get_role('role') - assert role.permissions['/foo/*'] == 'W' - - # Extend W to RW - role.permissions['/foo/*'] = 'WR' - role = self.client.get_role('role') - assert role.permissions['/foo/*'] == 'RW' - - # NO-OP RW to RW - role.permissions['/foo/*'] = 'RW' - role = self.client.get_role('role') - assert role.permissions['/foo/*'] == 'RW' - - # Reduce RW to W - role.permissions['/foo/*'] = 'W' - role = self.client.get_role('role') - assert role.permissions['/foo/*'] == 'W' - - def test_role_names_empty(self): - assert self.client.role_names == ['root'] - - def test_role_names(self): - self.client.create_role('role') - assert self.client.role_names == ['role', 'root'] - - def test_roles(self): - self.client.create_role('role') - assert len(self.client.roles) == 2 - - def test_enable_auth(self): - # Store a value, lock out guests - self.client.write('/foo', 'bar') - self.client.create_user('root', 'rootpassword') - # Creating role before auth is enabled prevents default permissions - self.client.create_role('guest') - self.client.toggle_auth(True) - - # Now we can't access key: - try: - self.client.get('/foo') - self.fail('Expected exception') - except etcd.EtcdException as e: - assert 'Insufficient credentials' in str(e) - - # But an authenticated client can: - root_client = etcd.Client(port=6001, - username='root', - password='rootpassword') - assert root_client.get('/foo').value == 'bar' - - def test_enable_auth_before_root_created(self): - self.assertRaises(etcd.EtcdException, self.client.toggle_auth, True) From c8f9a159a8b9d9f29ada877c03205cbfc1e81bae Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Sat, 28 Nov 2015 16:56:33 +0100 Subject: [PATCH 039/101] Re-Adding the auth module. This new, reworked version of auth guarantees: - A simple, ORM-like interface, centered on Users and Roles and not on the client - No useless repetition of code - Fixes some shortcomings of the old interface (deleting objects is now possible, more than one ACL is allowed per role(!!!)) - Doesn't write/read without explicit authorization from the user - Better error handling --- .gitignore | 1 - src/etcd/auth.py | 255 ++++++++++++++++++++++ src/etcd/tests/integration/test_simple.py | 3 +- src/etcd/tests/test_auth.py | 161 ++++++++++++++ 4 files changed, 418 insertions(+), 2 deletions(-) create mode 100644 src/etcd/auth.py create mode 100644 src/etcd/tests/test_auth.py diff --git a/.gitignore b/.gitignore index 765321b4..3f90b7fd 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,4 @@ tmp build dist docs -etcd .coverage diff --git a/src/etcd/auth.py b/src/etcd/auth.py new file mode 100644 index 00000000..796772d7 --- /dev/null +++ b/src/etcd/auth.py @@ -0,0 +1,255 @@ +import json + +import logging +import etcd + +_log = logging.getLogger(__name__) + + +class EtcdAuthBase(object): + entity = 'example' + + def __init__(self, client, name): + self.client = client + self.name = name + self.uri = "{}/auth/{}s/{}".format(self.client.version_prefix, + self.entity, self.name) + + @property + def names(self): + key = "{}s".format(self.entity) + uri = "{}/auth/{}".format(self.client.version_prefix, key) + response = self.client.api_execute(uri, self.client._MGET) + return json.loads(response.data.decode('utf-8'))[key] + + def read(self): + try: + response = self.client.api_execute(self.uri, self.client._MGET) + except etcd.EtcdInsufficientPermissions as e: + _log.error("Any action on the authorization requires the root role") + raise + except etcd.EtcdKeyNotFound: + _log.info("%s '%s' not found", self.entity, self.name) + raise + except Exception as e: + _log.error("Failed to fetch %s in %s%s: %r", + self.entity, self.client._base_uri, + self.client.version_prefix, e) + raise etcd.EtcdException( + "Could not fetch {} '{}'".format(self.entity, self.name)) + + self._from_net(response.data) + + def write(self): + try: + r = self.__class__(self.client, self.name) + r.read() + except etcd.EtcdKeyNotFound: + r = None + try: + for payload in self._to_net(r): + response = self.client.api_execute_json(self.uri, + self.client._MPUT, + params=payload) + # This will fail if the response is an error + self._from_net(response.data) + except etcd.EtcdInsufficientPermissions as e: + _log.error("Any action on the authorization requires the root role") + raise + except Exception as e: + _log.error("Failed to write %s '%s'", self.entity, self.name) + # TODO: fine-grained exception handling + raise etcd.EtcdException( + "Could not write {} '{}': {}".format(self.entity, + self.name, e)) + + def delete(self): + try: + _ = self.client.api_execute(self.uri, self.client._MDELETE) + except etcd.EtcdInsufficientPermissions as e: + _log.error("Any action on the authorization requires the root role") + raise + except etcd.EtcdKeyNotFound: + _log.info("%s '%s' not found", self.entity, self.name) + raise + except Exception as e: + _log.error("Failed to delete %s in %s%s: %r", + self.entity, self._base_uri, self.version_prefix, e) + raise etcd.EtcdException( + "Could not delete {} '{}'".format(self.entity, self.name)) + + def _from_net(self, data): + raise NotImplementedError() + + def _to_net(self, old=None): + raise NotImplementedError() + + @classmethod + def new(cls, client, data): + c = cls(client, data[cls.entity]) + c._from_net(data) + return c + + +class EtcdUser(EtcdAuthBase): + """Class to manage in a orm-like way etcd users""" + entity = 'user' + + def __init__(self, client, name): + super(EtcdUser, self).__init__(client, name) + self._roles = set() + self._password = None + + def _from_net(self, data): + d = json.loads(data.decode('utf-8')) + self.roles = d.get('roles', []) + self.name = d.get('user') + + def _to_net(self, prevobj=None): + if prevobj is None: + retval = [{"user": self.name, "password": self._password, + "roles": list(self.roles)}] + else: + retval = [] + if self._password: + retval.append({"user": self.name, "password": self._password}) + to_grant = list(self.roles - prevobj.roles) + to_revoke = list(prevobj.roles - self.roles) + if to_grant: + retval.append({"user": self.name, "grant": to_grant}) + if to_revoke: + retval.append({"user": self.name, "revoke": to_revoke}) + # Let's blank the password now + # Even if the user can't be written we don't want it to leak anymore. + self._password = None + return retval + + @property + def roles(self): + return self._roles + + @roles.setter + def roles(self, val): + self._roles = set(val) + + @property + def password(self): + """Empty property for password.""" + return None + + @password.setter + def password(self, new_password): + """Change user's password.""" + self._password = new_password + + def __str__(self): + return json.dumps(self._to_net()[0]) + + + +class EtcdRole(EtcdAuthBase): + entity = 'role' + + def __init__(self, client, name): + super(EtcdRole, self).__init__(client, name) + self._read_paths = set() + self._write_paths = set() + + def _from_net(self, data): + d = json.loads(data.decode('utf-8')) + self.name = d.get('role') + + try: + kv = d["permissions"]["kv"] + except: + self._read_paths = set() + self._write_paths = set() + return + + self._read_paths = set(kv.get('read', [])) + self._write_paths = set(kv.get('write', [])) + + def _to_net(self, prevobj=None): + retval = [] + if prevobj is None: + retval.append({ + "role": self.name, + "permissions": + { + "kv": + { + "read": list(self._read_paths), + "write": list(self._write_paths) + } + } + }) + else: + to_grant = { + 'read': list(self._read_paths - prevobj._read_paths), + 'write': list(self._write_paths - prevobj._write_paths) + } + to_revoke = { + 'read': list(prevobj._read_paths - self._read_paths), + 'write': list(prevobj._write_paths - self._write_paths) + } + if [path for sublist in to_revoke.values() for path in sublist]: + retval.append({'role': self.name, 'revoke': {'kv': to_revoke}}) + if [path for sublist in to_grant.values() for path in sublist]: + retval.append({'role': self.name, 'grant': {'kv': to_grant}}) + return retval + + def grant(self, path, permission): + if permission.upper().find('R') >= 0: + self._read_paths.add(path) + if permission.upper().find('W') >= 0: + self._write_paths.add(path) + + def revoke(self, path, permission): + if permission.upper().find('R') >= 0 and \ + path in self._read_paths: + self._read_paths.remove(path) + if permission.upper().find('W') >= 0 and \ + path in self._write_paths: + self._write_paths.remove(path) + + @property + def acls(self): + perms = {} + try: + for path in self._read_paths: + perms[path] = 'R' + for path in self._write_paths: + if path in perms: + perms[path] += 'W' + else: + perms[path] = 'W' + except: + pass + return perms + + @acls.setter + def acls(self, acls): + self._read_paths = set() + self._write_paths = set() + for path, permission in acls.items(): + self.grant(path, permission) + + def __str__(self): + return json.dumps({"role": self.name, 'acls': self.acls}) + + +class Auth(object): + def __init__(self, client): + self.client = client + self.uri = "{}/auth/enable".format(self.client.version_prefix) + + @property + def active(self): + resp = self.client.api_execute(self.uri, self.client._MGET) + return json.loads(resp.data.decode('utf-8'))['enabled'] + + @active.setter + def active(self, value): + if value != self.active: + method = value and self.client._MPUT or self.client._MDELETE + self.client.api_execute(self.uri, method) diff --git a/src/etcd/tests/integration/test_simple.py b/src/etcd/tests/integration/test_simple.py index da0954dc..660caa81 100644 --- a/src/etcd/tests/integration/test_simple.py +++ b/src/etcd/tests/integration/test_simple.py @@ -18,6 +18,7 @@ class EtcdIntegrationTest(unittest.TestCase): + cl_size = 3 @classmethod def setUpClass(cls): @@ -28,7 +29,7 @@ def setUpClass(cls): proc_name=program, port_range_start=6001, internal_port_range_start=8001) - cls.processHelper.run(number=3) + cls.processHelper.run(number=cls.cl_size) cls.client = etcd.Client(port=6001) @classmethod diff --git a/src/etcd/tests/test_auth.py b/src/etcd/tests/test_auth.py new file mode 100644 index 00000000..fc6ce705 --- /dev/null +++ b/src/etcd/tests/test_auth.py @@ -0,0 +1,161 @@ +from etcd.tests.integration.test_simple import EtcdIntegrationTest +from etcd import auth +import etcd + + +class TestEtcdAuthBase(EtcdIntegrationTest): + cl_size = 1 + + def setUp(self): + # Sets up the root user, toggles auth + u = auth.EtcdUser(self.client, 'root') + u.password = 'testpass' + u.write() + self.client = etcd.Client(port=6001, username='root', + password='testpass') + self.unauth_client = etcd.Client(port=6001) + a = auth.Auth(self.client) + a.active = True + + def tearDown(self): + u = auth.EtcdUser(self.client, 'test_user') + r = auth.EtcdRole(self.client, 'test_role') + try: + u.delete() + except: + pass + try: + r.delete() + except: + pass + a = auth.Auth(self.client) + a.active = False + + +class EtcdUserTest(TestEtcdAuthBase): + def test_names(self): + u = auth.EtcdUser(self.client, 'test_user') + self.assertEquals(u.names, ['root']) + + def test_read(self): + u = auth.EtcdUser(self.client, 'root') + # Reading an existing user succeeds + try: + u.read() + except Exception: + self.fail("reading the root user raised an exception") + + # roles for said user are fetched + self.assertEquals(u.roles, set(['root'])) + + # The user is correctly rendered out + self.assertEquals(u._to_net(), [{'user': 'root', 'password': None, + 'roles': ['root']}]) + + # An inexistent user raises the appropriate exception + u = auth.EtcdUser(self.client, 'user.does.not.exist') + self.assertRaises(etcd.EtcdKeyNotFound, u.read) + + # Reading with an unauthenticated client raises an exception + u = auth.EtcdUser(self.unauth_client, 'root') + self.assertRaises(etcd.EtcdInsufficientPermissions, u.read) + + # Generic errors are caught + c = etcd.Client(port=9999) + u = auth.EtcdUser(c, 'root') + self.assertRaises(etcd.EtcdException, u.read) + + def test_write_and_delete(self): + # Create an user + u = auth.EtcdUser(self.client, 'test_user') + u.roles.add('guest') + u.roles.add('root') + # directly from my suitcase + u.password = '123456' + try: + u.write() + except: + self.fail("creating a user doesn't work") + # Password gets wiped + self.assertEquals(u.password, None) + u.read() + # Verify we can log in as this user and access the auth (it has the + # root role) + cl = etcd.Client(port=6001, username='test_user', + password='123456') + ul = auth.EtcdUser(cl, 'root') + try: + ul.read() + except etcd.EtcdInsufficientPermissions: + self.fail("Reading auth with the new user is not possible") + + self.assertEquals(u.name, "test_user") + self.assertEquals(u.roles, set(['guest', 'root'])) + # set roles as a list, it works! + u.roles = ['guest', 'test_group'] + try: + u.write() + except: + self.fail("updating a user you previously created fails") + u.read() + self.assertIn('test_group', u.roles) + + # Unauthorized access is properly handled + ua = auth.EtcdUser(self.unauth_client, 'test_user') + self.assertRaises(etcd.EtcdInsufficientPermissions, ua.write) + + # now let's test deletion + du = auth.EtcdUser(self.client, 'user.does.not.exist') + self.assertRaises(etcd.EtcdKeyNotFound, du.delete) + + # Delete test_user + u.delete() + self.assertRaises(etcd.EtcdKeyNotFound, u.read) + # Permissions are properly handled + self.assertRaises(etcd.EtcdInsufficientPermissions, ua.delete) + + +class EtcdRoleTest(TestEtcdAuthBase): + def test_names(self): + r = auth.EtcdRole(self.client, 'guest') + self.assertListEqual(r.names, [u'guest', u'root']) + + def test_read(self): + r = auth.EtcdRole(self.client, 'guest') + try: + r.read() + except: + self.fail('Reading an existing role failed') + + self.assertEquals(r.acls, {'*': 'RW'}) + # We can actually skip most other read tests as they are common + # with EtcdUser + + def test_write_and_delete(self): + r = auth.EtcdRole(self.client, 'test_role') + r.acls = {'*': 'R', '/test/*': 'RW'} + try: + r.write() + except: + self.fail("Writing a simple groups should not fail") + + r1 = auth.EtcdRole(self.client, 'test_role') + r1.read() + self.assertEquals(r1.acls, r.acls) + r.revoke('/test/*', 'W') + r.write() + r1.read() + self.assertEquals(r1.acls, {'*': 'R', '/test/*': 'R'}) + r.grant('/pub/*', 'RW') + r.write() + r1.read() + self.assertEquals(r1.acls['/pub/*'], 'RW') + # All other exceptions are tested by the user tests + r1.name = None + self.assertRaises(etcd.EtcdException, r1.write) + # ditto for delete + try: + r.delete() + except: + self.fail("A normal delete should not fail") + self.assertRaises(etcd.EtcdKeyNotFound, r.read) From 31f57fa0143777240371c0a5f85a920ca6347807 Mon Sep 17 00:00:00 2001 From: Bartlomiej Biernacki Date: Mon, 30 Nov 2015 10:29:28 +0100 Subject: [PATCH 040/101] Make response False on exception When exception will be raised on _ = response.data we will handle exception, but as response is not None the while loop won't be repeated. This may lead to an error on _handle_server_response as we may get a response with status == 200 and empty data. --- src/etcd/client.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/etcd/client.py b/src/etcd/client.py index a5c656d9..de898b6f 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -821,6 +821,11 @@ def wrapper(self, path, method, params=None, timeout=None): # machines left to try, breaking out of the loop. self._base_uri = self._next_server(cause=e) some_request_failed = True + + # if exception is raised on _ = response.data + # the condition for while loop will be False + # but we should retry + response = False else: _log.debug("Reconnection disabled, giving up.") raise etcd.EtcdConnectionFailed( From 469f29f4bfc39cbb31c786311500e3c81717cc16 Mon Sep 17 00:00:00 2001 From: Nick Bartos Date: Tue, 24 Nov 2015 12:34:13 -0800 Subject: [PATCH 041/101] Cluster ID change shouldn't log a traceback. Having the Cluster ID shouldn't log a traceback, because: 1. It's not a coding error. 2. Having a trace isn't really helpful in this case. --- src/etcd/client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/etcd/client.py b/src/etcd/client.py index de898b6f..d01524a0 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -832,6 +832,9 @@ def wrapper(self, path, method, params=None, timeout=None): "Connection to etcd failed due to %r" % e, cause=e ) + except etcd.EtcdClusterIdChanged as e: + _log.warning(e) + raise except: _log.exception("Unexpected request failure, re-raising.") raise From 0129a43e1abe66b2844e97443e8c1ee9f36527a6 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Wed, 2 Dec 2015 18:18:24 +0100 Subject: [PATCH 042/101] Release 0.4.3 Enough changes were important enough to grant a release, namely python 3.5 compatibility and authentication/ACLs. Also a ton of fixes that people would probably love to have. Added an AUTHORS file to acknowledge openly the work of all the contributors to the project. --- AUTHORS | 37 +++++++++++++++++++++++++++++++++++++ NEWS.txt | 14 +++++++++++++- docs-source/conf.py | 4 ++-- setup.py | 2 +- 4 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 AUTHORS diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 00000000..0f07ed1e --- /dev/null +++ b/AUTHORS @@ -0,0 +1,37 @@ +Maintainers: +----------- +Jose Plana (jplana) +Giuseppe Lavagetto (lavagetto) + +Contributors: +------------ +Aleksandar Veselinovic +Alex Chan +Alex Ianchici +Bartlomiej Biernacki +Bradley Cicenas +Christoph Heer +Hogenmiller +Jimmy Zelinskie +Jim Rollenhagen +John Kristensen +Joshua Conner +Matthias Urlichs +Michal Witkowski +Nick Bartos +Peter Wagner +Roberto Aguilar +Roy Smith +Ryan Fowler +Samuel Marks +Sergio Castaño Arteaga +Shaun Crampton +Sigmund Augdal +Simeon Visser +Simon Gomizelj +SkyLothar +Spike Curtis +Tomas Kral +Tom Denham +WillPlatnick +WooParadog diff --git a/NEWS.txt b/NEWS.txt index 2158c21f..00b6a715 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -1,9 +1,21 @@ News ==== +0.4.3 +----- +*Release date: 3-Dec-2015* + +* Python 3.5 compatibility and general python3 cleanups +* Added authentication and module for managing ACLs +* Added srv record-based DNS discovery +* Fixed (again) logging of cluster id changes +* Fixed leader lookup +* Properly retry request on exception +* Client: clean up open connections when deleting + 0.4.2 ----- -*Release data: 8-Oct-2015* +*Release date: 8-Oct-2015* * Fixed lock documentation * Fixed lock sequences due to etcd 2.2 change diff --git a/docs-source/conf.py b/docs-source/conf.py index 996cb61b..5148c23a 100644 --- a/docs-source/conf.py +++ b/docs-source/conf.py @@ -52,7 +52,7 @@ def __getattr__(cls, name): # General information about the project. project = u'python-etcd' -copyright = u'2013, Jose Plana' +copyright = u'2013-2015 Jose Plana, Giuseppe Lavagetto' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -61,7 +61,7 @@ def __getattr__(cls, name): # The short X.Y version. version = '0.4' # The full version, including alpha/beta/rc tags. -release = '0.4.2' +release = '0.4.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 3d0d4509..5387ef4d 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ NEWS = open(os.path.join(here, 'NEWS.txt')).read() -version = '0.4.2' +version = '0.4.3' # Dnspython is two different packages depending on python version if sys.version_info.major == 2: From 19c5be70816818170089e1c04e79e61f539f17df Mon Sep 17 00:00:00 2001 From: Matthias Urlichs Date: Fri, 11 Dec 2015 23:41:28 +0100 Subject: [PATCH 043/101] Test fix --- src/etcd/tests/integration/test_simple.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/etcd/tests/integration/test_simple.py b/src/etcd/tests/integration/test_simple.py index 660caa81..4baeadeb 100644 --- a/src/etcd/tests/integration/test_simple.py +++ b/src/etcd/tests/integration/test_simple.py @@ -68,7 +68,8 @@ def test_machines(self): def test_leader(self): """ INTEGRATION: retrieve leader """ - self.assertEquals(self.client.leader['clientURLs'], ['http://127.0.0.1:6001']) + self.assertIn(self.client.leader['clientURLs'][0], + ['http://127.0.0.1:6001','http://127.0.0.1:6002','http://127.0.0.1:6003']) def test_get_set_delete(self): """ INTEGRATION: set a new value """ From 2fe2f2f48dfd1b74878737085c31e19432f3c752 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Mon, 14 Dec 2015 12:42:42 +0100 Subject: [PATCH 044/101] Fix check for parameters in case of connection error. If any direct call to api_execute was made, and a connection error occurred, this would result in an error because the params would be None. --- src/etcd/client.py | 7 ++-- src/etcd/tests/unit/test_request.py | 54 +++++++++++++++++++---------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index d01524a0..afeabe8a 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -804,9 +804,10 @@ def wrapper(self, path, method, params=None, timeout=None): # don't wrap socket errors either. except (urllib3.exceptions.HTTPError, HTTPException, socket.error) as e: - if (params.get("wait") == "true" and - isinstance(e, - urllib3.exceptions.ReadTimeoutError)): + if (isinstance(params, dict) and + params.get("wait") == "true" and + isinstance(e, + urllib3.exceptions.ReadTimeoutError)): _log.debug("Watch timed out.") raise etcd.EtcdWatchTimedOut( "Watch timed out: %r" % e, diff --git a/src/etcd/tests/unit/test_request.py b/src/etcd/tests/unit/test_request.py index 2456ae1d..0942523f 100644 --- a/src/etcd/tests/unit/test_request.py +++ b/src/etcd/tests/unit/test_request.py @@ -1,3 +1,4 @@ +import socket import urllib3 import etcd @@ -68,7 +69,8 @@ def test_write_no_params(self): self._mock_api(200, d) self.client.write('/newdir', None, dir=True) self.assertEquals(self.client.api_execute.call_args, - (('/v2/keys/newdir', 'PUT'), dict(params={'dir': 'true'}))) + (('/v2/keys/newdir', 'PUT'), + dict(params={'dir': 'true'}))) class TestClientApiInterface(TestClientApiBase): @@ -89,7 +91,9 @@ def test_machines(self, mocker): @mock.patch('etcd.Client.machines', new_callable=mock.PropertyMock) def test_use_proxies(self, mocker): """Do not overwrite the machines cache when using proxies""" - mocker.return_value = ['https://10.0.0.2:4001', 'https://10.0.0.3:4001', 'https://10.0.0.4:4001'] + mocker.return_value = ['https://10.0.0.2:4001', + 'https://10.0.0.3:4001', + 'https://10.0.0.4:4001'] c = etcd.Client( host=(('localhost', 4001), ('localproxy', 4001)), protocol='https', @@ -99,17 +103,16 @@ def test_use_proxies(self, mocker): self.assertEquals(c._machines_cache, ['https://localproxy:4001']) self.assertEquals(c._base_uri, 'https://localhost:4001') - self.assertNotIn(c.base_uri,c._machines_cache) + self.assertNotIn(c.base_uri, c._machines_cache) c = etcd.Client( - host=(('localhost', 4001), ('10.0.0.2',4001)), + host=(('localhost', 4001), ('10.0.0.2', 4001)), protocol='https', allow_reconnect=True, use_proxies=False ) self.assertIn('https://10.0.0.3:4001', c._machines_cache) - self.assertNotIn(c.base_uri,c._machines_cache) - + self.assertNotIn(c.base_uri, c._machines_cache) def test_members(self): """ Can request machines """ @@ -453,20 +456,32 @@ def test_api_method_not_supported(self): def test_read_cluster_id_changed(self): """ Read timeout set to the default """ - d = {u'action': u'set', - u'node': { + d = { + u'action': u'set', + u'node': { u'expiration': u'2013-09-14T00:56:59.316195568+02:00', u'modifiedIndex': 6, u'key': u'/testkey', u'ttl': 19, - u'value': u'test' - } - } + u'value': u'test', + } + } self._mock_api(200, d, cluster_id="notabcd1234") self.assertRaises(etcd.EtcdClusterIdChanged, self.client.read, '/testkey') self.client.read("/testkey") + def test_read_connection_error(self): + self.client.http.request = mock.create_autospec( + self.client.http.request, + side_effect=socket.error() + ) + self.assertRaises(etcd.EtcdConnectionFailed, + self.client.read, '/something') + # Direct GET request + self.assertRaises(etcd.EtcdConnectionFailed, + self.client.api_execute, '/a', 'GET') + def test_not_in(self): pass @@ -475,22 +490,23 @@ def test_in(self): def test_update_fails(self): """ Non-atomic updates fail """ - d = {u'action': u'set', - u'node': { + d = { + u'action': u'set', + u'node': { u'expiration': u'2013-09-14T00:56:59.316195568+02:00', u'modifiedIndex': 6, u'key': u'/testkey', u'ttl': 19, u'value': u'test' - } - } + } + } res = etcd.EtcdResult(**d) error = { - "errorCode":101, - "message":"Compare failed", - "cause":"[ != bar] [7 != 6]", - "index":6} + "errorCode": 101, + "message": "Compare failed", + "cause": "[ != bar] [7 != 6]", + "index": 6} self._mock_api(412, error) res.value = 'bar' self.assertRaises(ValueError, self.client.update, res) From d29eabf0d798aadfcf7cb881d3f9ad403c99542b Mon Sep 17 00:00:00 2001 From: Jose Plana Date: Mon, 14 Dec 2015 21:48:45 +0100 Subject: [PATCH 045/101] Include latest fix to the 0.4.3 release --- NEWS.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS.txt b/NEWS.txt index 00b6a715..52e7c4bf 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -3,8 +3,9 @@ News 0.4.3 ----- -*Release date: 3-Dec-2015* +*Release date: 14-Dec-2015* +* Fix check for parameters in case of connection error * Python 3.5 compatibility and general python3 cleanups * Added authentication and module for managing ACLs * Added srv record-based DNS discovery From 40fba2a6db42c82567a4ffa43a144e3699378c56 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 16 Dec 2015 08:37:36 -0700 Subject: [PATCH 046/101] Fix build error on Python 2.6 In Python 2.6, sys.version_info does not return a named tuple, resulting in a stacktrace. Tested on Python 2.6, 2.7 and 3.x --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5387ef4d..52892d7e 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ version = '0.4.3' # Dnspython is two different packages depending on python version -if sys.version_info.major == 2: +if sys.version_info[0] == 2: dns = 'dnspython' else: dns = 'dnspython3' From 3a932de3c6c7a0e00f10ad29630ab58e98cf05fc Mon Sep 17 00:00:00 2001 From: Toshiya Kawasaki Date: Thu, 3 Mar 2016 14:50:40 +0900 Subject: [PATCH 047/101] add explanation about read error --- README.rst | 8 ++++++++ docs-source/index.rst | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/README.rst b/README.rst index c9d6dabc..62a38a46 100644 --- a/README.rst +++ b/README.rst @@ -67,6 +67,14 @@ Read a key client.read('/nodes', recursive = True) #get all the values of a directory, recursively. client.get('/nodes/n2').value + # raises etcd.EtcdKeyNotFound when key not found + try: + client.read('/invalid/path') + except etcd.EtcdKeyNotFound: + # do something + print "error" + + Delete a key ~~~~~~~~~~~~ diff --git a/docs-source/index.rst b/docs-source/index.rst index 05851d78..0f4db39c 100644 --- a/docs-source/index.rst +++ b/docs-source/index.rst @@ -88,6 +88,12 @@ Get a key client.read('/nodes/n2', wait=True) #Waits for a change in value in the key before returning. client.read('/nodes/n2', wait=True, waitIndex=10) + # raises etcd.EtcdKeyNotFound when key not found + try: + client.read('/invalid/path') + except etcd.EtcdKeyNotFound: + # do something + print "error" Delete a key From f2e1531ee8aba9002dcbefe39678a2f872e53c62 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Sat, 16 Apr 2016 16:15:13 +0200 Subject: [PATCH 048/101] Fix part of the readthedocs documentation Closes #162 --- docs-source/index.rst | 50 +++++++++++++------------------------------ 1 file changed, 15 insertions(+), 35 deletions(-) diff --git a/docs-source/index.rst b/docs-source/index.rst index 0f4db39c..181c0b5c 100644 --- a/docs-source/index.rst +++ b/docs-source/index.rst @@ -57,7 +57,7 @@ Set a key client.write('/nodes/n3', 'test2', prevValue='test1') #this fails to write client.write('/nodes/n3', 'test2', prevIndex=10) #this fails to write # mkdir - client.write('/nodes/queue', dir=True) + client.write('/nodes/queue', None, dir=True) # Append a value to a queue dir client.write('/nodes/queue', 'test', append=True) #will write i.e. /nodes/queue/11 client.write('/nodes/queue', 'test2', append=True) #will write i.e. /nodes/queue/12 @@ -105,51 +105,31 @@ Delete a key client.delete('/nodes', dir=True) #spits an error if dir is not empty client.delete('/nodes', recursive=True) #this works recursively +Locking module +~~~~~~~~~~~~~~ - - -Use lock primitives -................... - -.. code-block:: python +.. code:: python # Initialize the lock object: # NOTE: this does not acquire a lock yet client = etcd.Client() - lock = client.get_lock('/customer1', ttl=60) + lock = etcd.Lock(client, 'my_lock_name') # Use the lock object: - lock.acquire() - lock.is_locked() # True - lock.renew(60) - lock.release() - lock.is_locked() # False + lock.acquire(blocking=True, # will block until the lock is acquired + lock_ttl=None) # lock will live until we release it + lock.is_acquired() # + lock.acquire(lock_ttl=60) # renew a lock + lock.release() # release an existing lock + lock.is_acquired() # False # The lock object may also be used as a context manager: client = etcd.Client() - lock = client.get_lock('/customer1', ttl=60) - with lock as my_lock: + with etcd.Lock(client, 'customer1') as my_lock: do_stuff() - lock.is_locked() # True - lock.renew(60) - lock.is_locked() # False - -Use the leader election primitives -.................................. - -.. code-block:: python - - # Set a leader object with a name; if no name is given, the local hostname - # is used. - # Zero or no ttl means the leader object is persistent. - client = etcd.Client() - client.election.set('/mysql', name='foo.example.com', ttl=120) # returns the etcd index - - # Get the name - print(client.election.get('/mysql')) # 'foo.example.com' - # Delete it! - print(client.election.delete('/mysql', name='foo.example.com')) - + my_lock.is_acquired() # True + my_lock.acquire(lock_ttl = 60) + my_lock.is_acquired() # False Get machines in the cluster From 319b9fa80863c9d185c5a6f3460b51b65fe74eb4 Mon Sep 17 00:00:00 2001 From: tmacro Date: Sat, 16 Apr 2016 16:03:14 -0500 Subject: [PATCH 049/101] Python3 fix when blocking on contented lock, episode 2 Same issue as [PR#126](https://github.com/jplana/python-etcd/pull/126) timout = None can cascade into the call to self._aquired triggering: TypeError: unorderable types: NoneType() > int() --- src/etcd/lock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etcd/lock.py b/src/etcd/lock.py index 687a548f..44fd3a0a 100644 --- a/src/etcd/lock.py +++ b/src/etcd/lock.py @@ -54,7 +54,7 @@ def is_acquired(self): self.is_taken = False return False - def acquire(self, blocking=True, lock_ttl=3600, timeout=None): + def acquire(self, blocking=True, lock_ttl=3600, timeout=0): """ Acquire the lock. From 0e5ba0ccb4e6ce9624a24e4d5d842b174e1c3a6c Mon Sep 17 00:00:00 2001 From: boyxuper Date: Sat, 7 May 2016 18:20:48 +0800 Subject: [PATCH 050/101] PEP8 fixes --- src/etcd/client.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index afeabe8a..1936e296 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -397,10 +397,9 @@ def _sanitize_key(self, key): key = "/{}".format(key) return key - def write(self, key, value, ttl=None, dir=False, append=False, **kwdargs): """ - Writes the value for a key, possibly doing atomit Compare-and-Swap + Writes the value for a key, possibly doing atomic Compare-and-Swap Args: key (str): Key. @@ -430,7 +429,7 @@ def write(self, key, value, ttl=None, dir=False, append=False, **kwdargs): """ _log.debug("Writing %s to key %s ttl=%s dir=%s append=%s", - value, key, ttl, dir, append) + value, key, ttl, dir, append) key = self._sanitize_key(key) params = {} if value is not None: From ddc7db0055d27eb5c356b2ca1912de3532868569 Mon Sep 17 00:00:00 2001 From: boyxuper Date: Sat, 7 May 2016 18:33:29 +0800 Subject: [PATCH 051/101] add refresh key method --- src/etcd/client.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index 1936e296..04e200bf 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -41,7 +41,7 @@ class Client(object): _MPUT = 'PUT' _MPOST = 'POST' _MDELETE = 'DELETE' - _comparison_conditions = set(('prevValue', 'prevIndex', 'prevExist')) + _comparison_conditions = set(('prevValue', 'prevIndex', 'prevExist', 'refresh')) _read_options = set(('recursive', 'wait', 'waitIndex', 'sorted', 'quorum')) _del_conditions = set(('prevValue', 'prevIndex')) @@ -421,6 +421,8 @@ def write(self, key, value, ttl=None, dir=False, append=False, **kwdargs): prevExist (bool): If false, only create key; if true, only update key. + refresh (bool): since 2.3.0, If true, only update the ttl, prev key must existed(prevExist=True). + Returns: client.EtcdResult @@ -460,6 +462,28 @@ def write(self, key, value, ttl=None, dir=False, append=False, **kwdargs): response = self.api_execute(path, method, params=params) return self._result_from_response(response) + def refresh(self, key, ttl, **kwdargs): + """ + (Since 2.3.0) Refresh the ttl of a key without notifying watchers. + + Keys in etcd can be refreshed without notifying watchers, + this can be achieved by setting the refresh to true when updating a TTL + + You cannot update the value of a key when refreshing it + + @see: https://github.com/coreos/etcd/blob/release-2.3/Documentation/api.md#refreshing-key-ttl + + Args: + key (str): Key. + + ttl (int): Time in seconds of expiration (optional). + + Other parameters modifying the write method are accepted as `EtcdClient.write`. + """ + # overwrite kwdargs' prevExist + kwdargs['prevExist'] = True + return self.write(key=key, value=None, ttl=ttl, refresh=True, **kwdargs) + def update(self, obj): """ Updates the value for a key atomically. Typical usage would be: From 47cbbd5d38b9767b14a5bdbad2cb188c1177c1ce Mon Sep 17 00:00:00 2001 From: tmacro Date: Sat, 14 May 2016 00:03:12 -0500 Subject: [PATCH 052/101] update dnspython dependency remove the requirement for dnspython2 and dnspython3 as the two branches have been merged as of dnspython 1.13.0 --- setup.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 52892d7e..f1ab7cb4 100644 --- a/setup.py +++ b/setup.py @@ -8,15 +8,9 @@ version = '0.4.3' -# Dnspython is two different packages depending on python version -if sys.version_info[0] == 2: - dns = 'dnspython' -else: - dns = 'dnspython3' - install_requires = [ 'urllib3>=1.7.1', - dns + 'dnspython>=1.13.0' ] test_requires = [ From 2b56338a42797f7c9d95ba54e3124ba37979c25d Mon Sep 17 00:00:00 2001 From: boyxuper Date: Mon, 30 May 2016 14:47:33 +0800 Subject: [PATCH 053/101] add refresh docs --- README.rst | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 62a38a46..1550a92f 100644 --- a/README.rst +++ b/README.rst @@ -48,8 +48,9 @@ Create a client object client = etcd.Client(srv_domain='example.com', protocol="https") # create a client against https://api.example.com:443/etcd client = etcd.Client(host='api.example.com', protocol='https', port=443, version_prefix='/etcd') + Write a key -~~~~~~~~~ +~~~~~~~~~~~ .. code:: python @@ -59,7 +60,7 @@ Write a key client.set('/nodes/n2', 1) # Equivalent, for compatibility reasons. Read a key -~~~~~~~~~ +~~~~~~~~~~ .. code:: python @@ -83,7 +84,7 @@ Delete a key client.delete('/nodes/n1') Atomic Compare and Swap -~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~ .. code:: python @@ -113,6 +114,20 @@ Watch a key client.watch('/nodes/n1') #equivalent to client.read('/nodes/n1', wait = True) client.watch('/nodes/n1', index = 10) +Refreshing key TTL +~~~~~~~~~~~~~~~~~~ + +(Since etcd 2.3.0) Keys in etcd can be refreshed without notifying current watchers. + +This can be achieved by setting the refresh to true when updating a TTL. + +You cannot update the value of a key when refreshing it. + +.. code:: python + + client.write('/nodes/n1', 'value', ttl=30) # sets the ttl to 30 seconds + client.refresh('/nodes/n1', ttl=600) # refresh ttl to 600 seconds, without notifying current watchers + Locking module ~~~~~~~~~~~~~~ @@ -155,7 +170,7 @@ Get leader of the cluster client.leader Generate a sequential key in a directory -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code:: python From db584033eff7f877db9273a53a09397ec2e52459 Mon Sep 17 00:00:00 2001 From: boyxuper Date: Tue, 31 May 2016 16:45:09 +0800 Subject: [PATCH 054/101] add test for refresh method --- src/etcd/tests/unit/test_request.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/etcd/tests/unit/test_request.py b/src/etcd/tests/unit/test_request.py index 0942523f..b972a8c5 100644 --- a/src/etcd/tests/unit/test_request.py +++ b/src/etcd/tests/unit/test_request.py @@ -217,7 +217,22 @@ def test_newkey(self): d['node']['newKey'] = True self.assertEquals(res, etcd.EtcdResult(**d)) + def test_refresh(self): + """ Can refresh a new value """ + d = { + u'action': u'update', + u'node': { + u'expiration': u'2016-05-31T08:27:54.660337Z', + u'modifiedIndex': 183, + u'key': u'/testkey', + u'ttl': 600, + u'value': u'test' + } + } + self._mock_api(200, d) + res = self.client.refresh('/testkey', ttl=600) + self.assertEquals(res, etcd.EtcdResult(**d)) def test_not_found_response(self): """ Can handle server not found response """ From a489971967c155f93b51f4570cb360fedc8dc0af Mon Sep 17 00:00:00 2001 From: realityone Date: Wed, 15 Jun 2016 11:53:53 +0800 Subject: [PATCH 055/101] property setter's method name must be matched with decorator --- src/etcd/lock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etcd/lock.py b/src/etcd/lock.py index 44fd3a0a..9fe1c8c6 100644 --- a/src/etcd/lock.py +++ b/src/etcd/lock.py @@ -30,7 +30,7 @@ def uuid(self): return self._uuid @uuid.setter - def set_uuid(self, value): + def uuid(self, value): old_uuid = self._uuid self._uuid = value if not self._find_lock(): From d2f3e09e0fd7b4a4629eb43a3f9afae256e451eb Mon Sep 17 00:00:00 2001 From: MingQing Lee Date: Mon, 18 Jul 2016 21:33:12 +0800 Subject: [PATCH 056/101] Add custom lock prefix support --- README.rst | 10 ++++++---- src/etcd/client.py | 11 ++++++++++- src/etcd/lock.py | 2 +- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 62a38a46..f855d0ee 100644 --- a/README.rst +++ b/README.rst @@ -121,23 +121,25 @@ Locking module # Initialize the lock object: # NOTE: this does not acquire a lock yet client = etcd.Client() + # Or you can custom lock prefix, default is '/_locks/' + client = etcd.Client(lock_prefix='/my_etcd_root/_locks') lock = etcd.Lock(client, 'my_lock_name') # Use the lock object: lock.acquire(blocking=True, # will block until the lock is acquired lock_ttl=None) # lock will live until we release it - lock.is_acquired() # + lock.is_acquired # True lock.acquire(lock_ttl=60) # renew a lock lock.release() # release an existing lock - lock.is_acquired() # False + lock.is_acquired # False # The lock object may also be used as a context manager: client = etcd.Client() with etcd.Lock(client, 'customer1') as my_lock: do_stuff() - my_lock.is_acquired() # True + my_lock.is_acquired # True my_lock.acquire(lock_ttl = 60) - my_lock.is_acquired() # False + my_lock.is_acquired # False Get machines in the cluster diff --git a/src/etcd/client.py b/src/etcd/client.py index 04e200bf..f0ef440d 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -63,7 +63,8 @@ def __init__( allow_reconnect=False, use_proxies=False, expected_cluster_id=None, - per_host_pool_size=10 + per_host_pool_size=10, + lock_prefix="/_locks" ): """ Initialize the client. @@ -111,6 +112,8 @@ def __init__( per_host_pool_size (int): specifies maximum number of connections to pool by host. By default this will use up to 10 connections. + lock_prefix (str): Set the key prefix at etcd when client to lock object. + By default this will be use /_locks. """ # If a DNS record is provided, use it to get the hosts list @@ -143,6 +146,7 @@ def uri(protocol, host, port): self._allow_redirect = allow_redirect self._use_proxies = use_proxies self._allow_reconnect = allow_reconnect + self._lock_prefix = lock_prefix # SSL Client certificate support @@ -258,6 +262,11 @@ def allow_redirect(self): """Allow the client to connect to other nodes.""" return self._allow_redirect + @property + def lock_prefix(self): + """Get the key prefix at etcd when client to lock object.""" + return self._lock_prefix + @property def machines(self): """ diff --git a/src/etcd/lock.py b/src/etcd/lock.py index 44fd3a0a..8d1eec78 100644 --- a/src/etcd/lock.py +++ b/src/etcd/lock.py @@ -17,7 +17,7 @@ def __init__(self, client, lock_name): # prevent us from getting back the full path name. We prefix our # lock name with a uuid and can check for its presence on retry. self._uuid = uuid.uuid4().hex - self.path = "/_locks/{}".format(lock_name) + self.path = "{}/{}".format(client.lock_prefix, lock_name) self.is_taken = False self._sequence = None _log.debug("Initiating lock for %s with uuid %s", self.path, self._uuid) From 1a5fb5f6e981dadf85e3c529793e871a15d6649e Mon Sep 17 00:00:00 2001 From: Steve Milner Date: Fri, 12 Aug 2016 14:21:25 -0400 Subject: [PATCH 057/101] doc: Fix lock documentation. --- README.rst | 4 ++-- docs-source/index.rst | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index f855d0ee..038160a0 100644 --- a/README.rst +++ b/README.rst @@ -121,7 +121,7 @@ Locking module # Initialize the lock object: # NOTE: this does not acquire a lock yet client = etcd.Client() - # Or you can custom lock prefix, default is '/_locks/' + # Or you can custom lock prefix, default is '/_locks/' if you are using HEAD client = etcd.Client(lock_prefix='/my_etcd_root/_locks') lock = etcd.Lock(client, 'my_lock_name') @@ -138,7 +138,7 @@ Locking module with etcd.Lock(client, 'customer1') as my_lock: do_stuff() my_lock.is_acquired # True - my_lock.acquire(lock_ttl = 60) + my_lock.acquire(lock_ttl=60) my_lock.is_acquired # False diff --git a/docs-source/index.rst b/docs-source/index.rst index 181c0b5c..e35d6857 100644 --- a/docs-source/index.rst +++ b/docs-source/index.rst @@ -117,19 +117,19 @@ Locking module # Use the lock object: lock.acquire(blocking=True, # will block until the lock is acquired - lock_ttl=None) # lock will live until we release it - lock.is_acquired() # - lock.acquire(lock_ttl=60) # renew a lock - lock.release() # release an existing lock - lock.is_acquired() # False + lock_ttl=None) # lock will live until we release it + lock.is_acquired # True + lock.acquire(lock_ttl=60) # renew a lock + lock.release() # release an existing lock + lock.is_acquired # False # The lock object may also be used as a context manager: client = etcd.Client() with etcd.Lock(client, 'customer1') as my_lock: do_stuff() - my_lock.is_acquired() # True - my_lock.acquire(lock_ttl = 60) - my_lock.is_acquired() # False + my_lock.is_acquired # True + my_lock.acquire(lock_ttl=60) + my_lock.is_acquired # False Get machines in the cluster From 8aa3a12bb9dc87ab757393ff58469ead0cb564bf Mon Sep 17 00:00:00 2001 From: Chet Nichols III Date: Wed, 17 Aug 2016 11:04:56 -0700 Subject: [PATCH 058/101] tweak pre-requirements wording The way this read made it sound like I actually had to install etcd client-side, as if the library wrapped etcdctl commands, so I actually installed it. ..but then, after looking at the source, and seeing log output, saw it does in fact use the API (phew)! --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index f855d0ee..74055151 100644 --- a/README.rst +++ b/README.rst @@ -17,7 +17,7 @@ Installation Pre-requirements ~~~~~~~~~~~~~~~~ -Install etcd (2.0.1 or later). This version of python-etcd will only work correctly with the etcd version 2.0.x or later. If you are running an older version of etcd, please use python-etcd 0.3.3 or earlier. +This version of python-etcd will only work correctly with the etcd server version 2.0.x or later. If you are running an older version of etcd, please use python-etcd 0.3.3 or earlier. This client is known to work with python 2.7 and with python 3.3 or above. It is not tested or expected to work in more outdated versions of python. From 80f936963773053c36ebdff5185147b5e127f94d Mon Sep 17 00:00:00 2001 From: Alexander Brand Date: Tue, 9 Aug 2016 10:26:48 -0400 Subject: [PATCH 059/101] Don't set TLSv1 protocol in the client --- src/etcd/client.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index f0ef440d..a71da5ae 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -157,12 +157,6 @@ def uri(protocol, host, port): if self._read_timeout > 0: kw['timeout'] = self._read_timeout - if protocol == 'https': - # If we don't allow TLSv1, clients using older version of OpenSSL - # (<1.0) won't be able to connect. - _log.debug("HTTPS enabled.") - kw['ssl_version'] = ssl.PROTOCOL_TLSv1 - if cert: if isinstance(cert, tuple): # Key and cert are separate From 81196698bd39e43f6f3d8049505f76a155b4ae36 Mon Sep 17 00:00:00 2001 From: Steve Milner Date: Wed, 16 Nov 2016 16:15:04 -0500 Subject: [PATCH 060/101] Ported @mbarnes auth test fix for acls --- src/etcd/tests/test_auth.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/etcd/tests/test_auth.py b/src/etcd/tests/test_auth.py index fc6ce705..14475f91 100644 --- a/src/etcd/tests/test_auth.py +++ b/src/etcd/tests/test_auth.py @@ -127,7 +127,13 @@ def test_read(self): except: self.fail('Reading an existing role failed') - self.assertEquals(r.acls, {'*': 'RW'}) + # XXX The ACL path result changed from '*' to '/*' at some point + # between etcd-2.2.2 and 2.2.5. They're equivalent so allow + # for both. + if '/*' in r.acls: + self.assertEquals(r.acls, {'/*': 'RW'}) + else: + self.assertEquals(r.acls, {'*': 'RW'}) # We can actually skip most other read tests as they are common # with EtcdUser From 1f6e5118dcbbf995680fdefa89b8b761f81ecdff Mon Sep 17 00:00:00 2001 From: Steve Milner Date: Wed, 16 Nov 2016 17:21:55 -0500 Subject: [PATCH 061/101] Added version/cluster_version properties to client. The client has two new properties which will populate upon first access. If never requested, the properties will be left as None. - version: The version of the etcd server as reported by the server - cluster_version: The version of the cluster as reported by the server --- src/etcd/client.py | 36 +++++++++++++++++++++ src/etcd/tests/unit/test_client.py | 52 ++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/src/etcd/client.py b/src/etcd/client.py index a71da5ae..6292bf05 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -205,6 +205,23 @@ def uri(protocol, host, port): _log.debug("Machines cache initialised to %s", self._machines_cache) + # Versions set to None. They will be set upon first usage. + self._version = self._cluster_version = None + + def _set_version_info(self): + """ + Sets the version information provided by the server. + """ + # Set the version + version_info = json.loads(self.http.request( + self._MGET, + self._base_uri + '/version', + headers=self._get_headers(), + timeout=self.read_timeout, + redirect=self.allow_redirect).data.decode('utf-8')) + self._version = version_info['etcdserver'] + self._cluster_version = version_info['etcdcluster'] + def _discover(self, domain): srv_name = "_etcd._tcp.{}".format(domain) answers = dns.resolver.query(srv_name, 'SRV') @@ -375,6 +392,25 @@ def _stats(self, what='self'): except (TypeError,ValueError): raise etcd.EtcdException("Cannot parse json data in the response") + @property + def version(self): + """ + Version of etcd. + """ + if not self._version: + self._set_version_info() + return self._version + + @property + def cluster_version(self): + """ + Version of the etcd cluster. + """ + if not self._cluster_version: + self._set_version_info() + + return self._cluster_version + @property def key_endpoint(self): """ diff --git a/src/etcd/tests/unit/test_client.py b/src/etcd/tests/unit/test_client.py index bb05a66a..b0b53b2c 100644 --- a/src/etcd/tests/unit/test_client.py +++ b/src/etcd/tests/unit/test_client.py @@ -121,6 +121,58 @@ def test_get_headers_with_auth(self): 'authorization': 'Basic dXNlcm5hbWU6cGFzc3dvcmQ=' } + def test__set_version_info(self): + """Verify _set_version_info makes the proper call to the server""" + with mock.patch('urllib3.PoolManager') as _pm: + _request = _pm().request + # Return the expected data type + _request.return_value = mock.MagicMock( + data=b'{"etcdserver": "2.2.3", "etcdcluster": "2.3.0"}') + + # Create the client and make the call. + client = etcd.Client() + client._set_version_info() + + # Verify we call the proper endpoint + _request.assert_called_once_with( + client._MGET, + client._base_uri + '/version', + headers=mock.ANY, + redirect=mock.ANY, + timeout=mock.ANY) + + # Verify the properties while we are here + self.assertEquals('2.2.3', client.version) + self.assertEquals('2.3.0', client.cluster_version) + + def test_version_property(self): + """Ensure the version property is set on first access.""" + with mock.patch('urllib3.PoolManager') as _pm: + _request = _pm().request + # Return the expected data type + _request.return_value = mock.MagicMock( + data=b'{"etcdserver": "2.2.3", "etcdcluster": "2.3.0"}') + + # Create the client. + client = etcd.Client() + + # Verify the version property is set + self.assertEquals('2.2.3', client.version) + + def test_cluster_version_property(self): + """Ensure the cluster version property is set on first access.""" + with mock.patch('urllib3.PoolManager') as _pm: + _request = _pm().request + # Return the expected data type + _request.return_value = mock.MagicMock( + data=b'{"etcdserver": "2.2.3", "etcdcluster": "2.3.0"}') + + # Create the client. + client = etcd.Client() + + # Verify the cluster_version property is set + self.assertEquals('2.3.0', client.cluster_version) + def test_get_headers_without_auth(self): client = etcd.Client() assert client._get_headers() == {} From 9944176a1e39fb80de3de96a6dfbfaecdaebe552 Mon Sep 17 00:00:00 2001 From: huangdong Date: Mon, 5 Dec 2016 21:58:21 +0800 Subject: [PATCH 062/101] Fix bug when lock in contextmanager way: lock ttl was set to 0 which cause immediately expires --- src/etcd/lock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etcd/lock.py b/src/etcd/lock.py index 8d1eec78..1696d5ad 100644 --- a/src/etcd/lock.py +++ b/src/etcd/lock.py @@ -94,7 +94,7 @@ def __enter__(self): """ You can use the lock as a contextmanager """ - self.acquire(blocking=True, lock_ttl=0) + self.acquire(blocking=True) def __exit__(self, type, value, traceback): self.release() From 3eb214603b858b215a0498bdc96d3c7db27b832a Mon Sep 17 00:00:00 2001 From: Wei Tie Date: Fri, 6 Jan 2017 17:15:03 -0800 Subject: [PATCH 063/101] Import urllib3 exception classes explicitly `urllib3.exceptions` module will be renamed as "requests.packages.urllib3.exceptions" when `python-requests` is loaded, when tries to catch urllib3.exceptions, it actually tries to catch requests.packages.urllib3.exceptions, at the same time urllib3 will always tries to throw exceptions with its own module name. As a result, no exceptions from urllib3 will be treated correctly (e.g. failed on reconnect). This commit imports urllib3 exception classes explicitly, so when tries to compare exception, it will always compares with the one from the same module. --- AUTHORS | 1 + src/etcd/client.py | 13 +++++-------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/AUTHORS b/AUTHORS index 0f07ed1e..fff978e7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -35,3 +35,4 @@ Tomas Kral Tom Denham WillPlatnick WooParadog +Wei Tie diff --git a/src/etcd/client.py b/src/etcd/client.py index a71da5ae..6af824fc 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -15,7 +15,8 @@ from httplib import HTTPException import socket import urllib3 -import urllib3.util +from urllib3.exceptions import HTTPError +from urllib3.exceptions import ReadTimeoutError import json import ssl import dns.resolver @@ -288,9 +289,7 @@ def machines(self): ] _log.debug("Retrieved list of machines: %s", machines) return machines - except (urllib3.exceptions.HTTPError, - HTTPException, - socket.error) as e: + except (HTTPError, HTTPException, socket.error) as e: # We can't get the list of machines, if one server is in the # machines cache, try on it _log.error("Failed to get list of machines from %s%s: %r", @@ -828,12 +827,10 @@ def wrapper(self, path, method, params=None, timeout=None): _ = response.data # urllib3 doesn't wrap all httplib exceptions and earlier versions # don't wrap socket errors either. - except (urllib3.exceptions.HTTPError, - HTTPException, socket.error) as e: + except (HTTPError, HTTPException, socket.error) as e: if (isinstance(params, dict) and params.get("wait") == "true" and - isinstance(e, - urllib3.exceptions.ReadTimeoutError)): + isinstance(e, ReadTimeoutError)): _log.debug("Watch timed out.") raise etcd.EtcdWatchTimedOut( "Watch timed out: %r" % e, From 8b6f0d4b3adc9b33dc68cfbce6c03d4d2d308cd3 Mon Sep 17 00:00:00 2001 From: Steve Milner Date: Mon, 9 Jan 2017 08:49:15 -0500 Subject: [PATCH 064/101] Minor fixes for error classes. - Fixed documentation link in EtcdError - Noted 200 not part of v2 error codes - Added explicit pass to empty EtcdLockExpired --- src/etcd/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/etcd/__init__.py b/src/etcd/__init__.py index f52852c0..33b1d679 100644 --- a/src/etcd/__init__.py +++ b/src/etcd/__init__.py @@ -241,14 +241,16 @@ class EtcdDirNotEmpty(EtcdValueError): """ pass + class EtcdLockExpired(EtcdException): """ Our lock apparently expired while we were trying to acquire it. """ + pass class EtcdError(object): - # See https://github.com/coreos/etcd/blob/master/Documentation/errorcode.md + # See https://github.com/coreos/etcd/blob/master/Documentation/v2/errorcode.md error_exceptions = { 100: EtcdKeyNotFound, 101: EtcdCompareFailed, @@ -262,7 +264,7 @@ class EtcdError(object): # 109: Non-public: existing peer addr. 110: EtcdInsufficientPermissions, - 200: EtcdValueError, + 200: EtcdValueError, # Not part of v2 201: EtcdValueError, 202: EtcdValueError, 203: EtcdValueError, From a5c8b79cb73b7242cf534cddc68c174646f4c4ca Mon Sep 17 00:00:00 2001 From: Jose Plana Date: Tue, 10 Jan 2017 00:54:34 +0100 Subject: [PATCH 065/101] Release version 0.4.4 --- AUTHORS | 8 ++++++++ NEWS.txt | 11 +++++++++++ setup.py | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 0f07ed1e..03485651 100644 --- a/AUTHORS +++ b/AUTHORS @@ -2,10 +2,12 @@ Maintainers: ----------- Jose Plana (jplana) Giuseppe Lavagetto (lavagetto) +Shaun Crampton (fasaxc) Contributors: ------------ Aleksandar Veselinovic +Alexander Brand Alex Chan Alex Ianchici Bartlomiej Biernacki @@ -18,7 +20,9 @@ John Kristensen Joshua Conner Matthias Urlichs Michal Witkowski +Mike Place Nick Bartos +Mingqing Peter Wagner Roberto Aguilar Roy Smith @@ -31,7 +35,11 @@ Simeon Visser Simon Gomizelj SkyLothar Spike Curtis +Stephen Milner +Taylor McKinnon Tomas Kral Tom Denham +Toshiya Kawasaki WillPlatnick +Weizheng Xu WooParadog diff --git a/NEWS.txt b/NEWS.txt index 52e7c4bf..b410e80b 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -1,5 +1,16 @@ News ==== +0.4.4 +----- +*Release date: 10-Jan-2017* + +* Fix some tests +* Use sys,version_info tuple, instead of named tuple +* Improve & fix documentation +* Fix python3 specific problem when blocking on contented lock +* Add refresh key method +* Add custom lock prefix support + 0.4.3 ----- diff --git a/setup.py b/setup.py index 52892d7e..2fe8517b 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ NEWS = open(os.path.join(here, 'NEWS.txt')).read() -version = '0.4.3' +version = '0.4.4' # Dnspython is two different packages depending on python version if sys.version_info[0] == 2: From 87a1955665d6c0fccdd2784b05017e6d4afcad63 Mon Sep 17 00:00:00 2001 From: ainlolcat Date: Sat, 29 Oct 2016 01:48:10 +0400 Subject: [PATCH 066/101] return not just key pass but also modifiedIndex to watch changes starting from it fix tests which expected only string with path --- src/etcd/lock.py | 11 ++++++----- src/etcd/tests/unit/test_lock.py | 7 +++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/etcd/lock.py b/src/etcd/lock.py index 8d1eec78..d1353ab4 100644 --- a/src/etcd/lock.py +++ b/src/etcd/lock.py @@ -112,20 +112,21 @@ def _acquired(self, blocking=True, timeout=0): if not blocking: return False # Let's look for the lock - watch_key = nearest + watch_key = nearest.key _log.debug("Lock not acquired, now watching %s", watch_key) t = max(0, timeout) while True: try: - r = self.client.watch(watch_key, timeout=t) + r = self.client.watch(watch_key, timeout=t, index=nearest.modifiedIndex + 1) _log.debug("Detected variation for %s: %s", r.key, r.action) return self._acquired(blocking=True, timeout=timeout) except etcd.EtcdKeyNotFound: _log.debug("Key %s not present anymore, moving on", watch_key) return self._acquired(blocking=True, timeout=timeout) + except etcd.EtcdLockExpired as e: + raise e except etcd.EtcdException: - # TODO: log something... - pass + _log.exception("Unexpected exception") @property def lock_key(self): @@ -168,7 +169,7 @@ def _get_locker(self): return (l[0], None) else: _log.debug("Locker: %s, key to watch: %s", l[0], l[i-1]) - return (l[0], l[i-1]) + return (l[0], next(x for x in results if x.key == l[i-1])) except ValueError: # Something very wrong is going on, most probably # our lock has expired diff --git a/src/etcd/tests/unit/test_lock.py b/src/etcd/tests/unit/test_lock.py index 107169f6..b114c1fc 100644 --- a/src/etcd/tests/unit/test_lock.py +++ b/src/etcd/tests/unit/test_lock.py @@ -118,7 +118,10 @@ def side_effect(): def test_acquired_no_timeout(self): self.locker._sequence = 4 - returns = [('/_locks/test_lock/4', None), ('/_locks/test_lock/1', '/_locks/test_lock/4')] + returns = [ + ('/_locks/test_lock/4', None), + ('/_locks/test_lock/1', etcd.EtcdResult(node={"key": '/_locks/test_lock/4', "modifiedIndex": 1})) + ] def side_effect(): return returns.pop() @@ -172,7 +175,7 @@ def test_find_lock(self): def test_get_locker(self): self.recursive_read() - self.assertEquals((u'/_locks/test_lock/1', u'/_locks/test_lock/1'), + self.assertEquals((u'/_locks/test_lock/1', etcd.EtcdResult(node={'newKey': False, '_children': [], 'createdIndex': 33, 'modifiedIndex': 33, 'value': u'2qwwwq', 'expiration': None, 'key': u'/_locks/test_lock/1', 'ttl': None, 'action': None, 'dir': False})), self.locker._get_locker()) with self.assertRaises(etcd.EtcdLockExpired): self.locker._sequence = '35' From 577d215bde8f8cac7264c5f15ec3177956227b32 Mon Sep 17 00:00:00 2001 From: Lars Bahner Date: Thu, 26 Jan 2017 13:41:02 +0100 Subject: [PATCH 067/101] Fix context manager that returned NoneType and didn't allow for exceptions to be raised. --- src/etcd/lock.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/etcd/lock.py b/src/etcd/lock.py index 8d1eec78..5e18e064 100644 --- a/src/etcd/lock.py +++ b/src/etcd/lock.py @@ -95,9 +95,11 @@ def __enter__(self): You can use the lock as a contextmanager """ self.acquire(blocking=True, lock_ttl=0) + return self def __exit__(self, type, value, traceback): self.release() + return False def _acquired(self, blocking=True, timeout=0): locker, nearest = self._get_locker() From 7d245b447c6236837786fa41ceba4d2fadf28309 Mon Sep 17 00:00:00 2001 From: tobe Date: Wed, 8 Feb 2017 09:45:16 +0800 Subject: [PATCH 068/101] Add documentation to connect with etcd cluster with host tuple --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 9e7381eb..9520aad5 100644 --- a/README.rst +++ b/README.rst @@ -43,6 +43,7 @@ Create a client object client = etcd.Client() # this will create a client against etcd server running on localhost on port 4001 client = etcd.Client(port=4002) client = etcd.Client(host='127.0.0.1', port=4003) + client = etcd.Client(host=(('127.0.0.1', 4001), ('127.0.0.1', 4002), ('127.0.0.1', 4003))) client = etcd.Client(host='127.0.0.1', port=4003, allow_redirect=False) # wont let you run sensitive commands on non-leader machines, default is true # If you have defined a SRV record for _etcd._tcp.example.com pointing to the clients client = etcd.Client(srv_domain='example.com', protocol="https") From f29e9d61b09ecca83c71956fc57bc0d4f63dcaae Mon Sep 17 00:00:00 2001 From: Gigi Sayfan Date: Sun, 12 Feb 2017 09:03:57 -0800 Subject: [PATCH 069/101] Fix doc comment of client.watch() (#164) * Fix doc comment of client.watch() The exception raised when the timeout expires is etcd.EtcdWatchTimedOut --- src/etcd/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index db7eef29..23495695 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -754,9 +754,9 @@ def watch(self, key, index=None, timeout=None, recursive=None): client.EtcdResult Raises: - KeyValue: If the key doesn't exists. + KeyValue: If the key doesn't exist. - urllib3.exceptions.TimeoutError: If timeout is reached. + etcd.EtcdWatchTimedOut: If timeout is reached. >>> print client.watch('/key').value 'value' From 64936be5d1be787657b8631e3114f9a5d8a4ed9f Mon Sep 17 00:00:00 2001 From: Ilya Kotusev Date: Thu, 24 Dec 2015 12:58:11 +0000 Subject: [PATCH 070/101] remove _base_uri only after refresh from cluster --- src/etcd/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index 23495695..7a495f05 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -903,7 +903,7 @@ def wrapper(self, path, method, params=None, timeout=None): if not self._use_proxies: # The cluster may have changed since last invocation self._machines_cache = self.machines - self._machines_cache.remove(self._base_uri) + self._machines_cache.remove(self._base_uri) return self._handle_server_response(response) return wrapper From 642048626572a1297aa10fcf609b5df3afe25536 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 24 Dec 2015 09:30:10 +0100 Subject: [PATCH 071/101] reset some_request_failed to False on each iteration of the loop otherwise we could get into the situation when the first request has failed, then _machines_cache has been updated and request executed successfully and it will try to update _machines_cache once again. --- src/etcd/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index 7a495f05..74ad8fc1 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -837,7 +837,6 @@ def _next_server(self, cause=None): def _wrap_request(payload): @wraps(payload) def wrapper(self, path, method, params=None, timeout=None): - some_request_failed = False response = False if timeout is None: @@ -850,6 +849,7 @@ def wrapper(self, path, method, params=None, timeout=None): raise ValueError('Path does not start with /') while not response: + some_request_failed = False try: response = payload(self, path, method, params=params, timeout=timeout) From 96f2ff4e1d5143d617e0f9c2c57eb08fd3b2ea03 Mon Sep 17 00:00:00 2001 From: Jose Plana Date: Fri, 3 Mar 2017 00:03:02 +0100 Subject: [PATCH 072/101] Prepare release 0.4.5 --- AUTHORS | 7 +++++++ NEWS.txt | 18 ++++++++++++++++++ setup.py | 2 +- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index aca13b2b..b12bda93 100644 --- a/AUTHORS +++ b/AUTHORS @@ -8,22 +8,28 @@ Contributors: ------------ Aleksandar Veselinovic Alexander Brand +Alexander Kukushkin Alex Chan Alex Ianchici +Ainlolcat Bartlomiej Biernacki Bradley Cicenas Christoph Heer +Gigi Sayfan Hogenmiller +Huangdong Jimmy Zelinskie Jim Rollenhagen John Kristensen Joshua Conner +Lars Bahner Matthias Urlichs Michal Witkowski Mike Place Nick Bartos Mingqing Peter Wagner +Realityone Roberto Aguilar Roy Smith Ryan Fowler @@ -37,6 +43,7 @@ SkyLothar Spike Curtis Stephen Milner Taylor McKinnon +Tobe Tomas Kral Tom Denham Toshiya Kawasaki diff --git a/NEWS.txt b/NEWS.txt index b410e80b..00d8cd25 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -1,5 +1,23 @@ News ==== +0.4.5 +----- +*Release date: 3-Mar-2017* + +* Remove dnspython2/3 requirement +* Change property name setter in lock +* Fixed acl tests +* Added version/cluster_version properties to client +* Fixes in lock when used as context manager +* Fixed improper usage of urllib3 exceptions +* Minor fixes for error classes +* In lock return modifiedIndex to watch changes +* In lock fix context manager exception handling +* Improvments to the documentation +* Remove _base_uri only after refresh from cluster +* Avoid double update of _machines_cache + + 0.4.4 ----- *Release date: 10-Jan-2017* diff --git a/setup.py b/setup.py index 52f6994a..a4c6d01d 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ NEWS = open(os.path.join(here, 'NEWS.txt')).read() -version = '0.4.4' +version = '0.4.5' install_requires = [ 'urllib3>=1.7.1', From 8102bafa46a418f5697296c207313a244282cf24 Mon Sep 17 00:00:00 2001 From: Steve Milner Date: Thu, 2 Mar 2017 21:40:47 -0500 Subject: [PATCH 073/101] Add AUTHORS and LICENSE.txt to dist --- MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 1e7e5684..e34a526c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,4 @@ +include AUTHORS +include LICENSE.txt include README.rst include NEWS.txt From 18c75196ffd04b3104f3ba44356ec1fcbcc4d09a Mon Sep 17 00:00:00 2001 From: Steve Milner Date: Thu, 2 Mar 2017 21:41:22 -0500 Subject: [PATCH 074/101] Add Matthew Barnes to AUTHORS @mbarnes was a co-auther of some of the work merged by Stephen Milner. --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index b12bda93..716c8835 100644 --- a/AUTHORS +++ b/AUTHORS @@ -23,6 +23,7 @@ Jim Rollenhagen John Kristensen Joshua Conner Lars Bahner +Matthew Barnes Matthias Urlichs Michal Witkowski Mike Place From 363391445b35be66b7c2205914401afd6135c775 Mon Sep 17 00:00:00 2001 From: Ian Wells Date: Fri, 31 Mar 2017 11:32:07 -0700 Subject: [PATCH 075/101] Reduce reraised-exception log to debug When an exception occurs during a request, it can be for harmless reasons (using an eventlet.timeout, for example). Also, the exception is already returned to the caller, who can better judge its severity. Logging it here as an exception puts a lot of ERROR-level logs out that might not, in fact, reflect errors, This lowers the log level to a single-line debug. --- src/etcd/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index 74ad8fc1..ed1d3335 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -896,7 +896,7 @@ def wrapper(self, path, method, params=None, timeout=None): _log.warning(e) raise except: - _log.exception("Unexpected request failure, re-raising.") + _log.debug("Unexpected request failure, re-raising.") raise if some_request_failed: From b9990019c5c0608713bb6dcc013c08dbf31a9628 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Mon, 13 Feb 2017 08:30:34 +0100 Subject: [PATCH 076/101] Update etcd, urllib3, pyopenssl versions in travis builds Specifically, use the latest pyopenssl, a fairly recent urllib3, and the most recent version of etcd 2.x --- .travis.yml | 4 ++-- build_etcd.sh | 8 +++++++- buildout.cfg | 4 ++-- download_etcd.sh | 6 ++++++ 4 files changed, 17 insertions(+), 5 deletions(-) create mode 100755 download_etcd.sh diff --git a/.travis.yml b/.travis.yml index 2c3ba505..b3db1220 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ python: - "3.5" before_install: - - ./build_etcd.sh v2.2.0 + - ./download_etcd.sh 2.3.7 - pip install --upgrade setuptools # command to install dependencies @@ -16,7 +16,7 @@ install: # command to run tests script: - PATH=$PATH:./etcd/bin coverage run --source=src/etcd --omit="src/etcd/tests/*" bin/test + PATH=$PATH:./bin coverage run --source=src/etcd --omit="src/etcd/tests/*" bin/test after_success: coveralls # Add env var to detect it during build diff --git a/build_etcd.sh b/build_etcd.sh index fc319919..5ce9d664 100755 --- a/build_etcd.sh +++ b/build_etcd.sh @@ -9,10 +9,16 @@ fi echo "Using ETCD version $ETCD_VERSION" +BASE=$PWD +mkdir -p gopath/src/coreos/ +export GOPATH=$BASE/gopath/ +cd $GOPATH/src/coreos git clone https://github.com/coreos/etcd.git cd etcd -git checkout $ETCD_VERSION +git checkout -b buildout $ETCD_VERSION ./build +cd $BASE +cp -r $GOPATH/src/coreos/etcd/bin . ${TRAVIS:?"This is not a Travis build. All Done"} diff --git a/buildout.cfg b/buildout.cfg index 4de90366..3a1e0baf 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -5,8 +5,8 @@ parts = python coverage develop = . eggs = - urllib3==1.7.1 - pyOpenSSL==0.13.1 + urllib3==1.19.1 + pyOpenSSL==16.2 ${deps:extraeggs} [python] diff --git a/download_etcd.sh b/download_etcd.sh new file mode 100755 index 00000000..bdd592de --- /dev/null +++ b/download_etcd.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e +VERSION=${1:-2.3.7} +mkdir -p bin +URL="https://github.com/coreos/etcd/releases/download/v${VERSION}/etcd-v${VERSION}-linux-amd64.tar.gz" +curl -L $URL | tar -C ./bin --strip-components=1 -xzvf - "etcd-v${VERSION}-linux-amd64/etcd" From 4e5dd28891539e4b7d4f67cc38b61e9278c1225b Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Mon, 13 Feb 2017 12:34:09 +0100 Subject: [PATCH 077/101] Use api_execute for getting the version of the cluster This will allow the same kind of safety net everything gets besides the fetch of machines. --- src/etcd/client.py | 15 +++---- src/etcd/tests/unit/test_client.py | 70 ++++++++++++------------------ 2 files changed, 33 insertions(+), 52 deletions(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index ed1d3335..0723ecff 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -214,12 +214,8 @@ def _set_version_info(self): Sets the version information provided by the server. """ # Set the version - version_info = json.loads(self.http.request( - self._MGET, - self._base_uri + '/version', - headers=self._get_headers(), - timeout=self.read_timeout, - redirect=self.allow_redirect).data.decode('utf-8')) + data = self.api_execute('/version', self._MGET).data + version_info = json.loads(data.decode('utf-8')) self._version = version_info['etcdserver'] self._cluster_version = version_info['etcdcluster'] @@ -856,7 +852,7 @@ def wrapper(self, path, method, params=None, timeout=None): # Check the cluster ID hasn't changed under us. We use # preload_content=False above so we can read the headers # before we wait for the content of a watch. - self._check_cluster_id(response) + self._check_cluster_id(response, path) # Now force the data to be preloaded in order to trigger any # IO-related errors in this method rather than when we try to # access it later. @@ -950,10 +946,11 @@ def api_execute_json(self, path, method, params=None, timeout=None): headers=headers, preload_content=False) - def _check_cluster_id(self, response): + def _check_cluster_id(self, response, path): cluster_id = response.getheader("x-etcd-cluster-id") if not cluster_id: - _log.warning("etcd response did not contain a cluster ID") + if self.version_prefix in path: + _log.warning("etcd response did not contain a cluster ID") return id_changed = (self.expected_cluster_id and cluster_id != self.expected_cluster_id) diff --git a/src/etcd/tests/unit/test_client.py b/src/etcd/tests/unit/test_client.py index b0b53b2c..2981de7c 100644 --- a/src/etcd/tests/unit/test_client.py +++ b/src/etcd/tests/unit/test_client.py @@ -3,13 +3,15 @@ import dns.name import dns.rdtypes.IN.SRV import dns.resolver +from etcd.tests.unit import TestClientApiBase try: import mock except ImportError: from unittest import mock -class TestClient(unittest.TestCase): +class TestClient(TestClientApiBase): + def test_instantiate(self): """ client can be instantiated""" @@ -123,55 +125,37 @@ def test_get_headers_with_auth(self): def test__set_version_info(self): """Verify _set_version_info makes the proper call to the server""" - with mock.patch('urllib3.PoolManager') as _pm: - _request = _pm().request - # Return the expected data type - _request.return_value = mock.MagicMock( - data=b'{"etcdserver": "2.2.3", "etcdcluster": "2.3.0"}') - - # Create the client and make the call. - client = etcd.Client() - client._set_version_info() - - # Verify we call the proper endpoint - _request.assert_called_once_with( - client._MGET, - client._base_uri + '/version', - headers=mock.ANY, - redirect=mock.ANY, - timeout=mock.ANY) - - # Verify the properties while we are here - self.assertEquals('2.2.3', client.version) - self.assertEquals('2.3.0', client.cluster_version) + data = {"etcdserver": "2.2.3", "etcdcluster": "2.3.0"} + self._mock_api(200, data) + self.client.api_execute.return_value.getheader.return_value = None + # Create the client and make the call. + self.client._set_version_info() + + # Verify we call the proper endpoint + self.client.api_execute.assert_called_once_with( + '/version', + self.client._MGET + ) + # Verify the properties while we are here + self.assertEquals('2.2.3', self.client.version) + self.assertEquals('2.3.0', self.client.cluster_version) def test_version_property(self): """Ensure the version property is set on first access.""" - with mock.patch('urllib3.PoolManager') as _pm: - _request = _pm().request - # Return the expected data type - _request.return_value = mock.MagicMock( - data=b'{"etcdserver": "2.2.3", "etcdcluster": "2.3.0"}') - - # Create the client. - client = etcd.Client() + data = {"etcdserver": "2.2.3", "etcdcluster": "2.3.0"} + self._mock_api(200, data) + self.client.api_execute.return_value.getheader.return_value = None - # Verify the version property is set - self.assertEquals('2.2.3', client.version) + # Verify the version property is set + self.assertEquals('2.2.3', self.client.version) def test_cluster_version_property(self): """Ensure the cluster version property is set on first access.""" - with mock.patch('urllib3.PoolManager') as _pm: - _request = _pm().request - # Return the expected data type - _request.return_value = mock.MagicMock( - data=b'{"etcdserver": "2.2.3", "etcdcluster": "2.3.0"}') - - # Create the client. - client = etcd.Client() - - # Verify the cluster_version property is set - self.assertEquals('2.3.0', client.cluster_version) + data = {"etcdserver": "2.2.3", "etcdcluster": "2.3.0"} + self._mock_api(200, data) + self.client.api_execute.return_value.getheader.return_value = None + # Verify the cluster_version property is set + self.assertEquals('2.3.0', self.client.cluster_version) def test_get_headers_without_auth(self): client = etcd.Client() From 7f3dd65e5dc79cc456ef58a052501ec256d5070b Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Mon, 13 Feb 2017 14:12:39 +0100 Subject: [PATCH 078/101] Support auth API both <= 2.2.5 and >= 2.3.0 Closes #210 --- src/etcd/auth.py | 28 ++++++++++++++++++++++++++-- src/etcd/tests/test_auth.py | 4 ++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/etcd/auth.py b/src/etcd/auth.py index 796772d7..c5c73465 100644 --- a/src/etcd/auth.py +++ b/src/etcd/auth.py @@ -14,13 +14,28 @@ def __init__(self, client, name): self.name = name self.uri = "{}/auth/{}s/{}".format(self.client.version_prefix, self.entity, self.name) + # This will be lazily evaluated if not manually set + self._legacy_api = None + + @property + def legacy_api(self): + if self._legacy_api is None: + # The auth API has changed between 2.2 and 2.3, true story! + major, minor, _ = map(int, self.client.version.split('.')) + self._legacy_api = (major < 3 and minor < 3) + return self._legacy_api + @property def names(self): key = "{}s".format(self.entity) uri = "{}/auth/{}".format(self.client.version_prefix, key) response = self.client.api_execute(uri, self.client._MGET) - return json.loads(response.data.decode('utf-8'))[key] + if self.legacy_api: + return json.loads(response.data.decode('utf-8'))[key] + else: + return [obj[self.entity] + for obj in json.loads(response.data.decode('utf-8'))[key]] def read(self): try: @@ -102,7 +117,16 @@ def __init__(self, client, name): def _from_net(self, data): d = json.loads(data.decode('utf-8')) - self.roles = d.get('roles', []) + roles = d.get('roles', []) + try: + self.roles = roles + except TypeError: + # with the change of API, PUT responses are different + # from GET reponses, which makes everything so funny. + # Specifically, PUT responses are the same as before... + if self.legacy_api: + raise + self.roles = [obj['role'] for obj in roles] self.name = d.get('user') def _to_net(self, prevobj=None): diff --git a/src/etcd/tests/test_auth.py b/src/etcd/tests/test_auth.py index 14475f91..5c8c0b07 100644 --- a/src/etcd/tests/test_auth.py +++ b/src/etcd/tests/test_auth.py @@ -93,6 +93,10 @@ def test_write_and_delete(self): self.assertEquals(u.roles, set(['guest', 'root'])) # set roles as a list, it works! u.roles = ['guest', 'test_group'] + # We need this or the new API will return an internal error + r = auth.EtcdRole(self.client, 'test_group') + r.acls = {'*': 'R', '/test/*': 'RW'} + r.write() try: u.write() except: From 001d85ebdebd12317a999a70284f1a89a09fea79 Mon Sep 17 00:00:00 2001 From: Bernard McKeever Date: Wed, 29 Mar 2017 11:15:57 +0100 Subject: [PATCH 079/101] Fixing a couple of typos Nothing fancy, just a couple of typos I spotted in logging and exception messages. --- src/etcd/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index 0723ecff..7c4d9858 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -228,7 +228,7 @@ def _discover(self, domain): (answer.target.to_text(omit_final_dot=True), answer.port)) _log.debug("Found %s", hosts) if not len(hosts): - raise ValueError("The SRV record is present but no host were found") + raise ValueError("The SRV record is present but no hosts were found") return tuple(hosts) def __del__(self): @@ -818,7 +818,7 @@ def _result_from_response(self, response): def _next_server(self, cause=None): """ Selects the next server in the list, refreshes the server list. """ - _log.debug("Selection next machine in cache. Available machines: %s", + _log.debug("Selecting next machine in cache. Available machines: %s", self._machines_cache) try: mach = self._machines_cache.pop() From 2593c8137e8f36579037b3df3653d0bbd5730db6 Mon Sep 17 00:00:00 2001 From: cr0hn Date: Fri, 31 Mar 2017 09:59:50 +0200 Subject: [PATCH 080/101] Update README.rst Add instructions for install from Pypi --- README.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.rst b/README.rst index 9520aad5..e1aa8981 100644 --- a/README.rst +++ b/README.rst @@ -27,6 +27,13 @@ From source .. code:: bash $ python setup.py install + +From Pypi +~~~~~~~~~ + +.. code:: bash + + $ python3.5 -m pip install aio_etcd Usage ----- From b227f496c038b2b856c4d76c9525b3547e5c8dc4 Mon Sep 17 00:00:00 2001 From: cr0hn Date: Thu, 22 Jun 2017 02:31:30 +0200 Subject: [PATCH 081/101] Update README.rst fixed name --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e1aa8981..0df9c602 100644 --- a/README.rst +++ b/README.rst @@ -33,7 +33,7 @@ From Pypi .. code:: bash - $ python3.5 -m pip install aio_etcd + $ python3.5 -m pip install python-etcd Usage ----- From cfa0e66fb8b49833690cc9b1a09e465ddcfcfa39 Mon Sep 17 00:00:00 2001 From: Sebastien Coutu Date: Wed, 31 May 2017 10:14:13 -0400 Subject: [PATCH 082/101] Update DNS discovery to better match ETCD documentation. --- src/etcd/client.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index 7c4d9858..071cea79 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -65,7 +65,8 @@ def __init__( use_proxies=False, expected_cluster_id=None, per_host_pool_size=10, - lock_prefix="/_locks" + lock_prefix="/_locks", + srv_use_ssl=False ): """ Initialize the client. @@ -115,12 +116,14 @@ def __init__( connections. lock_prefix (str): Set the key prefix at etcd when client to lock object. By default this will be use /_locks. + + srv_use_ssl (bool): Should we use SSL alias for cluster autodiscovery. """ # If a DNS record is provided, use it to get the hosts list if srv_domain is not None: try: - host = self._discover(srv_domain) + host = self._discover(srv_domain, use_ssl=srv_use_ssl) except Exception as e: _log.error("Could not discover the etcd hosts from %s: %s", srv_domain, e) @@ -219,8 +222,11 @@ def _set_version_info(self): self._version = version_info['etcdserver'] self._cluster_version = version_info['etcdcluster'] - def _discover(self, domain): - srv_name = "_etcd._tcp.{}".format(domain) + def _discover(self, domain, use_ssl=False): + if use_ssl: + srv_name = "_etcd-client-ssl._tcp.{}".format(domain) + else: + srv_name = "_etcd-client._tcp.{}".format(domain) answers = dns.resolver.query(srv_name, 'SRV') hosts = [] for answer in answers: From 12b142b8931b8ea2ad778af1da7aa8dd400a83fe Mon Sep 17 00:00:00 2001 From: Sebastien Coutu Date: Sun, 18 Jun 2017 12:03:07 -0400 Subject: [PATCH 083/101] chore(gitignore): Adding virtual environments usual directories. --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 3f90b7fd..d782d163 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ build dist docs .coverage +.venv +.env From 69e23a5413194a33db5b93dbd6086272eb5da759 Mon Sep 17 00:00:00 2001 From: Sebastien Coutu Date: Sun, 18 Jun 2017 12:05:49 -0400 Subject: [PATCH 084/101] fix(dns-discovery): Fixes regarding PR comments. - Removing most parameters I've added - Adding a list of DNS names to try instead of a single one. - If '-ssl' is found in the name of the DNS entry, set protocol to 'https'. --- src/etcd/client.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index 071cea79..dfcfef90 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -22,6 +22,8 @@ import dns.resolver from functools import wraps import etcd +from dns.resolver import NXDOMAIN +import re try: from urlparse import urlparse @@ -65,8 +67,7 @@ def __init__( use_proxies=False, expected_cluster_id=None, per_host_pool_size=10, - lock_prefix="/_locks", - srv_use_ssl=False + lock_prefix="/_locks" ): """ Initialize the client. @@ -116,8 +117,6 @@ def __init__( connections. lock_prefix (str): Set the key prefix at etcd when client to lock object. By default this will be use /_locks. - - srv_use_ssl (bool): Should we use SSL alias for cluster autodiscovery. """ # If a DNS record is provided, use it to get the hosts list @@ -222,12 +221,29 @@ def _set_version_info(self): self._version = version_info['etcdserver'] self._cluster_version = version_info['etcdcluster'] - def _discover(self, domain, use_ssl=False): - if use_ssl: - srv_name = "_etcd-client-ssl._tcp.{}".format(domain) - else: - srv_name = "_etcd-client._tcp.{}".format(domain) - answers = dns.resolver.query(srv_name, 'SRV') + def _discover(self, domain): + srv_names = [ + "_etcd-client-ssl._tcp.{}".format(domain), + "_etcd-client._tcp.{}".format(domain), + "_etcd-ssl._tcp.{}".format(domain), + "_etcd._tcp.{}".format(domain) + ] + found = False + for srv_name in srv_names: + try: + answers = dns.resolver.query(srv_name, 'SRV') + if len(answers): + found = True + break + except NXDOMAIN: + continue + + if not found: + raise ValueError('Could not find SRV record for domain {}.'.format(domain)) + + if re.search('-ssl', srv_name): + self._protocol = 'https' + hosts = [] for answer in answers: hosts.append( From 44d2725071a2ae9d73fccae40c1425f1fa952b51 Mon Sep 17 00:00:00 2001 From: Sebastien Coutu Date: Sun, 18 Jun 2017 12:13:29 -0400 Subject: [PATCH 085/101] fix(dns-discovery): Forgot to remove an instance of my previous change --- src/etcd/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index dfcfef90..12c1863d 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -122,7 +122,7 @@ def __init__( # If a DNS record is provided, use it to get the hosts list if srv_domain is not None: try: - host = self._discover(srv_domain, use_ssl=srv_use_ssl) + host = self._discover(srv_domain) except Exception as e: _log.error("Could not discover the etcd hosts from %s: %s", srv_domain, e) From 68d8d5dd4ae7ece6c63f446e3c38448998cde3e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Mon, 22 May 2023 14:04:55 +0200 Subject: [PATCH 086/101] Replace the usage of assertEquals unit test alias removed in Python 3.12 with assertEqual --- src/etcd/tests/unit/test_client.py | 14 ++++----- src/etcd/tests/unit/test_lock.py | 26 ++++++++-------- src/etcd/tests/unit/test_old_request.py | 30 +++++++++---------- src/etcd/tests/unit/test_request.py | 40 ++++++++++++------------- 4 files changed, 55 insertions(+), 55 deletions(-) diff --git a/src/etcd/tests/unit/test_client.py b/src/etcd/tests/unit/test_client.py index 2981de7c..d99de9bd 100644 --- a/src/etcd/tests/unit/test_client.py +++ b/src/etcd/tests/unit/test_client.py @@ -137,8 +137,8 @@ def test__set_version_info(self): self.client._MGET ) # Verify the properties while we are here - self.assertEquals('2.2.3', self.client.version) - self.assertEquals('2.3.0', self.client.cluster_version) + self.assertEqual('2.2.3', self.client.version) + self.assertEqual('2.3.0', self.client.cluster_version) def test_version_property(self): """Ensure the version property is set on first access.""" @@ -147,7 +147,7 @@ def test_version_property(self): self.client.api_execute.return_value.getheader.return_value = None # Verify the version property is set - self.assertEquals('2.2.3', self.client.version) + self.assertEqual('2.2.3', self.client.version) def test_cluster_version_property(self): """Ensure the cluster version property is set on first access.""" @@ -155,7 +155,7 @@ def test_cluster_version_property(self): self._mock_api(200, data) self.client.api_execute.return_value.getheader.return_value = None # Verify the cluster_version property is set - self.assertEquals('2.3.0', self.client.cluster_version) + self.assertEqual('2.3.0', self.client.cluster_version) def test_get_headers_without_auth(self): client = etcd.Client() @@ -191,7 +191,7 @@ def test_discover(self): etcd.Client.machines = mock.create_autospec(etcd.Client.machines, return_value=[u'https://etcd2.example.com:2379']) c = etcd.Client(srv_domain="example.com", allow_reconnect=True, protocol="https") etcd.Client.machines = self.machines - self.assertEquals(c.host, u'etcd1.example.com') - self.assertEquals(c.port, 2379) - self.assertEquals(c._machines_cache, + self.assertEqual(c.host, u'etcd1.example.com') + self.assertEqual(c.port, 2379) + self.assertEqual(c._machines_cache, [u'https://etcd2.example.com:2379']) diff --git a/src/etcd/tests/unit/test_lock.py b/src/etcd/tests/unit/test_lock.py index b114c1fc..7384084a 100644 --- a/src/etcd/tests/unit/test_lock.py +++ b/src/etcd/tests/unit/test_lock.py @@ -31,9 +31,9 @@ def test_initialization(self): """ Verify the lock gets initialized correctly """ - self.assertEquals(self.locker.name, u'test_lock') - self.assertEquals(self.locker.path, u'/_locks/test_lock') - self.assertEquals(self.locker.is_taken, False) + self.assertEqual(self.locker.name, u'test_lock') + self.assertEqual(self.locker.path, u'/_locks/test_lock') + self.assertEqual(self.locker.is_taken, False) def test_acquire(self): """ @@ -52,8 +52,8 @@ def test_acquire(self): } } self._mock_api(200, d) - self.assertEquals(l.acquire(), True) - self.assertEquals(l._sequence, '1') + self.assertEqual(l.acquire(), True) + self.assertEqual(l._sequence, '1') def test_is_acquired(self): """ @@ -70,7 +70,7 @@ def test_is_acquired(self): } self._mock_api(200, d) self.locker.is_taken = True - self.assertEquals(self.locker.is_acquired, True) + self.assertEqual(self.locker.is_acquired, True) def test_is_not_acquired(self): """ @@ -78,11 +78,11 @@ def test_is_not_acquired(self): """ self.locker._sequence = '2' self.locker.is_taken = False - self.assertEquals(self.locker.is_acquired, False) + self.assertEqual(self.locker.is_acquired, False) self.locker.is_taken = True self._mock_exception(etcd.EtcdKeyNotFound, self.locker.lock_key) - self.assertEquals(self.locker.is_acquired, False) - self.assertEquals(self.locker.is_taken, False) + self.assertEqual(self.locker.is_acquired, False) + self.assertEqual(self.locker.is_taken, False) def test_acquired(self): """ @@ -147,11 +147,11 @@ def test_lock_key(self): with self.assertRaises(ValueError): self.locker.lock_key self.locker._sequence = '5' - self.assertEquals(u'/_locks/test_lock/5',self.locker.lock_key) + self.assertEqual(u'/_locks/test_lock/5',self.locker.lock_key) def test_set_sequence(self): self.locker._set_sequence('/_locks/test_lock/10') - self.assertEquals('10', self.locker._sequence) + self.assertEqual('10', self.locker._sequence) def test_find_lock(self): d = { @@ -171,11 +171,11 @@ def test_find_lock(self): self.locker._sequence = None self.recursive_read() self.assertTrue(self.locker._find_lock()) - self.assertEquals(self.locker._sequence, '34') + self.assertEqual(self.locker._sequence, '34') def test_get_locker(self): self.recursive_read() - self.assertEquals((u'/_locks/test_lock/1', etcd.EtcdResult(node={'newKey': False, '_children': [], 'createdIndex': 33, 'modifiedIndex': 33, 'value': u'2qwwwq', 'expiration': None, 'key': u'/_locks/test_lock/1', 'ttl': None, 'action': None, 'dir': False})), + self.assertEqual((u'/_locks/test_lock/1', etcd.EtcdResult(node={'newKey': False, '_children': [], 'createdIndex': 33, 'modifiedIndex': 33, 'value': u'2qwwwq', 'expiration': None, 'key': u'/_locks/test_lock/1', 'ttl': None, 'action': None, 'dir': False})), self.locker._get_locker()) with self.assertRaises(etcd.EtcdLockExpired): self.locker._sequence = '35' diff --git a/src/etcd/tests/unit/test_old_request.py b/src/etcd/tests/unit/test_old_request.py index 0d437131..5fb75581 100644 --- a/src/etcd/tests/unit/test_old_request.py +++ b/src/etcd/tests/unit/test_old_request.py @@ -42,7 +42,7 @@ def test_set(self): result = client.set('/testkey', 'test', ttl=19) - self.assertEquals( + self.assertEqual( etcd.EtcdResult( **{u'action': u'SET', 'node': { @@ -67,7 +67,7 @@ def test_test_and_set(self): '"ttl":49,"modifiedIndex":203}}') ) result = client.test_and_set('/testkey', 'newvalue', 'test', ttl=19) - self.assertEquals( + self.assertEqual( etcd.EtcdResult( **{u'action': u'SET', u'node': { @@ -94,7 +94,7 @@ def test_test_and_test_failure(self): 'test', ttl=19) except ValueError as e: #from ipdb import set_trace; set_trace() - self.assertEquals( + self.assertEqual( 'The given PrevValue is not equal' ' to the value of the key : TestAndSet: 1!=3', str(e)) @@ -111,7 +111,7 @@ def test_delete(self): '"modifiedIndex":189}}') ) result = client.delete('/testkey') - self.assertEquals(etcd.EtcdResult( + self.assertEqual(etcd.EtcdResult( **{u'action': u'DELETE', u'node': { u'expiration': u'2013-09-14T01:06:35.5242587+02:00', @@ -133,7 +133,7 @@ def test_get(self): ) result = client.get('/testkey') - self.assertEquals(etcd.EtcdResult( + self.assertEqual(etcd.EtcdResult( **{u'action': u'GET', u'node': { u'modifiedIndex': 190, @@ -154,7 +154,7 @@ def test_not_in(self): client = etcd.Client() client.get = mock.Mock(side_effect=etcd.EtcdKeyNotFound()) result = '/testkey' not in client - self.assertEquals(True, result) + self.assertEqual(True, result) def test_in(self): """ Can check if key is in client """ @@ -169,7 +169,7 @@ def test_in(self): ) result = '/testkey' in client - self.assertEquals(True, result) + self.assertEqual(True, result) def test_simple_watch(self): """ Can watch values """ @@ -186,7 +186,7 @@ def test_simple_watch(self): '"modifiedIndex":192}}') ) result = client.watch('/testkey') - self.assertEquals( + self.assertEqual( etcd.EtcdResult( **{u'action': u'SET', u'node': { @@ -213,7 +213,7 @@ def test_index_watch(self): '"modifiedIndex":180}}') ) result = client.watch('/testkey', index=180) - self.assertEquals( + self.assertEqual( etcd.EtcdResult( **{u'action': u'SET', u'node': { @@ -267,7 +267,7 @@ def test_get(self): response = FakeHTTPResponse(status=200, data='arbitrary json data') client.http.request = mock.Mock(return_value=response) result = client.api_execute('/v1/keys/testkey', client._MGET) - self.assertEquals('arbitrary json data'.encode('utf-8'), result.data) + self.assertEqual('arbitrary json data'.encode('utf-8'), result.data) def test_delete(self): """ http delete request """ @@ -275,7 +275,7 @@ def test_delete(self): response = FakeHTTPResponse(status=200, data='arbitrary json data') client.http.request = mock.Mock(return_value=response) result = client.api_execute('/v1/keys/testkey', client._MDELETE) - self.assertEquals('arbitrary json data'.encode('utf-8'), result.data) + self.assertEqual('arbitrary json data'.encode('utf-8'), result.data) def test_get_error(self): """ http get error request 101""" @@ -289,7 +289,7 @@ def test_get_error(self): client.api_execute('/v2/keys/testkey', client._MGET) assert False except etcd.EtcdKeyNotFound as e: - self.assertEquals(str(e), 'message : cause') + self.assertEqual(str(e), 'message : cause') def test_put(self): """ http put request """ @@ -297,7 +297,7 @@ def test_put(self): response = FakeHTTPResponse(status=200, data='arbitrary json data') client.http.request_encode_body = mock.Mock(return_value=response) result = client.api_execute('/v2/keys/testkey', client._MPUT) - self.assertEquals('arbitrary json data'.encode('utf-8'), result.data) + self.assertEqual('arbitrary json data'.encode('utf-8'), result.data) def test_test_and_set_error(self): """ http post error request 101 """ @@ -311,7 +311,7 @@ def test_test_and_set_error(self): client.api_execute('/v2/keys/testkey', client._MPUT, payload) self.fail() except ValueError as e: - self.assertEquals('message : cause', str(e)) + self.assertEqual('message : cause', str(e)) def test_set_not_file_error(self): """ http post error request 102 """ @@ -325,7 +325,7 @@ def test_set_not_file_error(self): client.api_execute('/v2/keys/testkey', client._MPUT, payload) self.fail() except etcd.EtcdNotFile as e: - self.assertEquals('message : cause', str(e)) + self.assertEqual('message : cause', str(e)) def test_get_error_unknown(self): """ http get error request unknown """ diff --git a/src/etcd/tests/unit/test_request.py b/src/etcd/tests/unit/test_request.py index b972a8c5..9c0fcd64 100644 --- a/src/etcd/tests/unit/test_request.py +++ b/src/etcd/tests/unit/test_request.py @@ -68,7 +68,7 @@ def test_write_no_params(self): } self._mock_api(200, d) self.client.write('/newdir', None, dir=True) - self.assertEquals(self.client.api_execute.call_args, + self.assertEqual(self.client.api_execute.call_args, (('/v2/keys/newdir', 'PUT'), dict(params={'dir': 'true'}))) @@ -86,7 +86,7 @@ def test_machines(self, mocker): 'http://127.0.0.1:4002', 'http://127.0.0.1:4003'] d = ','.join(data) mocker.return_value = self._prepare_response(200, d) - self.assertEquals(data, self.client.machines) + self.assertEqual(data, self.client.machines) @mock.patch('etcd.Client.machines', new_callable=mock.PropertyMock) def test_use_proxies(self, mocker): @@ -101,8 +101,8 @@ def test_use_proxies(self, mocker): use_proxies=True ) - self.assertEquals(c._machines_cache, ['https://localproxy:4001']) - self.assertEquals(c._base_uri, 'https://localhost:4001') + self.assertEqual(c._machines_cache, ['https://localproxy:4001']) + self.assertEqual(c._base_uri, 'https://localhost:4001') self.assertNotIn(c.base_uri, c._machines_cache) c = etcd.Client( @@ -128,7 +128,7 @@ def test_members(self): ] } self._mock_api(200, data) - self.assertEquals(self.client.members["ce2a822cea30bfca"]["id"], "ce2a822cea30bfca") + self.assertEqual(self.client.members["ce2a822cea30bfca"]["id"], "ce2a822cea30bfca") def test_self_stats(self): """ Request for stats """ @@ -148,13 +148,13 @@ def test_self_stats(self): "state": "StateFollower" } self._mock_api(200,data) - self.assertEquals(self.client.stats['name'], "node3") + self.assertEqual(self.client.stats['name'], "node3") def test_leader_stats(self): """ Request for leader stats """ data = {"leader": "924e2e83e93f2560", "followers": {}} self._mock_api(200,data) - self.assertEquals(self.client.leader_stats['leader'], "924e2e83e93f2560") + self.assertEqual(self.client.leader_stats['leader'], "924e2e83e93f2560") @mock.patch('etcd.Client.members', new_callable=mock.PropertyMock) @@ -163,7 +163,7 @@ def test_leader(self, mocker): members = {"ce2a822cea30bfca": {"id": "ce2a822cea30bfca", "name": "default"}} mocker.return_value = members self._mock_api(200, {"leaderInfo":{"leader": "ce2a822cea30bfca", "followers": {}}}) - self.assertEquals(self.client.leader, members["ce2a822cea30bfca"]) + self.assertEqual(self.client.leader, members["ce2a822cea30bfca"]) def test_set_plain(self): """ Can set a value """ @@ -179,7 +179,7 @@ def test_set_plain(self): self._mock_api(200, d) res = self.client.write('/testkey', 'test') - self.assertEquals(res, etcd.EtcdResult(**d)) + self.assertEqual(res, etcd.EtcdResult(**d)) def test_update(self): """Can update a result.""" @@ -198,7 +198,7 @@ def test_update(self): d['node']['value'] = 'ciao' self._mock_api(200,d) newres = self.client.update(res) - self.assertEquals(newres.value, 'ciao') + self.assertEqual(newres.value, 'ciao') def test_newkey(self): """ Can set a new value """ @@ -215,7 +215,7 @@ def test_newkey(self): self._mock_api(201, d) res = self.client.write('/testkey', 'test') d['node']['newKey'] = True - self.assertEquals(res, etcd.EtcdResult(**d)) + self.assertEqual(res, etcd.EtcdResult(**d)) def test_refresh(self): """ Can refresh a new value """ @@ -232,7 +232,7 @@ def test_refresh(self): self._mock_api(200, d) res = self.client.refresh('/testkey', ttl=600) - self.assertEquals(res, etcd.EtcdResult(**d)) + self.assertEqual(res, etcd.EtcdResult(**d)) def test_not_found_response(self): """ Can handle server not found response """ @@ -253,7 +253,7 @@ def test_compare_and_swap(self): self._mock_api(200, d) res = self.client.write('/testkey', 'test', prevValue='test_old') - self.assertEquals(res, etcd.EtcdResult(**d)) + self.assertEqual(res, etcd.EtcdResult(**d)) def test_compare_and_swap_failure(self): """ Exception will be raised if prevValue != value in test_set """ @@ -279,7 +279,7 @@ def test_set_append(self): } self._mock_api(201, d) res = self.client.write('/testdir', 'test') - self.assertEquals(res.createdIndex, 190) + self.assertEqual(res.createdIndex, 190) def test_set_dir_with_value(self): """ Creating a directory with a value raises an error. """ @@ -298,7 +298,7 @@ def test_delete(self): } self._mock_api(200, d) res = self.client.delete('/testKey') - self.assertEquals(res, etcd.EtcdResult(**d)) + self.assertEqual(res, etcd.EtcdResult(**d)) def test_pop(self): """ Can pop a value """ @@ -316,7 +316,7 @@ def test_pop(self): self._mock_api(200, d) res = self.client.pop(d['node']['key']) - self.assertEquals({attr: getattr(res, attr) for attr in dir(res) + self.assertEqual({attr: getattr(res, attr) for attr in dir(res) if attr in etcd.EtcdResult._node_props}, d['prevNode']) self.assertEqual(res.value, d['prevNode']['value']) @@ -332,7 +332,7 @@ def test_read(self): } self._mock_api(200, d) res = self.client.read('/testKey') - self.assertEquals(res, etcd.EtcdResult(**d)) + self.assertEqual(res, etcd.EtcdResult(**d)) def test_get_dir(self): """Can get values in dirs""" @@ -358,7 +358,7 @@ def test_get_dir(self): } self._mock_api(200, d) res = self.client.read('/testDir', recursive=True) - self.assertEquals(res, etcd.EtcdResult(**d)) + self.assertEqual(res, etcd.EtcdResult(**d)) def test_not_in(self): """ Can check if key is not in client """ @@ -390,7 +390,7 @@ def test_watch(self): } self._mock_api(200, d) res = self.client.read('/testkey', wait=True) - self.assertEquals(res, etcd.EtcdResult(**d)) + self.assertEqual(res, etcd.EtcdResult(**d)) def test_watch_index(self): """ Can watch a key starting from the given Index """ @@ -404,7 +404,7 @@ def test_watch_index(self): } self._mock_api(200, d) res = self.client.read('/testkey', wait=True, waitIndex=True) - self.assertEquals(res, etcd.EtcdResult(**d)) + self.assertEqual(res, etcd.EtcdResult(**d)) class TestClientRequest(TestClientApiInterface): From a5db6db6792aeb87b7de11fd5a79647f57627de0 Mon Sep 17 00:00:00 2001 From: TCgogogo <1073709473@qq.com> Date: Fri, 11 Oct 2019 23:45:40 +0800 Subject: [PATCH 087/101] fix EtcdWatchTimeOut does not been raised issue --- src/etcd/lock.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/etcd/lock.py b/src/etcd/lock.py index a77aaa76..8baa1680 100644 --- a/src/etcd/lock.py +++ b/src/etcd/lock.py @@ -61,6 +61,11 @@ def acquire(self, blocking=True, lock_ttl=3600, timeout=0): :param blocking Block until the lock is obtained, or timeout is reached :param lock_ttl The duration of the lock we acquired, set to None for eternal locks :param timeout The time to wait before giving up on getting a lock + + Raises: + etcd.EtcdLockExpired: If lock expired when try to acquire. + + etcd.EtcdWatchTimeOut: If timeout is reached. """ # First of all try to write, if our lock is not present. if not self._find_lock(): @@ -125,7 +130,7 @@ def _acquired(self, blocking=True, timeout=0): except etcd.EtcdKeyNotFound: _log.debug("Key %s not present anymore, moving on", watch_key) return self._acquired(blocking=True, timeout=timeout) - except etcd.EtcdLockExpired as e: + except etcd.EtcdLockExpired | etcd.EtcdWatchTimeOut as e: raise e except etcd.EtcdException: _log.exception("Unexpected exception") From 4a51a52122ad6739db987f5cb617b37d915d8e82 Mon Sep 17 00:00:00 2001 From: TCgogogo <1073709473@qq.com> Date: Fri, 11 Oct 2019 23:55:56 +0800 Subject: [PATCH 088/101] change style --- src/etcd/lock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etcd/lock.py b/src/etcd/lock.py index 8baa1680..4a5bc007 100644 --- a/src/etcd/lock.py +++ b/src/etcd/lock.py @@ -130,7 +130,7 @@ def _acquired(self, blocking=True, timeout=0): except etcd.EtcdKeyNotFound: _log.debug("Key %s not present anymore, moving on", watch_key) return self._acquired(blocking=True, timeout=timeout) - except etcd.EtcdLockExpired | etcd.EtcdWatchTimeOut as e: + except (etcd.EtcdLockExpired, etcd.EtcdWatchTimeOut) as e: raise e except etcd.EtcdException: _log.exception("Unexpected exception") From 696865ce2725168791a0fcc0fc1f59466f0a4749 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste BESNARD Date: Mon, 10 Jul 2017 22:18:01 +0200 Subject: [PATCH 089/101] AUTH : Fix version check to avoid crashing on API detection through version Using the GIT version of Etcd, you get versions such as "3.2.0-rc.1+git" Previous code led to this error: ----- File "/usr/lib/python2.7/site-packages/etcd/auth.py", line 25, in legacy_api major, minor, _ = map(int, self.client.version.split('.')) ValueError: invalid literal for int() with base 10: '0-rc' ----- --- src/etcd/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etcd/auth.py b/src/etcd/auth.py index c5c73465..0ed196ee 100644 --- a/src/etcd/auth.py +++ b/src/etcd/auth.py @@ -21,7 +21,7 @@ def __init__(self, client, name): def legacy_api(self): if self._legacy_api is None: # The auth API has changed between 2.2 and 2.3, true story! - major, minor, _ = map(int, self.client.version.split('.')) + major, minor = map(int, self.client.version[:3].split('.')) self._legacy_api = (major < 3 and minor < 3) return self._legacy_api From 4babc1f3646a782e89edef4b14a4c1acd0439ad0 Mon Sep 17 00:00:00 2001 From: Harald Laabs Date: Sun, 18 Jun 2023 18:07:42 +0200 Subject: [PATCH 090/101] disable ssl cert validation if ca_cert is not set despite new urllib3 defaults --- src/etcd/client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/etcd/client.py b/src/etcd/client.py index 12c1863d..aeae142e 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -172,6 +172,9 @@ def uri(protocol, host, port): if ca_cert: kw['ca_certs'] = ca_cert kw['cert_reqs'] = ssl.CERT_REQUIRED + else: + kw['cert_reqs'] = ssl.CERT_NONE + urllib3.disable_warnings() self.username = None self.password = None From 5d960c2cb01b08b56fe29d6768c77d383886182b Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Mon, 30 Oct 2023 07:48:48 +0100 Subject: [PATCH 091/101] Fix typo --- src/etcd/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etcd/client.py b/src/etcd/client.py index aeae142e..eceed562 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -92,7 +92,7 @@ def __init__( cert (mixed): If a string, the whole ssl client certificate; if a tuple, the cert and key file names. - ca_cert (str): The ca certificate. If pressent it will enable + ca_cert (str): The ca certificate. If present it will enable validation. username (str): username for etcd authentication. From 1e505966c464ffa1dcebbbfb2620dc1cfa0c4bc4 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Mon, 30 Oct 2023 07:56:09 +0100 Subject: [PATCH 092/101] Format using black --- bootstrap.py | 102 ++-- docs-source/conf.py | 146 +++--- setup.py | 38 +- src/etcd/__init__.py | 59 ++- src/etcd/auth.py | 129 +++--- src/etcd/client.py | 338 +++++++------- src/etcd/lock.py | 16 +- src/etcd/tests/integration/helpers.py | 199 ++++---- src/etcd/tests/integration/test_simple.py | 252 +++++----- src/etcd/tests/integration/test_ssl.py | 156 ++++--- src/etcd/tests/test_auth.py | 85 ++-- src/etcd/tests/unit/__init__.py | 6 +- src/etcd/tests/unit/test_client.py | 116 +++-- src/etcd/tests/unit/test_lock.py | 176 ++++--- src/etcd/tests/unit/test_old_request.py | 455 ++++++++++-------- src/etcd/tests/unit/test_request.py | 541 +++++++++++----------- src/etcd/tests/unit/test_result.py | 112 +++-- 17 files changed, 1533 insertions(+), 1393 deletions(-) diff --git a/bootstrap.py b/bootstrap.py index 1b28969a..f7d49e2f 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -27,7 +27,7 @@ tmpeggs = tempfile.mkdtemp() -usage = '''\ +usage = """\ [DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options] Bootstraps a buildout-based project. @@ -37,25 +37,34 @@ Note that by using --find-links to point to local resources, you can keep this script from going over the network. -''' +""" parser = OptionParser(usage=usage) parser.add_option("-v", "--version", help="use a specific zc.buildout version") -parser.add_option("-t", "--accept-buildout-test-releases", - dest='accept_buildout_test_releases', - action="store_true", default=False, - help=("Normally, if you do not specify a --version, the " - "bootstrap script and buildout gets the newest " - "*final* versions of zc.buildout and its recipes and " - "extensions for you. If you use this flag, " - "bootstrap and buildout will get the newest releases " - "even if they are alphas or betas.")) -parser.add_option("-c", "--config-file", - help=("Specify the path to the buildout configuration " - "file to be used.")) -parser.add_option("-f", "--find-links", - help=("Specify a URL to search for buildout releases")) +parser.add_option( + "-t", + "--accept-buildout-test-releases", + dest="accept_buildout_test_releases", + action="store_true", + default=False, + help=( + "Normally, if you do not specify a --version, the " + "bootstrap script and buildout gets the newest " + "*final* versions of zc.buildout and its recipes and " + "extensions for you. If you use this flag, " + "bootstrap and buildout will get the newest releases " + "even if they are alphas or betas." + ), +) +parser.add_option( + "-c", + "--config-file", + help=("Specify the path to the buildout configuration " "file to be used."), +) +parser.add_option( + "-f", "--find-links", help=("Specify a URL to search for buildout releases") +) options, args = parser.parse_args() @@ -76,14 +85,17 @@ from urllib2 import urlopen # XXX use a more permanent ez_setup.py URL when available. - exec(urlopen('https://bitbucket.org/pypa/setuptools/raw/0.7.2/ez_setup.py' - ).read(), ez) + exec( + urlopen("https://bitbucket.org/pypa/setuptools/raw/0.7.2/ez_setup.py").read(), + ez, + ) setup_args = dict(to_dir=tmpeggs, download_delay=0) - ez['use_setuptools'](**setup_args) + ez["use_setuptools"](**setup_args) if to_reload: reload(pkg_resources) import pkg_resources + # This does not (always?) update the default working set. We will # do it. for path in sys.path: @@ -95,36 +107,43 @@ ws = pkg_resources.working_set -cmd = [sys.executable, '-c', - 'from setuptools.command.easy_install import main; main()', - '-mZqNxd', tmpeggs] +cmd = [ + sys.executable, + "-c", + "from setuptools.command.easy_install import main; main()", + "-mZqNxd", + tmpeggs, +] find_links = os.environ.get( - 'bootstrap-testing-find-links', - options.find_links or - ('http://downloads.buildout.org/' - if options.accept_buildout_test_releases else None) - ) + "bootstrap-testing-find-links", + options.find_links + or ( + "http://downloads.buildout.org/" + if options.accept_buildout_test_releases + else None + ), +) if find_links: - cmd.extend(['-f', find_links]) + cmd.extend(["-f", find_links]) -setuptools_path = ws.find( - pkg_resources.Requirement.parse('setuptools')).location +setuptools_path = ws.find(pkg_resources.Requirement.parse("setuptools")).location -requirement = 'zc.buildout' +requirement = "zc.buildout" version = options.version if version is None and not options.accept_buildout_test_releases: # Figure out the most recent final version of zc.buildout. import setuptools.package_index - _final_parts = '*final-', '*final' + + _final_parts = "*final-", "*final" def _final_version(parsed_version): for part in parsed_version: - if (part[:1] == '*') and (part not in _final_parts): + if (part[:1] == "*") and (part not in _final_parts): return False return True - index = setuptools.package_index.PackageIndex( - search_path=[setuptools_path]) + + index = setuptools.package_index.PackageIndex(search_path=[setuptools_path]) if find_links: index.add_find_links((find_links,)) req = pkg_resources.Requirement.parse(requirement) @@ -143,14 +162,13 @@ def _final_version(parsed_version): best.sort() version = best[-1].version if version: - requirement = '=='.join((requirement, version)) + requirement = "==".join((requirement, version)) cmd.append(requirement) import subprocess + if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0: - raise Exception( - "Failed to execute command:\n%s", - repr(cmd)[1:-1]) + raise Exception("Failed to execute command:\n%s", repr(cmd)[1:-1]) ###################################################################### # Import and run buildout @@ -159,12 +177,12 @@ def _final_version(parsed_version): ws.require(requirement) import zc.buildout.buildout -if not [a for a in args if '=' not in a]: - args.append('bootstrap') +if not [a for a in args if "=" not in a]: + args.append("bootstrap") # if -c was provided, we push it back into args for buildout' main function if options.config_file is not None: - args[0:0] = ['-c', options.config_file] + args[0:0] = ["-c", options.config_file] zc.buildout.buildout.main(args) shutil.rmtree(tmpeggs) diff --git a/docs-source/conf.py b/docs-source/conf.py index 5148c23a..8b94a705 100644 --- a/docs-source/conf.py +++ b/docs-source/conf.py @@ -2,6 +2,7 @@ import sys, os + class Mock(object): def __init__(self, *args, **kwargs): pass @@ -11,8 +12,8 @@ def __call__(self, *args, **kwargs): @classmethod def __getattr__(cls, name): - if name in ('__file__', '__path__'): - return '/dev/null' + if name in ("__file__", "__path__"): + return "/dev/null" elif name[0] == name[0].upper(): mockType = type(name, (), {}) mockType.__module__ = __name__ @@ -20,216 +21,211 @@ def __getattr__(cls, name): else: return Mock() -MOCK_MODULES = ['urllib3'] + +MOCK_MODULES = ["urllib3"] for mod_name in MOCK_MODULES: sys.modules[mod_name] = Mock() # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('../src')) +sys.path.insert(0, os.path.abspath("../src")) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc'] +extensions = ["sphinx.ext.autodoc"] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'python-etcd' -copyright = u'2013-2015 Jose Plana, Giuseppe Lavagetto' +project = "python-etcd" +copyright = "2013-2015 Jose Plana, Giuseppe Lavagetto" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '0.4' +version = "0.4" # The full version, including alpha/beta/rc tags. -release = '0.4.3' +release = "0.4.3" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinxdoc' +html_theme = "sphinxdoc" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'python-etcddoc' +htmlhelp_basename = "python-etcddoc" # -- Options for LaTeX output -------------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'python-etcd.tex', u'python-etcd Documentation', - u'Jose Plana', 'manual'), + ("index", "python-etcd.tex", "python-etcd Documentation", "Jose Plana", "manual"), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'python-etcd', u'python-etcd Documentation', - [u'Jose Plana'], 1) -] +man_pages = [("index", "python-etcd", "python-etcd Documentation", ["Jose Plana"], 1)] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ @@ -238,16 +234,22 @@ def __getattr__(cls, name): # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'python-etcd', u'python-etcd Documentation', - u'Jose Plana', 'python-etcd', 'One line description of project.', - 'Miscellaneous'), + ( + "index", + "python-etcd", + "python-etcd Documentation", + "Jose Plana", + "python-etcd", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' diff --git a/setup.py b/setup.py index a4c6d01d..8f6e9484 100644 --- a/setup.py +++ b/setup.py @@ -2,46 +2,38 @@ import sys, os here = os.path.abspath(os.path.dirname(__file__)) -README = open(os.path.join(here, 'README.rst')).read() -NEWS = open(os.path.join(here, 'NEWS.txt')).read() +README = open(os.path.join(here, "README.rst")).read() +NEWS = open(os.path.join(here, "NEWS.txt")).read() -version = '0.4.5' +version = "0.5.0" -install_requires = [ - 'urllib3>=1.7.1', - 'dnspython>=1.13.0' -] +install_requires = ["urllib3>=1.7.1", "dnspython>=1.13.0"] -test_requires = [ - 'mock', - 'nose', - 'pyOpenSSL>=0.14' -] +test_requires = ["mock", "nose", "pyOpenSSL>=0.14"] setup( - name='python-etcd', + name="python-etcd", version=version, description="A python client for etcd", - long_description=README + '\n\n' + NEWS, + long_description=README + "\n\n" + NEWS, classifiers=[ "Topic :: System :: Distributed Computing", "Topic :: Software Development :: Libraries", "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "Topic :: Database :: Front-Ends", ], - keywords='etcd raft distributed log api client', - author='Jose Plana', - author_email='jplana@gmail.com', - url='http://github.com/jplana/python-etcd', - license='MIT', - packages=find_packages('src'), - package_dir = {'': 'src'}, + keywords="etcd raft distributed log api client", + author="Jose Plana", + author_email="jplana@gmail.com", + url="http://github.com/jplana/python-etcd", + license="MIT", + packages=find_packages("src"), + package_dir={"": "src"}, include_package_data=True, zip_safe=False, install_requires=install_requires, tests_require=test_requires, - test_suite='nose.collector', + test_suite="nose.collector", ) diff --git a/src/etcd/__init__.py b/src/etcd/__init__.py index 33b1d679..d716e9be 100644 --- a/src/etcd/__init__.py +++ b/src/etcd/__init__.py @@ -13,19 +13,21 @@ class NullHandler(logging.Handler): def emit(self, record): pass + + _log.addHandler(NullHandler()) class EtcdResult(object): _node_props = { - 'key': None, - 'value': None, - 'expiration': None, - 'ttl': None, - 'modifiedIndex': None, - 'createdIndex': None, - 'newKey': False, - 'dir': False, + "key": None, + "value": None, + "expiration": None, + "ttl": None, + "modifiedIndex": None, + "createdIndex": None, + "newKey": False, + "dir": False, } def __init__(self, action=None, node=None, prevNode=None, **kwdargs): @@ -41,16 +43,16 @@ def __init__(self, action=None, node=None, prevNode=None, **kwdargs): """ self.action = action - for (key, default) in self._node_props.items(): + for key, default in self._node_props.items(): if key in node: setattr(self, key, node[key]) else: setattr(self, key, default) self._children = [] - if self.dir and 'nodes' in node: + if self.dir and "nodes" in node: # We keep the data in raw format, converting them only when needed - self._children = node['nodes'] + self._children = node["nodes"] if prevNode: self._prev_node = EtcdResult(None, node=prevNode) @@ -60,8 +62,8 @@ def __init__(self, action=None, node=None, prevNode=None, **kwdargs): def parse_headers(self, response): headers = response.getheaders() - self.etcd_index = int(headers.get('x-etcd-index', 1)) - self.raft_index = int(headers.get('x-raft-index', 1)) + self.etcd_index = int(headers.get("x-etcd-index", 1)) + self.raft_index = int(headers.get("x-raft-index", 1)) def get_subtree(self, leaves_only=False): """ @@ -73,7 +75,7 @@ def get_subtree(self, leaves_only=False): """ if not self._children: - #if the current result is a leaf, return itself + # if the current result is a leaf, return itself yield self return else: @@ -92,7 +94,7 @@ def leaves(self): @property def children(self): - """ Deprecated, use EtcdResult.leaves instead """ + """Deprecated, use EtcdResult.leaves instead""" return self.leaves def __eq__(self, other): @@ -120,6 +122,7 @@ class EtcdException(Exception): """ Generic Etcd Exception. """ + def __init__(self, message=None, payload=None): super(EtcdException, self).__init__(message) self.payload = payload @@ -129,6 +132,7 @@ class EtcdValueError(EtcdException, ValueError): """ Base class for Etcd value-related errors. """ + pass @@ -136,6 +140,7 @@ class EtcdCompareFailed(EtcdValueError): """ Compare-and-swap failure """ + pass @@ -145,6 +150,7 @@ class EtcdClusterIdChanged(EtcdException): with a backup. Raised to prevent waiting on an etcd_index that was only valid on the old cluster. """ + pass @@ -152,6 +158,7 @@ class EtcdKeyError(EtcdException): """ Etcd Generic KeyError Exception """ + pass @@ -159,6 +166,7 @@ class EtcdKeyNotFound(EtcdKeyError): """ Etcd key not found exception (100) """ + pass @@ -166,6 +174,7 @@ class EtcdNotFile(EtcdKeyError): """ Etcd not a file exception (102) """ + pass @@ -173,6 +182,7 @@ class EtcdNotDir(EtcdKeyError): """ Etcd not a directory exception (104) """ + pass @@ -180,6 +190,7 @@ class EtcdAlreadyExist(EtcdKeyError): """ Etcd already exist exception (105) """ + pass @@ -187,6 +198,7 @@ class EtcdEventIndexCleared(EtcdException): """ Etcd event index is outdated and cleared exception (401) """ + pass @@ -194,9 +206,9 @@ class EtcdConnectionFailed(EtcdException): """ Connection to etcd failed. """ + def __init__(self, message=None, payload=None, cause=None): - super(EtcdConnectionFailed, self).__init__(message=message, - payload=payload) + super(EtcdConnectionFailed, self).__init__(message=message, payload=payload) self.cause = cause @@ -204,6 +216,7 @@ class EtcdInsufficientPermissions(EtcdException): """ Request failed because of insufficient permissions. """ + pass @@ -211,6 +224,7 @@ class EtcdWatchTimedOut(EtcdConnectionFailed): """ A watch timed out without returning a result. """ + pass @@ -218,6 +232,7 @@ class EtcdWatcherCleared(EtcdException): """ Watcher is cleared due to etcd recovery. """ + pass @@ -225,6 +240,7 @@ class EtcdLeaderElectionInProgress(EtcdException): """ Request failed due to in-progress leader election. """ + pass @@ -232,6 +248,7 @@ class EtcdRootReadOnly(EtcdKeyError): """ Operation is not valid on the root, which is read only. """ + pass @@ -239,6 +256,7 @@ class EtcdDirNotEmpty(EtcdValueError): """ Directory not empty. """ + pass @@ -246,6 +264,7 @@ class EtcdLockExpired(EtcdException): """ Our lock apparently expired while we were trying to acquire it. """ + pass @@ -263,7 +282,6 @@ class EtcdError(object): 108: EtcdDirNotEmpty, # 109: Non-public: existing peer addr. 110: EtcdInsufficientPermissions, - 200: EtcdValueError, # Not part of v2 201: EtcdValueError, 202: EtcdValueError, @@ -275,10 +293,8 @@ class EtcdError(object): 208: EtcdValueError, 209: EtcdValueError, 210: EtcdValueError, - # 300: Non-public: Raft internal error. 301: EtcdLeaderElectionInProgress, - 400: EtcdWatcherCleared, 401: EtcdEventIndexCleared, } @@ -293,7 +309,7 @@ def handle(cls, payload): error_code = payload.get("errorCode") message = payload.get("message") cause = payload.get("cause") - msg = '{} : {}'.format(message, cause) + msg = "{} : {}".format(message, cause) status = payload.get("status") # Some general status handling, as # not all endpoints return coherent error messages @@ -312,6 +328,7 @@ def handle(cls, payload): # Blatantly copied from requests. try: from urllib3.contrib import pyopenssl + pyopenssl.inject_into_urllib3() except ImportError: pass diff --git a/src/etcd/auth.py b/src/etcd/auth.py index 0ed196ee..a1930681 100644 --- a/src/etcd/auth.py +++ b/src/etcd/auth.py @@ -7,13 +7,12 @@ class EtcdAuthBase(object): - entity = 'example' + entity = "example" def __init__(self, client, name): self.client = client self.name = name - self.uri = "{}/auth/{}s/{}".format(self.client.version_prefix, - self.entity, self.name) + self.uri = "{}/auth/{}s/{}".format(self.client.version_prefix, self.entity, self.name) # This will be lazily evaluated if not manually set self._legacy_api = None @@ -21,21 +20,19 @@ def __init__(self, client, name): def legacy_api(self): if self._legacy_api is None: # The auth API has changed between 2.2 and 2.3, true story! - major, minor = map(int, self.client.version[:3].split('.')) - self._legacy_api = (major < 3 and minor < 3) + major, minor = map(int, self.client.version[:3].split(".")) + self._legacy_api = major < 3 and minor < 3 return self._legacy_api - @property def names(self): key = "{}s".format(self.entity) uri = "{}/auth/{}".format(self.client.version_prefix, key) response = self.client.api_execute(uri, self.client._MGET) if self.legacy_api: - return json.loads(response.data.decode('utf-8'))[key] + return json.loads(response.data.decode("utf-8"))[key] else: - return [obj[self.entity] - for obj in json.loads(response.data.decode('utf-8'))[key]] + return [obj[self.entity] for obj in json.loads(response.data.decode("utf-8"))[key]] def read(self): try: @@ -47,11 +44,14 @@ def read(self): _log.info("%s '%s' not found", self.entity, self.name) raise except Exception as e: - _log.error("Failed to fetch %s in %s%s: %r", - self.entity, self.client._base_uri, - self.client.version_prefix, e) - raise etcd.EtcdException( - "Could not fetch {} '{}'".format(self.entity, self.name)) + _log.error( + "Failed to fetch %s in %s%s: %r", + self.entity, + self.client._base_uri, + self.client.version_prefix, + e, + ) + raise etcd.EtcdException("Could not fetch {} '{}'".format(self.entity, self.name)) self._from_net(response.data) @@ -63,9 +63,7 @@ def write(self): r = None try: for payload in self._to_net(r): - response = self.client.api_execute_json(self.uri, - self.client._MPUT, - params=payload) + response = self.client.api_execute_json(self.uri, self.client._MPUT, params=payload) # This will fail if the response is an error self._from_net(response.data) except etcd.EtcdInsufficientPermissions as e: @@ -75,8 +73,8 @@ def write(self): _log.error("Failed to write %s '%s'", self.entity, self.name) # TODO: fine-grained exception handling raise etcd.EtcdException( - "Could not write {} '{}': {}".format(self.entity, - self.name, e)) + "Could not write {} '{}': {}".format(self.entity, self.name, e) + ) def delete(self): try: @@ -88,10 +86,14 @@ def delete(self): _log.info("%s '%s' not found", self.entity, self.name) raise except Exception as e: - _log.error("Failed to delete %s in %s%s: %r", - self.entity, self._base_uri, self.version_prefix, e) - raise etcd.EtcdException( - "Could not delete {} '{}'".format(self.entity, self.name)) + _log.error( + "Failed to delete %s in %s%s: %r", + self.entity, + self._base_uri, + self.version_prefix, + e, + ) + raise etcd.EtcdException("Could not delete {} '{}'".format(self.entity, self.name)) def _from_net(self, data): raise NotImplementedError() @@ -108,7 +110,8 @@ def new(cls, client, data): class EtcdUser(EtcdAuthBase): """Class to manage in a orm-like way etcd users""" - entity = 'user' + + entity = "user" def __init__(self, client, name): super(EtcdUser, self).__init__(client, name) @@ -116,8 +119,8 @@ def __init__(self, client, name): self._password = None def _from_net(self, data): - d = json.loads(data.decode('utf-8')) - roles = d.get('roles', []) + d = json.loads(data.decode("utf-8")) + roles = d.get("roles", []) try: self.roles = roles except TypeError: @@ -126,13 +129,18 @@ def _from_net(self, data): # Specifically, PUT responses are the same as before... if self.legacy_api: raise - self.roles = [obj['role'] for obj in roles] - self.name = d.get('user') + self.roles = [obj["role"] for obj in roles] + self.name = d.get("user") def _to_net(self, prevobj=None): if prevobj is None: - retval = [{"user": self.name, "password": self._password, - "roles": list(self.roles)}] + retval = [ + { + "user": self.name, + "password": self._password, + "roles": list(self.roles), + } + ] else: retval = [] if self._password: @@ -170,9 +178,8 @@ def __str__(self): return json.dumps(self._to_net()[0]) - class EtcdRole(EtcdAuthBase): - entity = 'role' + entity = "role" def __init__(self, client, name): super(EtcdRole, self).__init__(client, name) @@ -180,8 +187,8 @@ def __init__(self, client, name): self._write_paths = set() def _from_net(self, data): - d = json.loads(data.decode('utf-8')) - self.name = d.get('role') + d = json.loads(data.decode("utf-8")) + self.name = d.get("role") try: kv = d["permissions"]["kv"] @@ -190,50 +197,48 @@ def _from_net(self, data): self._write_paths = set() return - self._read_paths = set(kv.get('read', [])) - self._write_paths = set(kv.get('write', [])) + self._read_paths = set(kv.get("read", [])) + self._write_paths = set(kv.get("write", [])) def _to_net(self, prevobj=None): retval = [] if prevobj is None: - retval.append({ - "role": self.name, - "permissions": + retval.append( { - "kv": - { - "read": list(self._read_paths), - "write": list(self._write_paths) - } + "role": self.name, + "permissions": { + "kv": { + "read": list(self._read_paths), + "write": list(self._write_paths), + } + }, } - }) + ) else: to_grant = { - 'read': list(self._read_paths - prevobj._read_paths), - 'write': list(self._write_paths - prevobj._write_paths) + "read": list(self._read_paths - prevobj._read_paths), + "write": list(self._write_paths - prevobj._write_paths), } to_revoke = { - 'read': list(prevobj._read_paths - self._read_paths), - 'write': list(prevobj._write_paths - self._write_paths) + "read": list(prevobj._read_paths - self._read_paths), + "write": list(prevobj._write_paths - self._write_paths), } if [path for sublist in to_revoke.values() for path in sublist]: - retval.append({'role': self.name, 'revoke': {'kv': to_revoke}}) + retval.append({"role": self.name, "revoke": {"kv": to_revoke}}) if [path for sublist in to_grant.values() for path in sublist]: - retval.append({'role': self.name, 'grant': {'kv': to_grant}}) + retval.append({"role": self.name, "grant": {"kv": to_grant}}) return retval def grant(self, path, permission): - if permission.upper().find('R') >= 0: + if permission.upper().find("R") >= 0: self._read_paths.add(path) - if permission.upper().find('W') >= 0: + if permission.upper().find("W") >= 0: self._write_paths.add(path) def revoke(self, path, permission): - if permission.upper().find('R') >= 0 and \ - path in self._read_paths: + if permission.upper().find("R") >= 0 and path in self._read_paths: self._read_paths.remove(path) - if permission.upper().find('W') >= 0 and \ - path in self._write_paths: + if permission.upper().find("W") >= 0 and path in self._write_paths: self._write_paths.remove(path) @property @@ -241,12 +246,12 @@ def acls(self): perms = {} try: for path in self._read_paths: - perms[path] = 'R' + perms[path] = "R" for path in self._write_paths: if path in perms: - perms[path] += 'W' + perms[path] += "W" else: - perms[path] = 'W' + perms[path] = "W" except: pass return perms @@ -259,7 +264,7 @@ def acls(self, acls): self.grant(path, permission) def __str__(self): - return json.dumps({"role": self.name, 'acls': self.acls}) + return json.dumps({"role": self.name, "acls": self.acls}) class Auth(object): @@ -270,7 +275,7 @@ def __init__(self, client): @property def active(self): resp = self.client.api_execute(self.uri, self.client._MGET) - return json.loads(resp.data.decode('utf-8'))['enabled'] + return json.loads(resp.data.decode("utf-8"))["enabled"] @active.setter def active(self, value): diff --git a/src/etcd/client.py b/src/etcd/client.py index eceed562..a0117574 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -7,6 +7,7 @@ """ import logging + try: # Python 3 from http.client import HTTPException @@ -40,34 +41,34 @@ class Client(object): Client for etcd, the distributed log service using raft. """ - _MGET = 'GET' - _MPUT = 'PUT' - _MPOST = 'POST' - _MDELETE = 'DELETE' - _comparison_conditions = set(('prevValue', 'prevIndex', 'prevExist', 'refresh')) - _read_options = set(('recursive', 'wait', 'waitIndex', 'sorted', 'quorum')) - _del_conditions = set(('prevValue', 'prevIndex')) + _MGET = "GET" + _MPUT = "PUT" + _MPOST = "POST" + _MDELETE = "DELETE" + _comparison_conditions = set(("prevValue", "prevIndex", "prevExist", "refresh")) + _read_options = set(("recursive", "wait", "waitIndex", "sorted", "quorum")) + _del_conditions = set(("prevValue", "prevIndex")) http = None def __init__( - self, - host='127.0.0.1', - port=4001, - srv_domain=None, - version_prefix='/v2', - read_timeout=60, - allow_redirect=True, - protocol='http', - cert=None, - ca_cert=None, - username=None, - password=None, - allow_reconnect=False, - use_proxies=False, - expected_cluster_id=None, - per_host_pool_size=10, - lock_prefix="/_locks" + self, + host="127.0.0.1", + port=4001, + srv_domain=None, + version_prefix="/v2", + read_timeout=60, + allow_redirect=True, + protocol="http", + cert=None, + ca_cert=None, + username=None, + password=None, + allow_reconnect=False, + use_proxies=False, + expected_cluster_id=None, + per_host_pool_size=10, + lock_prefix="/_locks", ): """ Initialize the client. @@ -124,13 +125,12 @@ def __init__( try: host = self._discover(srv_domain) except Exception as e: - _log.error("Could not discover the etcd hosts from %s: %s", - srv_domain, e) + _log.error("Could not discover the etcd hosts from %s: %s", srv_domain, e) self._protocol = protocol def uri(protocol, host, port): - return '%s://%s:%d' % (protocol, host, port) + return "%s://%s:%d" % (protocol, host, port) if not isinstance(host, tuple): self._machines_cache = [] @@ -138,7 +138,9 @@ def uri(protocol, host, port): else: if not allow_reconnect: _log.error("List of hosts incompatible with allow_reconnect.") - raise etcd.EtcdException("A list of hosts to connect to was given, but reconnection not allowed?") + raise etcd.EtcdException( + "A list of hosts to connect to was given, but reconnection not allowed?" + ) self._machines_cache = [uri(self._protocol, *conn) for conn in host] self._base_uri = self._machines_cache.pop(0) @@ -153,27 +155,25 @@ def uri(protocol, host, port): # SSL Client certificate support - kw = { - 'maxsize': per_host_pool_size - } + kw = {"maxsize": per_host_pool_size} if self._read_timeout > 0: - kw['timeout'] = self._read_timeout + kw["timeout"] = self._read_timeout if cert: if isinstance(cert, tuple): # Key and cert are separate - kw['cert_file'] = cert[0] - kw['key_file'] = cert[1] + kw["cert_file"] = cert[0] + kw["key_file"] = cert[1] else: # combined certificate - kw['cert_file'] = cert + kw["cert_file"] = cert if ca_cert: - kw['ca_certs'] = ca_cert - kw['cert_reqs'] = ssl.CERT_REQUIRED + kw["ca_certs"] = ca_cert + kw["cert_reqs"] = ssl.CERT_REQUIRED else: - kw['cert_reqs'] = ssl.CERT_NONE + kw["cert_reqs"] = ssl.CERT_NONE urllib3.disable_warnings() self.username = None @@ -182,9 +182,9 @@ def uri(protocol, host, port): self.username = username self.password = password elif username: - _log.warning('Username provided without password, both are required for authentication') + _log.warning("Username provided without password, both are required for authentication") elif password: - _log.warning('Password provided without username, both are required for authentication') + _log.warning("Password provided without username, both are required for authentication") self.http = urllib3.PoolManager(num_pools=10, **kw) @@ -204,12 +204,10 @@ def uri(protocol, host, port): # extend the list given to the client with what we get # from self.machines if not self._use_proxies: - self._machines_cache = list(set(self._machines_cache) | - set(self.machines)) + self._machines_cache = list(set(self._machines_cache) | set(self.machines)) if self._base_uri in self._machines_cache: self._machines_cache.remove(self._base_uri) - _log.debug("Machines cache initialised to %s", - self._machines_cache) + _log.debug("Machines cache initialised to %s", self._machines_cache) # Versions set to None. They will be set upon first usage. self._version = self._cluster_version = None @@ -219,22 +217,22 @@ def _set_version_info(self): Sets the version information provided by the server. """ # Set the version - data = self.api_execute('/version', self._MGET).data - version_info = json.loads(data.decode('utf-8')) - self._version = version_info['etcdserver'] - self._cluster_version = version_info['etcdcluster'] + data = self.api_execute("/version", self._MGET).data + version_info = json.loads(data.decode("utf-8")) + self._version = version_info["etcdserver"] + self._cluster_version = version_info["etcdcluster"] def _discover(self, domain): srv_names = [ "_etcd-client-ssl._tcp.{}".format(domain), "_etcd-client._tcp.{}".format(domain), "_etcd-ssl._tcp.{}".format(domain), - "_etcd._tcp.{}".format(domain) + "_etcd._tcp.{}".format(domain), ] found = False for srv_name in srv_names: try: - answers = dns.resolver.query(srv_name, 'SRV') + answers = dns.resolver.query(srv_name, "SRV") if len(answers): found = True break @@ -242,15 +240,14 @@ def _discover(self, domain): continue if not found: - raise ValueError('Could not find SRV record for domain {}.'.format(domain)) + raise ValueError("Could not find SRV record for domain {}.".format(domain)) - if re.search('-ssl', srv_name): - self._protocol = 'https' + if re.search("-ssl", srv_name): + self._protocol = "https" hosts = [] for answer in answers: - hosts.append( - (answer.target.to_text(omit_final_dot=True), answer.port)) + hosts.append((answer.target.to_text(omit_final_dot=True), answer.port)) _log.debug("Found %s", hosts) if not len(hosts): raise ValueError("The SRV record is present but no hosts were found") @@ -273,12 +270,12 @@ def base_uri(self): @property def host(self): """Node to connect etcd.""" - return urlparse(self._base_uri).netloc.split(':')[0] + return urlparse(self._base_uri).netloc.split(":")[0] @property def port(self): """Port to connect etcd.""" - return int(urlparse(self._base_uri).netloc.split(':')[1]) + return int(urlparse(self._base_uri).netloc.split(":")[1]) @property def protocol(self): @@ -313,34 +310,41 @@ def machines(self): """ # We can't use api_execute here, or it causes a logical loop try: - uri = self._base_uri + self.version_prefix + '/machines' + uri = self._base_uri + self.version_prefix + "/machines" response = self.http.request( self._MGET, uri, headers=self._get_headers(), timeout=self.read_timeout, - redirect=self.allow_redirect) + redirect=self.allow_redirect, + ) machines = [ - node.strip() for node in - self._handle_server_response(response).data.decode('utf-8').split(',') + node.strip() + for node in self._handle_server_response(response).data.decode("utf-8").split(",") ] _log.debug("Retrieved list of machines: %s", machines) return machines except (HTTPError, HTTPException, socket.error) as e: # We can't get the list of machines, if one server is in the # machines cache, try on it - _log.error("Failed to get list of machines from %s%s: %r", - self._base_uri, self.version_prefix, e) + _log.error( + "Failed to get list of machines from %s%s: %r", + self._base_uri, + self.version_prefix, + e, + ) if self._machines_cache: self._base_uri = self._machines_cache.pop(0) _log.info("Retrying on %s", self._base_uri) # Call myself return self.machines else: - raise etcd.EtcdException("Could not get the list of servers, " - "maybe you provided the wrong " - "host(s) to connect to?") + raise etcd.EtcdException( + "Could not get the list of servers, " + "maybe you provided the wrong " + "host(s) to connect to?" + ) @property def members(self): @@ -352,14 +356,17 @@ def members(self): # Empty the members list self._members = {} try: - data = self.api_execute(self.version_prefix + '/members', - self._MGET).data.decode('utf-8') + data = self.api_execute(self.version_prefix + "/members", self._MGET).data.decode( + "utf-8" + ) res = json.loads(data) - for member in res['members']: - self._members[member['id']] = member + for member in res["members"]: + self._members[member["id"]] = member return self._members except: - raise etcd.EtcdException("Could not get the members list, maybe the cluster has gone away?") + raise etcd.EtcdException( + "Could not get the members list, maybe the cluster has gone away?" + ) @property def leader(self): @@ -371,11 +378,12 @@ def leader(self): {"id":"ce2a822cea30bfca","name":"default","peerURLs":["http://localhost:2380","http://localhost:7001"],"clientURLs":["http://127.0.0.1:4001"]} """ try: - leader = json.loads( - self.api_execute(self.version_prefix + '/stats/self', - self._MGET).data.decode('utf-8')) - return self.members[leader['leaderInfo']['leader']] + self.api_execute(self.version_prefix + "/stats/self", self._MGET).data.decode( + "utf-8" + ) + ) + return self.members[leader["leaderInfo"]["leader"]] except Exception as e: raise etcd.EtcdException("Cannot get leader data: %s" % e) @@ -393,7 +401,7 @@ def leader_stats(self): Returns: dict. the stats of the leader """ - return self._stats('leader') + return self._stats("leader") @property def store_stats(self): @@ -401,15 +409,16 @@ def store_stats(self): Returns: dict. the stats of the kv store """ - return self._stats('store') + return self._stats("store") - def _stats(self, what='self'): - """ Internal method to access the stats endpoints""" - data = self.api_execute(self.version_prefix - + '/stats/' + what, self._MGET).data.decode('utf-8') + def _stats(self, what="self"): + """Internal method to access the stats endpoints""" + data = self.api_execute(self.version_prefix + "/stats/" + what, self._MGET).data.decode( + "utf-8" + ) try: return json.loads(data) - except (TypeError,ValueError): + except (TypeError, ValueError): raise etcd.EtcdException("Cannot parse json data in the response") @property @@ -436,7 +445,7 @@ def key_endpoint(self): """ REST key endpoint. """ - return self.version_prefix + '/keys' + return self.version_prefix + "/keys" def __contains__(self, key): """ @@ -452,7 +461,7 @@ def __contains__(self, key): return False def _sanitize_key(self, key): - if not key.startswith('/'): + if not key.startswith("/"): key = "/{}".format(key) return key @@ -489,23 +498,21 @@ def write(self, key, value, ttl=None, dir=False, append=False, **kwdargs): 'newValue' """ - _log.debug("Writing %s to key %s ttl=%s dir=%s append=%s", - value, key, ttl, dir, append) + _log.debug("Writing %s to key %s ttl=%s dir=%s append=%s", value, key, ttl, dir, append) key = self._sanitize_key(key) params = {} if value is not None: - params['value'] = value + params["value"] = value if ttl is not None: - params['ttl'] = ttl + params["ttl"] = ttl if dir: if value: - raise etcd.EtcdException( - 'Cannot create a directory with a value') - params['dir'] = "true" + raise etcd.EtcdException("Cannot create a directory with a value") + params["dir"] = "true" - for (k, v) in kwdargs.items(): + for k, v in kwdargs.items(): if k in self._comparison_conditions: if type(v) == bool: params[k] = v and "true" or "false" @@ -513,8 +520,8 @@ def write(self, key, value, ttl=None, dir=False, append=False, **kwdargs): params[k] = v method = append and self._MPOST or self._MPUT - if '_endpoint' in kwdargs: - path = kwdargs['_endpoint'] + key + if "_endpoint" in kwdargs: + path = kwdargs["_endpoint"] + key else: path = self.key_endpoint + key @@ -540,7 +547,7 @@ def refresh(self, key, ttl, **kwdargs): Other parameters modifying the write method are accepted as `EtcdClient.write`. """ # overwrite kwdargs' prevExist - kwdargs['prevExist'] = True + kwdargs["prevExist"] = True return self.write(key=key, value=None, ttl=ttl, refresh=True, **kwdargs) def update(self, obj): @@ -557,15 +564,11 @@ def update(self, obj): """ _log.debug("Updating %s to %s.", obj.key, obj.value) - kwdargs = { - 'dir': obj.dir, - 'ttl': obj.ttl, - 'prevExist': True - } + kwdargs = {"dir": obj.dir, "ttl": obj.ttl, "prevExist": True} if not obj.dir: # prevIndex on a dir causes a 'not a file' error. d'oh! - kwdargs['prevIndex'] = obj.modifiedIndex + kwdargs["prevIndex"] = obj.modifiedIndex return self.write(obj.key, obj.value, **kwdargs) def read(self, key, **kwdargs): @@ -604,18 +607,18 @@ def read(self, key, **kwdargs): key = self._sanitize_key(key) params = {} - for (k, v) in kwdargs.items(): + for k, v in kwdargs.items(): if k in self._read_options: if type(v) == bool: params[k] = v and "true" or "false" elif v is not None: params[k] = v - timeout = kwdargs.get('timeout', None) + timeout = kwdargs.get("timeout", None) response = self.api_execute( - self.key_endpoint + key, self._MGET, params=params, - timeout=timeout) + self.key_endpoint + key, self._MGET, params=params, timeout=timeout + ) return self._result_from_response(response) def delete(self, key, recursive=None, dir=None, **kwdargs): @@ -647,23 +650,27 @@ def delete(self, key, recursive=None, dir=None, **kwdargs): '/key' """ - _log.debug("Deleting %s recursive=%s dir=%s extra args=%s", - key, recursive, dir, kwdargs) + _log.debug( + "Deleting %s recursive=%s dir=%s extra args=%s", + key, + recursive, + dir, + kwdargs, + ) key = self._sanitize_key(key) kwds = {} if recursive is not None: - kwds['recursive'] = recursive and "true" or "false" + kwds["recursive"] = recursive and "true" or "false" if dir is not None: - kwds['dir'] = dir and "true" or "false" + kwds["dir"] = dir and "true" or "false" for k in self._del_conditions: if k in kwdargs: kwds[k] = kwdargs[k] _log.debug("Calculated params = %s", kwds) - response = self.api_execute( - self.key_endpoint + key, self._MDELETE, params=kwds) + response = self.api_execute(self.key_endpoint + key, self._MDELETE, params=kwds) return self._result_from_response(response) def pop(self, key, recursive=None, dir=None, **kwdargs): @@ -785,11 +792,9 @@ def watch(self, key, index=None, timeout=None, recursive=None): """ _log.debug("About to wait on key %s, index %s", key, index) if index: - return self.read(key, wait=True, waitIndex=index, timeout=timeout, - recursive=recursive) + return self.read(key, wait=True, waitIndex=index, timeout=timeout, recursive=recursive) else: - return self.read(key, wait=True, timeout=timeout, - recursive=recursive) + return self.read(key, wait=True, timeout=timeout, recursive=recursive) def eternal_watch(self, key, index=None, recursive=None): """ @@ -817,20 +822,19 @@ def eternal_watch(self, key, index=None, recursive=None): yield response def get_lock(self, *args, **kwargs): - raise NotImplementedError('Lock primitives were removed from etcd 2.0') + raise NotImplementedError("Lock primitives were removed from etcd 2.0") @property def election(self): - raise NotImplementedError('Election primitives were removed from etcd 2.0') + raise NotImplementedError("Election primitives were removed from etcd 2.0") def _result_from_response(self, response): - """ Creates an EtcdResult from json dictionary """ + """Creates an EtcdResult from json dictionary""" raw_response = response.data try: - res = json.loads(raw_response.decode('utf-8')) + res = json.loads(raw_response.decode("utf-8")) except (TypeError, ValueError, UnicodeError) as e: - raise etcd.EtcdException( - 'Server response was not valid JSON: %r' % e) + raise etcd.EtcdException("Server response was not valid JSON: %r" % e) try: r = etcd.EtcdResult(**res) if response.status == 201: @@ -838,19 +842,19 @@ def _result_from_response(self, response): r.parse_headers(response) return r except Exception as e: - raise etcd.EtcdException( - 'Unable to decode server response: %r' % e) + raise etcd.EtcdException("Unable to decode server response: %r" % e) def _next_server(self, cause=None): - """ Selects the next server in the list, refreshes the server list. """ - _log.debug("Selecting next machine in cache. Available machines: %s", - self._machines_cache) + """Selects the next server in the list, refreshes the server list.""" + _log.debug( + "Selecting next machine in cache. Available machines: %s", + self._machines_cache, + ) try: mach = self._machines_cache.pop() except IndexError: _log.error("Machines cache is empty, no machines to try.") - raise etcd.EtcdConnectionFailed('No more machines in the cluster', - cause=cause) + raise etcd.EtcdConnectionFailed("No more machines in the cluster", cause=cause) else: _log.info("Selected new etcd server %s", mach) return mach @@ -866,14 +870,13 @@ def wrapper(self, path, method, params=None, timeout=None): if timeout == 0: timeout = None - if not path.startswith('/'): - raise ValueError('Path does not start with /') + if not path.startswith("/"): + raise ValueError("Path does not start with /") while not response: some_request_failed = False try: - response = payload(self, path, method, - params=params, timeout=timeout) + response = payload(self, path, method, params=params, timeout=timeout) # Check the cluster ID hasn't changed under us. We use # preload_content=False above so we can read the headers # before we wait for the content of a watch. @@ -885,19 +888,16 @@ def wrapper(self, path, method, params=None, timeout=None): # urllib3 doesn't wrap all httplib exceptions and earlier versions # don't wrap socket errors either. except (HTTPError, HTTPException, socket.error) as e: - if (isinstance(params, dict) and - params.get("wait") == "true" and - isinstance(e, ReadTimeoutError)): + if ( + isinstance(params, dict) + and params.get("wait") == "true" + and isinstance(e, ReadTimeoutError) + ): _log.debug("Watch timed out.") - raise etcd.EtcdWatchTimedOut( - "Watch timed out: %r" % e, - cause=e - ) - _log.error("Request to server %s failed: %r", - self._base_uri, e) + raise etcd.EtcdWatchTimedOut("Watch timed out: %r" % e, cause=e) + _log.error("Request to server %s failed: %r", self._base_uri, e) if self._allow_reconnect: - _log.info("Reconnection allowed, looking for another " - "server.") + _log.info("Reconnection allowed, looking for another " "server.") # _next_server() raises EtcdException if there are no # machines left to try, breaking out of the loop. self._base_uri = self._next_server(cause=e) @@ -910,8 +910,7 @@ def wrapper(self, path, method, params=None, timeout=None): else: _log.debug("Reconnection disabled, giving up.") raise etcd.EtcdConnectionFailed( - "Connection to etcd failed due to %r" % e, - cause=e + "Connection to etcd failed due to %r" % e, cause=e ) except etcd.EtcdClusterIdChanged as e: _log.warning(e) @@ -926,11 +925,12 @@ def wrapper(self, path, method, params=None, timeout=None): self._machines_cache = self.machines self._machines_cache.remove(self._base_uri) return self._handle_server_response(response) + return wrapper @_wrap_request def api_execute(self, path, method, params=None, timeout=None): - """ Executes the query. """ + """Executes the query.""" url = self._base_uri + path if (method == self._MGET) or (method == self._MDELETE): @@ -941,7 +941,8 @@ def api_execute(self, path, method, params=None, timeout=None): fields=params, redirect=self.allow_redirect, headers=self._get_headers(), - preload_content=False) + preload_content=False, + ) elif (method == self._MPUT) or (method == self._MPOST): return self.http.request_encode_body( @@ -952,24 +953,26 @@ def api_execute(self, path, method, params=None, timeout=None): encode_multipart=False, redirect=self.allow_redirect, headers=self._get_headers(), - preload_content=False) + preload_content=False, + ) else: - raise etcd.EtcdException( - 'HTTP method {} not supported'.format(method)) + raise etcd.EtcdException("HTTP method {} not supported".format(method)) @_wrap_request def api_execute_json(self, path, method, params=None, timeout=None): url = self._base_uri + path json_payload = json.dumps(params) headers = self._get_headers() - headers['Content-Type'] = 'application/json' - return self.http.urlopen(method, - url, - body=json_payload, - timeout=timeout, - redirect=self.allow_redirect, - headers=headers, - preload_content=False) + headers["Content-Type"] = "application/json" + return self.http.urlopen( + method, + url, + body=json_payload, + timeout=timeout, + redirect=self.allow_redirect, + headers=headers, + preload_content=False, + ) def _check_cluster_id(self, response, path): cluster_id = response.getheader("x-etcd-cluster-id") @@ -977,8 +980,7 @@ def _check_cluster_id(self, response, path): if self.version_prefix in path: _log.warning("etcd response did not contain a cluster ID") return - id_changed = (self.expected_cluster_id and - cluster_id != self.expected_cluster_id) + id_changed = self.expected_cluster_id and cluster_id != self.expected_cluster_id # Update the ID so we only raise the exception once. old_expected_cluster_id = self.expected_cluster_id self.expected_cluster_id = cluster_id @@ -987,29 +989,29 @@ def _check_cluster_id(self, response, path): # time. self.http.clear() raise etcd.EtcdClusterIdChanged( - 'The UUID of the cluster changed from {} to ' - '{}.'.format(old_expected_cluster_id, cluster_id)) + "The UUID of the cluster changed from {} to " + "{}.".format(old_expected_cluster_id, cluster_id) + ) def _handle_server_response(self, response): - """ Handles the server response """ + """Handles the server response""" if response.status in [200, 201]: return response else: - resp = response.data.decode('utf-8') + resp = response.data.decode("utf-8") # throw the appropriate exception try: r = json.loads(resp) - r['status'] = response.status + r["status"] = response.status except (TypeError, ValueError): # Bad JSON, make a response locally. - r = {"message": "Bad response", - "cause": str(resp)} + r = {"message": "Bad response", "cause": str(resp)} etcd.EtcdError.handle(r) def _get_headers(self): if self.username and self.password: - credentials = ':'.join((self.username, self.password)) + credentials = ":".join((self.username, self.password)) return urllib3.make_headers(basic_auth=credentials) return {} diff --git a/src/etcd/lock.py b/src/etcd/lock.py index 4a5bc007..013bc234 100644 --- a/src/etcd/lock.py +++ b/src/etcd/lock.py @@ -4,6 +4,7 @@ _log = logging.getLogger(__name__) + class Lock(object): """ Locking recipe for etcd, inspired by the kazoo recipe for zookeeper @@ -17,7 +18,7 @@ def __init__(self, client, lock_name): # prevent us from getting back the full path name. We prefix our # lock name with a uuid and can check for its presence on retry. self._uuid = uuid.uuid4().hex - self.path = "{}/{}".format(client.lock_prefix, lock_name) + self.path = "{}/{}".format(client.lock_prefix, lock_name) self.is_taken = False self._sequence = None _log.debug("Initiating lock for %s with uuid %s", self.path, self._uuid) @@ -139,10 +140,10 @@ def _acquired(self, blocking=True, timeout=0): def lock_key(self): if not self._sequence: raise ValueError("No sequence present.") - return self.path + '/' + str(self._sequence) + return self.path + "/" + str(self._sequence) def _set_sequence(self, key): - self._sequence = key.replace(self.path, '').lstrip('/') + self._sequence = key.replace(self.path, "").lstrip("/") def _find_lock(self): if self._sequence: @@ -163,8 +164,7 @@ def _find_lock(self): return False def _get_locker(self): - results = [res for res in - self.client.read(self.path, recursive=True).leaves] + results = [res for res in self.client.read(self.path, recursive=True).leaves] if not self._sequence: self._find_lock() l = sorted([r.key for r in results]) @@ -175,9 +175,9 @@ def _get_locker(self): _log.debug("No key before our one, we are the locker") return (l[0], None) else: - _log.debug("Locker: %s, key to watch: %s", l[0], l[i-1]) - return (l[0], next(x for x in results if x.key == l[i-1])) + _log.debug("Locker: %s, key to watch: %s", l[0], l[i - 1]) + return (l[0], next(x for x in results if x.key == l[i - 1])) except ValueError: # Something very wrong is going on, most probably # our lock has expired - raise etcd.EtcdLockExpired(u"Lock not found") + raise etcd.EtcdLockExpired("Lock not found") diff --git a/src/etcd/tests/integration/helpers.py b/src/etcd/tests/integration/helpers.py index 1f1d22bf..1a0933df 100644 --- a/src/etcd/tests/integration/helpers.py +++ b/src/etcd/tests/integration/helpers.py @@ -10,39 +10,49 @@ class EtcdProcessHelper(object): - def __init__( - self, - base_directory, - proc_name='etcd', - port_range_start=4001, - internal_port_range_start=7001, - cluster=False, - tls=False + self, + base_directory, + proc_name="etcd", + port_range_start=4001, + internal_port_range_start=7001, + cluster=False, + tls=False, ): - self.base_directory = base_directory self.proc_name = proc_name self.port_range_start = port_range_start self.internal_port_range_start = internal_port_range_start self.processes = {} self.cluster = cluster - self.schema = 'http://' + self.schema = "http://" if tls: - self.schema = 'https://' + self.schema = "https://" def run(self, number=1, proc_args=[]): if number > 1: - initial_cluster = ",".join([ "test-node-{}={}127.0.0.1:{}".format(slot, 'http://', self.internal_port_range_start + slot) for slot in range(0, number)]) - proc_args.extend([ - '-initial-cluster', initial_cluster, - '-initial-cluster-state', 'new' - ]) + initial_cluster = ",".join( + [ + "test-node-{}={}127.0.0.1:{}".format( + slot, "http://", self.internal_port_range_start + slot + ) + for slot in range(0, number) + ] + ) + proc_args.extend( + ["-initial-cluster", initial_cluster, "-initial-cluster-state", "new"] + ) else: - proc_args.extend([ - '-initial-cluster', 'test-node-0=http://127.0.0.1:{}'.format(self.internal_port_range_start), - '-initial-cluster-state', 'new' - ]) + proc_args.extend( + [ + "-initial-cluster", + "test-node-0=http://127.0.0.1:{}".format( + self.internal_port_range_start + ), + "-initial-cluster-state", + "new", + ] + ) for i in range(0, number): self.add_one(i, proc_args) @@ -55,29 +65,34 @@ def stop(self): def add_one(self, slot, proc_args=None): log = logging.getLogger() directory = tempfile.mkdtemp( - dir=self.base_directory, - prefix='python-etcd.%d-' % slot) + dir=self.base_directory, prefix="python-etcd.%d-" % slot + ) - log.debug('Created directory %s' % directory) - client = '%s127.0.0.1:%d' % (self.schema, self.port_range_start + slot) - peer = '%s127.0.0.1:%d' % ('http://', self.internal_port_range_start - + slot) + log.debug("Created directory %s" % directory) + client = "%s127.0.0.1:%d" % (self.schema, self.port_range_start + slot) + peer = "%s127.0.0.1:%d" % ("http://", self.internal_port_range_start + slot) daemon_args = [ self.proc_name, - '-data-dir', directory, - '-name', 'test-node-%d' % slot, - '-initial-advertise-peer-urls', peer, - '-listen-peer-urls', peer, - '-advertise-client-urls', client, - '-listen-client-urls', client + "-data-dir", + directory, + "-name", + "test-node-%d" % slot, + "-initial-advertise-peer-urls", + peer, + "-listen-peer-urls", + peer, + "-advertise-client-urls", + client, + "-listen-client-urls", + client, ] if proc_args: daemon_args.extend(proc_args) daemon = subprocess.Popen(daemon_args) - log.debug('Started %d' % daemon.pid) - log.debug('Params: %s' % daemon_args) + log.debug("Started %d" % daemon.pid) + log.debug("Params: %s" % daemon_args) time.sleep(2) self.processes[slot] = (directory, daemon) @@ -86,13 +101,12 @@ def kill_one(self, slot): data_dir, process = self.processes.pop(slot) process.kill() time.sleep(2) - log.debug('Killed etcd pid:%d', process.pid) + log.debug("Killed etcd pid:%d", process.pid) shutil.rmtree(data_dir) - log.debug('Removed directory %s' % data_dir) + log.debug("Removed directory %s" % data_dir) class TestingCA(object): - @classmethod def create_test_ca_certificate(cls, cert_path, key_path, cn=None): k = crypto.PKey() @@ -103,7 +117,7 @@ def create_test_ca_certificate(cls, cert_path, key_path, cn=None): serial = uuid.uuid4().int else: md5_hash = hashlib.md5() - md5_hash.update(cn.encode('utf-8')) + md5_hash.update(cn.encode("utf-8")) serial = int(md5_hash.hexdigest(), 36) cert.get_subject().CN = cn @@ -117,31 +131,43 @@ def create_test_ca_certificate(cls, cert_path, key_path, cn=None): cert.gmtime_adj_notAfter(315360000) cert.set_issuer(cert.get_subject()) cert.set_pubkey(k) - cert.add_extensions([ - crypto.X509Extension("basicConstraints".encode('ascii'), False, - "CA:TRUE".encode('ascii')), - crypto.X509Extension("keyUsage".encode('ascii'), False, - "keyCertSign, cRLSign".encode('ascii')), - crypto.X509Extension("subjectKeyIdentifier".encode('ascii'), False, - "hash".encode('ascii'), - subject=cert), - ]) - - cert.add_extensions([ - crypto.X509Extension( - "authorityKeyIdentifier".encode('ascii'), False, - "keyid:always".encode('ascii'), issuer=cert) - ]) - - cert.sign(k, 'sha1') - - with open(cert_path, 'w') as f: - f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert) - .decode('utf-8')) - - with open(key_path, 'w') as f: - f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, k) - .decode('utf-8')) + cert.add_extensions( + [ + crypto.X509Extension( + "basicConstraints".encode("ascii"), False, "CA:TRUE".encode("ascii") + ), + crypto.X509Extension( + "keyUsage".encode("ascii"), + False, + "keyCertSign, cRLSign".encode("ascii"), + ), + crypto.X509Extension( + "subjectKeyIdentifier".encode("ascii"), + False, + "hash".encode("ascii"), + subject=cert, + ), + ] + ) + + cert.add_extensions( + [ + crypto.X509Extension( + "authorityKeyIdentifier".encode("ascii"), + False, + "keyid:always".encode("ascii"), + issuer=cert, + ) + ] + ) + + cert.sign(k, "sha1") + + with open(cert_path, "w") as f: + f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode("utf-8")) + + with open(key_path, "w") as f: + f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, k).decode("utf-8")) return cert, k @@ -155,7 +181,7 @@ def create_test_certificate(cls, ca, ca_key, cert_path, key_path, cn=None): serial = uuid.uuid4().int else: md5_hash = hashlib.md5() - md5_hash.update(cn.encode('utf-8')) + md5_hash.update(cn.encode("utf-8")) serial = int(md5_hash.hexdigest(), 36) cert.get_subject().CN = cn @@ -165,20 +191,25 @@ def create_test_certificate(cls, ca, ca_key, cert_path, key_path, cn=None): cert.get_subject().O = "Organization" cert.get_subject().OU = "Organizational Unit" - cert.add_extensions([ - crypto.X509Extension( - "keyUsage".encode('ascii'), - False, - "nonRepudiation,digitalSignature,keyEncipherment".encode('ascii')), - crypto.X509Extension( - "extendedKeyUsage".encode('ascii'), - False, - "clientAuth,serverAuth".encode('ascii')), - crypto.X509Extension( - "subjectAltName".encode('ascii'), - False, - "IP: 127.0.0.1".encode('ascii')), - ]) + cert.add_extensions( + [ + crypto.X509Extension( + "keyUsage".encode("ascii"), + False, + "nonRepudiation,digitalSignature,keyEncipherment".encode("ascii"), + ), + crypto.X509Extension( + "extendedKeyUsage".encode("ascii"), + False, + "clientAuth,serverAuth".encode("ascii"), + ), + crypto.X509Extension( + "subjectAltName".encode("ascii"), + False, + "IP: 127.0.0.1".encode("ascii"), + ), + ] + ) cert.gmtime_adj_notBefore(0) cert.gmtime_adj_notAfter(315360000) @@ -186,12 +217,10 @@ def create_test_certificate(cls, ca, ca_key, cert_path, key_path, cn=None): cert.set_pubkey(k) cert.set_serial_number(serial) - cert.sign(ca_key, 'sha1') + cert.sign(ca_key, "sha1") - with open(cert_path, 'w') as f: - f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert) - .decode('utf-8')) + with open(cert_path, "w") as f: + f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode("utf-8")) - with open(key_path, 'w') as f: - f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, k) - .decode('utf-8')) + with open(key_path, "w") as f: + f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, k).decode("utf-8")) diff --git a/src/etcd/tests/integration/test_simple.py b/src/etcd/tests/integration/test_simple.py index 4baeadeb..3faf80dd 100644 --- a/src/etcd/tests/integration/test_simple.py +++ b/src/etcd/tests/integration/test_simple.py @@ -23,12 +23,13 @@ class EtcdIntegrationTest(unittest.TestCase): @classmethod def setUpClass(cls): program = cls._get_exe() - cls.directory = tempfile.mkdtemp(prefix='python-etcd') + cls.directory = tempfile.mkdtemp(prefix="python-etcd") cls.processHelper = helpers.EtcdProcessHelper( cls.directory, proc_name=program, port_range_start=6001, - internal_port_range_start=8001) + internal_port_range_start=8001, + ) cls.processHelper.run(number=cls.cl_size) cls.client = etcd.Client(port=6001) @@ -43,7 +44,7 @@ def _is_exe(cls, fpath): @classmethod def _get_exe(cls): - PROGRAM = 'etcd' + PROGRAM = "etcd" program_path = None @@ -55,196 +56,203 @@ def _get_exe(cls): break if not program_path: - raise Exception('etcd not in path!!') + raise Exception("etcd not in path!!") return program_path class TestSimple(EtcdIntegrationTest): - def test_machines(self): - """ INTEGRATION: retrieve machines """ - self.assertEquals(self.client.machines[0], 'http://127.0.0.1:6001') + """INTEGRATION: retrieve machines""" + self.assertEquals(self.client.machines[0], "http://127.0.0.1:6001") def test_leader(self): - """ INTEGRATION: retrieve leader """ - self.assertIn(self.client.leader['clientURLs'][0], - ['http://127.0.0.1:6001','http://127.0.0.1:6002','http://127.0.0.1:6003']) + """INTEGRATION: retrieve leader""" + self.assertIn( + self.client.leader["clientURLs"][0], + ["http://127.0.0.1:6001", "http://127.0.0.1:6002", "http://127.0.0.1:6003"], + ) def test_get_set_delete(self): - """ INTEGRATION: set a new value """ + """INTEGRATION: set a new value""" try: - get_result = self.client.get('/test_set') + get_result = self.client.get("/test_set") assert False except etcd.EtcdKeyNotFound as e: pass - self.assertFalse('/test_set' in self.client) + self.assertFalse("/test_set" in self.client) - set_result = self.client.set('/test_set', 'test-key') - self.assertEquals('set', set_result.action.lower()) - self.assertEquals('/test_set', set_result.key) - self.assertEquals('test-key', set_result.value) + set_result = self.client.set("/test_set", "test-key") + self.assertEquals("set", set_result.action.lower()) + self.assertEquals("/test_set", set_result.key) + self.assertEquals("test-key", set_result.value) - self.assertTrue('/test_set' in self.client) + self.assertTrue("/test_set" in self.client) - get_result = self.client.get('/test_set') - self.assertEquals('get', get_result.action.lower()) - self.assertEquals('/test_set', get_result.key) - self.assertEquals('test-key', get_result.value) + get_result = self.client.get("/test_set") + self.assertEquals("get", get_result.action.lower()) + self.assertEquals("/test_set", get_result.key) + self.assertEquals("test-key", get_result.value) - delete_result = self.client.delete('/test_set') - self.assertEquals('delete', delete_result.action.lower()) - self.assertEquals('/test_set', delete_result.key) + delete_result = self.client.delete("/test_set") + self.assertEquals("delete", delete_result.action.lower()) + self.assertEquals("/test_set", delete_result.key) - self.assertFalse('/test_set' in self.client) + self.assertFalse("/test_set" in self.client) try: - get_result = self.client.get('/test_set') + get_result = self.client.get("/test_set") assert False except etcd.EtcdKeyNotFound as e: pass def test_update(self): """INTEGRATION: update a value""" - self.client.set('/foo', 3) - c = self.client.get('/foo') + self.client.set("/foo", 3) + c = self.client.get("/foo") c.value = int(c.value) + 3 self.client.update(c) - newres = self.client.get('/foo') - self.assertEquals(newres.value, u'6') + newres = self.client.get("/foo") + self.assertEquals(newres.value, "6") self.assertRaises(ValueError, self.client.update, c) def test_retrieve_subkeys(self): - """ INTEGRATION: retrieve multiple subkeys """ - set_result = self.client.write('/subtree/test_set', 'test-key1') - set_result = self.client.write('/subtree/test_set1', 'test-key2') - set_result = self.client.write('/subtree/test_set2', 'test-key3') - get_result = self.client.read('/subtree', recursive=True) + """INTEGRATION: retrieve multiple subkeys""" + set_result = self.client.write("/subtree/test_set", "test-key1") + set_result = self.client.write("/subtree/test_set1", "test-key2") + set_result = self.client.write("/subtree/test_set2", "test-key3") + get_result = self.client.read("/subtree", recursive=True) result = [subkey.value for subkey in get_result.leaves] - self.assertEquals(['test-key1', 'test-key2', 'test-key3'].sort(), result.sort()) + self.assertEquals(["test-key1", "test-key2", "test-key3"].sort(), result.sort()) def test_directory_ttl_update(self): - """ INTEGRATION: should be able to update a dir TTL """ - self.client.write('/dir', None, dir=True, ttl=30) - res = self.client.write('/dir', None, dir=True, ttl=31, prevExist=True) + """INTEGRATION: should be able to update a dir TTL""" + self.client.write("/dir", None, dir=True, ttl=30) + res = self.client.write("/dir", None, dir=True, ttl=31, prevExist=True) self.assertEquals(res.ttl, 31) - res = self.client.get('/dir') + res = self.client.get("/dir") res.ttl = 120 new_res = self.client.update(res) self.assertEquals(new_res.ttl, 120) - class TestErrors(EtcdIntegrationTest): - def test_is_not_a_file(self): - """ INTEGRATION: try to write value to an existing directory """ + """INTEGRATION: try to write value to an existing directory""" - self.client.set('/directory/test-key', 'test-value') - self.assertRaises(etcd.EtcdNotFile, self.client.set, '/directory', 'test-value') + self.client.set("/directory/test-key", "test-value") + self.assertRaises(etcd.EtcdNotFile, self.client.set, "/directory", "test-value") def test_test_and_set(self): - """ INTEGRATION: try test_and_set operation """ + """INTEGRATION: try test_and_set operation""" - set_result = self.client.set('/test-key', 'old-test-value') + set_result = self.client.set("/test-key", "old-test-value") set_result = self.client.test_and_set( - '/test-key', - 'test-value', - 'old-test-value') + "/test-key", "test-value", "old-test-value" + ) - self.assertRaises(ValueError, self.client.test_and_set, '/test-key', 'new-value', 'old-test-value') + self.assertRaises( + ValueError, + self.client.test_and_set, + "/test-key", + "new-value", + "old-test-value", + ) def test_creating_already_existing_directory(self): - """ INTEGRATION: creating an already existing directory without - `prevExist=True` should fail """ - self.client.write('/mydir', None, dir=True) + """INTEGRATION: creating an already existing directory without + `prevExist=True` should fail""" + self.client.write("/mydir", None, dir=True) - self.assertRaises(etcd.EtcdNotFile, self.client.write, '/mydir', None, dir=True) - self.assertRaises(etcd.EtcdAlreadyExist, self.client.write, '/mydir', None, dir=True, prevExist=False) + self.assertRaises(etcd.EtcdNotFile, self.client.write, "/mydir", None, dir=True) + self.assertRaises( + etcd.EtcdAlreadyExist, + self.client.write, + "/mydir", + None, + dir=True, + prevExist=False, + ) class TestClusterFunctions(EtcdIntegrationTest): - @classmethod def setUpClass(cls): program = cls._get_exe() - cls.directory = tempfile.mkdtemp(prefix='python-etcd') + cls.directory = tempfile.mkdtemp(prefix="python-etcd") cls.processHelper = helpers.EtcdProcessHelper( cls.directory, proc_name=program, port_range_start=6001, internal_port_range_start=8001, - cluster=True) + cluster=True, + ) def test_reconnect(self): - """ INTEGRATION: get key after the server we're connected fails. """ + """INTEGRATION: get key after the server we're connected fails.""" self.processHelper.stop() self.processHelper.run(number=3) self.client = etcd.Client(port=6001, allow_reconnect=True) - set_result = self.client.set('/test_set', 'test-key1') - get_result = self.client.get('/test_set') + set_result = self.client.set("/test_set", "test-key1") + get_result = self.client.get("/test_set") - self.assertEquals('test-key1', get_result.value) + self.assertEquals("test-key1", get_result.value) self.processHelper.kill_one(0) - get_result = self.client.get('/test_set') - self.assertEquals('test-key1', get_result.value) + get_result = self.client.get("/test_set") + self.assertEquals("test-key1", get_result.value) def test_reconnect_with_several_hosts_passed(self): - """ INTEGRATION: receive several hosts at connection setup. """ + """INTEGRATION: receive several hosts at connection setup.""" self.processHelper.stop() self.processHelper.run(number=3) self.client = etcd.Client( - host=( - ('127.0.0.1', 6004), - ('127.0.0.1', 6001)), - allow_reconnect=True) - set_result = self.client.set('/test_set', 'test-key1') - get_result = self.client.get('/test_set') + host=(("127.0.0.1", 6004), ("127.0.0.1", 6001)), allow_reconnect=True + ) + set_result = self.client.set("/test_set", "test-key1") + get_result = self.client.get("/test_set") - self.assertEquals('test-key1', get_result.value) + self.assertEquals("test-key1", get_result.value) self.processHelper.kill_one(0) - get_result = self.client.get('/test_set') - self.assertEquals('test-key1', get_result.value) + get_result = self.client.get("/test_set") + self.assertEquals("test-key1", get_result.value) def test_reconnect_not_allowed(self): - """ INTEGRATION: fail on server kill if not allow_reconnect """ + """INTEGRATION: fail on server kill if not allow_reconnect""" self.processHelper.stop() self.processHelper.run(number=3) self.client = etcd.Client(port=6001, allow_reconnect=False) self.processHelper.kill_one(0) - self.assertRaises(etcd.EtcdConnectionFailed, self.client.get, - '/test_set') + self.assertRaises(etcd.EtcdConnectionFailed, self.client.get, "/test_set") def test_reconnet_fails(self): - """ INTEGRATION: fails to reconnect if no available machines """ + """INTEGRATION: fails to reconnect if no available machines""" self.processHelper.stop() # Start with three instances (0, 1, 2) self.processHelper.run(number=3) # Connect to instance 0 self.client = etcd.Client(port=6001, allow_reconnect=True) - set_result = self.client.set('/test_set', 'test-key1') + set_result = self.client.set("/test_set", "test-key1") - get_result = self.client.get('/test_set') - self.assertEquals('test-key1', get_result.value) + get_result = self.client.get("/test_set") + self.assertEquals("test-key1", get_result.value) self.processHelper.kill_one(2) self.processHelper.kill_one(1) self.processHelper.kill_one(0) - self.assertRaises(etcd.EtcdException, self.client.get, '/test_set') + self.assertRaises(etcd.EtcdException, self.client.get, "/test_set") class TestWatch(EtcdIntegrationTest): - def test_watch(self): - """ INTEGRATION: Receive a watch event from other process """ + """INTEGRATION: Receive a watch event from other process""" - set_result = self.client.set('/test-key', 'test-value') + set_result = self.client.set("/test-key", "test-value") queue = multiprocessing.Queue() @@ -257,10 +265,14 @@ def watch_value(key, queue): queue.put(c.watch(key).value) changer = multiprocessing.Process( - target=change_value, args=('/test-key', 'new-test-value',)) + target=change_value, + args=( + "/test-key", + "new-test-value", + ), + ) - watcher = multiprocessing.Process( - target=watch_value, args=('/test-key', queue)) + watcher = multiprocessing.Process(target=watch_value, args=("/test-key", queue)) watcher.start() time.sleep(1) @@ -271,16 +283,16 @@ def watch_value(key, queue): watcher.join(timeout=5) changer.join(timeout=5) - assert value == 'new-test-value' + assert value == "new-test-value" def test_watch_indexed(self): - """ INTEGRATION: Receive a watch event from other process, indexed """ + """INTEGRATION: Receive a watch event from other process, indexed""" - set_result = self.client.set('/test-key', 'test-value') - set_result = self.client.set('/test-key', 'test-value0') + set_result = self.client.set("/test-key", "test-value") + set_result = self.client.set("/test-key", "test-value0") original_index = int(set_result.modifiedIndex) - set_result = self.client.set('/test-key', 'test-value1') - set_result = self.client.set('/test-key', 'test-value2') + set_result = self.client.set("/test-key", "test-value1") + set_result = self.client.set("/test-key", "test-value2") queue = multiprocessing.Queue() @@ -295,10 +307,16 @@ def watch_value(key, index, queue): queue.put(c.watch(key, index=index + i).value) proc = multiprocessing.Process( - target=change_value, args=('/test-key', 'test-value3',)) + target=change_value, + args=( + "/test-key", + "test-value3", + ), + ) watcher = multiprocessing.Process( - target=watch_value, args=('/test-key', original_index, queue)) + target=watch_value, args=("/test-key", original_index, queue) + ) watcher.start() time.sleep(0.5) @@ -308,15 +326,15 @@ def watch_value(key, index, queue): for i in range(0, 3): value = queue.get() log.debug("index: %d: %s" % (i, value)) - self.assertEquals('test-value%d' % i, value) + self.assertEquals("test-value%d" % i, value) watcher.join(timeout=5) proc.join(timeout=5) def test_watch_generator(self): - """ INTEGRATION: Receive a watch event from other process (gen) """ + """INTEGRATION: Receive a watch event from other process (gen)""" - set_result = self.client.set('/test-key', 'test-value') + set_result = self.client.set("/test-key", "test-value") queue = multiprocessing.Queue() @@ -324,7 +342,7 @@ def change_value(key): time.sleep(0.5) c = etcd.Client(port=6001) for i in range(0, 3): - c.set(key, 'test-value%d' % i) + c.set(key, "test-value%d" % i) c.get(key) def watch_value(key, queue): @@ -333,16 +351,14 @@ def watch_value(key, queue): event = next(c.eternal_watch(key)).value queue.put(event) - changer = multiprocessing.Process( - target=change_value, args=('/test-key',)) + changer = multiprocessing.Process(target=change_value, args=("/test-key",)) - watcher = multiprocessing.Process( - target=watch_value, args=('/test-key', queue)) + watcher = multiprocessing.Process(target=watch_value, args=("/test-key", queue)) watcher.start() changer.start() - values = ['test-value0', 'test-value1', 'test-value2'] + values = ["test-value0", "test-value1", "test-value2"] for i in range(0, 1): value = queue.get() log.debug("index: %d: %s" % (i, value)) @@ -352,13 +368,13 @@ def watch_value(key, queue): changer.join(timeout=5) def test_watch_indexed_generator(self): - """ INTEGRATION: Receive a watch event from other process, ixd, (2) """ + """INTEGRATION: Receive a watch event from other process, ixd, (2)""" - set_result = self.client.set('/test-key', 'test-value') - set_result = self.client.set('/test-key', 'test-value0') + set_result = self.client.set("/test-key", "test-value") + set_result = self.client.set("/test-key", "test-value0") original_index = int(set_result.modifiedIndex) - set_result = self.client.set('/test-key', 'test-value1') - set_result = self.client.set('/test-key', 'test-value2') + set_result = self.client.set("/test-key", "test-value1") + set_result = self.client.set("/test-key", "test-value2") queue = multiprocessing.Queue() @@ -373,10 +389,16 @@ def watch_value(key, index, queue): queue.put(next(iterevents).value) proc = multiprocessing.Process( - target=change_value, args=('/test-key', 'test-value3',)) + target=change_value, + args=( + "/test-key", + "test-value3", + ), + ) watcher = multiprocessing.Process( - target=watch_value, args=('/test-key', original_index, queue)) + target=watch_value, args=("/test-key", original_index, queue) + ) watcher.start() time.sleep(0.5) @@ -385,7 +407,7 @@ def watch_value(key, index, queue): for i in range(0, 3): value = queue.get() log.debug("index: %d: %s" % (i, value)) - self.assertEquals('test-value%d' % i, value) + self.assertEquals("test-value%d" % i, value) watcher.join(timeout=5) proc.join(timeout=5) diff --git a/src/etcd/tests/integration/test_ssl.py b/src/etcd/tests/integration/test_ssl.py index 6ba6a3ad..2819fc97 100644 --- a/src/etcd/tests/integration/test_ssl.py +++ b/src/etcd/tests/integration/test_ssl.py @@ -15,166 +15,168 @@ log = logging.getLogger() -class TestEncryptedAccess(test_simple.EtcdIntegrationTest): +class TestEncryptedAccess(test_simple.EtcdIntegrationTest): @classmethod def setUpClass(cls): program = cls._get_exe() - cls.directory = tempfile.mkdtemp(prefix='python-etcd') + cls.directory = tempfile.mkdtemp(prefix="python-etcd") - cls.ca_cert_path = os.path.join(cls.directory, 'ca.crt') - ca_key_path = os.path.join(cls.directory, 'ca.key') + cls.ca_cert_path = os.path.join(cls.directory, "ca.crt") + ca_key_path = os.path.join(cls.directory, "ca.key") - cls.ca2_cert_path = os.path.join(cls.directory, 'ca2.crt') - ca2_key_path = os.path.join(cls.directory, 'ca2.key') + cls.ca2_cert_path = os.path.join(cls.directory, "ca2.crt") + ca2_key_path = os.path.join(cls.directory, "ca2.key") - server_cert_path = os.path.join(cls.directory, 'server.crt') - server_key_path = os.path.join(cls.directory, 'server.key') + server_cert_path = os.path.join(cls.directory, "server.crt") + server_key_path = os.path.join(cls.directory, "server.key") ca, ca_key = helpers.TestingCA.create_test_ca_certificate( - cls.ca_cert_path, ca_key_path, 'TESTCA') + cls.ca_cert_path, ca_key_path, "TESTCA" + ) ca2, ca2_key = helpers.TestingCA.create_test_ca_certificate( - cls.ca2_cert_path, ca2_key_path, 'TESTCA2') + cls.ca2_cert_path, ca2_key_path, "TESTCA2" + ) helpers.TestingCA.create_test_certificate( - ca, ca_key, server_cert_path, server_key_path, '127.0.0.1') + ca, ca_key, server_cert_path, server_key_path, "127.0.0.1" + ) cls.processHelper = helpers.EtcdProcessHelper( cls.directory, proc_name=program, port_range_start=6001, internal_port_range_start=8001, - tls=True + tls=True, ) - cls.processHelper.run(number=3, - proc_args=[ - '-cert-file=%s' % server_cert_path, - '-key-file=%s' % server_key_path - ]) + cls.processHelper.run( + number=3, + proc_args=[ + "-cert-file=%s" % server_cert_path, + "-key-file=%s" % server_key_path, + ], + ) def test_get_set_unauthenticated(self): - """ INTEGRATION: set/get a new value unauthenticated (http->https) """ + """INTEGRATION: set/get a new value unauthenticated (http->https)""" client = etcd.Client(port=6001) # Since python 3 raises a MaxRetryError here, this gets caught in # different code blocks in python 2 and python 3, thus messages are # different. Python 3 does the right thing(TM), for the record - self.assertRaises( - etcd.EtcdException, client.set, '/test_set', 'test-key') + self.assertRaises(etcd.EtcdException, client.set, "/test_set", "test-key") - self.assertRaises(etcd.EtcdException, client.get, '/test_set') + self.assertRaises(etcd.EtcdException, client.get, "/test_set") @nottest def test_get_set_unauthenticated_missing_ca(self): - """ INTEGRATION: try unauthenticated w/out validation (https->https)""" + """INTEGRATION: try unauthenticated w/out validation (https->https)""" # This doesn't work for now and will need further inspection - client = etcd.Client(protocol='https', port=6001) - set_result = client.set('/test_set', 'test-key') - get_result = client.get('/test_set') - + client = etcd.Client(protocol="https", port=6001) + set_result = client.set("/test_set", "test-key") + get_result = client.get("/test_set") def test_get_set_unauthenticated_with_ca(self): - """ INTEGRATION: try unauthenticated with validation (https->https)""" - client = etcd.Client( - protocol='https', port=6001, ca_cert=self.ca2_cert_path) + """INTEGRATION: try unauthenticated with validation (https->https)""" + client = etcd.Client(protocol="https", port=6001, ca_cert=self.ca2_cert_path) - self.assertRaises(etcd.EtcdConnectionFailed, client.set, '/test-set', 'test-key') - self.assertRaises(etcd.EtcdConnectionFailed, client.get, '/test-set') + self.assertRaises( + etcd.EtcdConnectionFailed, client.set, "/test-set", "test-key" + ) + self.assertRaises(etcd.EtcdConnectionFailed, client.get, "/test-set") def test_get_set_authenticated(self): - """ INTEGRATION: set/get a new value authenticated """ + """INTEGRATION: set/get a new value authenticated""" - client = etcd.Client( - port=6001, protocol='https', ca_cert=self.ca_cert_path) + client = etcd.Client(port=6001, protocol="https", ca_cert=self.ca_cert_path) - set_result = client.set('/test_set', 'test-key') - get_result = client.get('/test_set') + set_result = client.set("/test_set", "test-key") + get_result = client.get("/test_set") class TestClientAuthenticatedAccess(test_simple.EtcdIntegrationTest): - @classmethod def setUpClass(cls): program = cls._get_exe() - cls.directory = tempfile.mkdtemp(prefix='python-etcd') + cls.directory = tempfile.mkdtemp(prefix="python-etcd") - cls.ca_cert_path = os.path.join(cls.directory, 'ca.crt') - ca_key_path = os.path.join(cls.directory, 'ca.key') + cls.ca_cert_path = os.path.join(cls.directory, "ca.crt") + ca_key_path = os.path.join(cls.directory, "ca.key") - server_cert_path = os.path.join(cls.directory, 'server.crt') - server_key_path = os.path.join(cls.directory, 'server.key') + server_cert_path = os.path.join(cls.directory, "server.crt") + server_key_path = os.path.join(cls.directory, "server.key") - cls.client_cert_path = os.path.join(cls.directory, 'client.crt') - cls.client_key_path = os.path.join(cls.directory, 'client.key') + cls.client_cert_path = os.path.join(cls.directory, "client.crt") + cls.client_key_path = os.path.join(cls.directory, "client.key") - cls.client_all_cert = os.path.join(cls.directory, 'client-all.crt') + cls.client_all_cert = os.path.join(cls.directory, "client-all.crt") ca, ca_key = helpers.TestingCA.create_test_ca_certificate( - cls.ca_cert_path, ca_key_path) + cls.ca_cert_path, ca_key_path + ) helpers.TestingCA.create_test_certificate( - ca, ca_key, server_cert_path, server_key_path, '127.0.0.1') + ca, ca_key, server_cert_path, server_key_path, "127.0.0.1" + ) helpers.TestingCA.create_test_certificate( - ca, - ca_key, - cls.client_cert_path, - cls.client_key_path) + ca, ca_key, cls.client_cert_path, cls.client_key_path + ) cls.processHelper = helpers.EtcdProcessHelper( cls.directory, proc_name=program, port_range_start=6001, internal_port_range_start=8001, - tls=True + tls=True, ) - with open(cls.client_all_cert, 'w') as f: - with open(cls.client_key_path, 'r') as g: + with open(cls.client_all_cert, "w") as f: + with open(cls.client_key_path, "r") as g: f.write(g.read()) - with open(cls.client_cert_path, 'r') as g: + with open(cls.client_cert_path, "r") as g: f.write(g.read()) - cls.processHelper.run(number=3, - proc_args=[ - '-cert-file=%s' % server_cert_path, - '-key-file=%s' % server_key_path, - '-ca-file=%s' % cls.ca_cert_path, - ]) - + cls.processHelper.run( + number=3, + proc_args=[ + "-cert-file=%s" % server_cert_path, + "-key-file=%s" % server_key_path, + "-ca-file=%s" % cls.ca_cert_path, + ], + ) def test_get_set_unauthenticated(self): - """ INTEGRATION: set/get a new value unauthenticated (http->https) """ + """INTEGRATION: set/get a new value unauthenticated (http->https)""" client = etcd.Client(port=6001) # See above for the reason of this change - self.assertRaises( - etcd.EtcdException, client.set, '/test_set', 'test-key') - self.assertRaises(etcd.EtcdException, client.get, '/test_set') + self.assertRaises(etcd.EtcdException, client.set, "/test_set", "test-key") + self.assertRaises(etcd.EtcdException, client.get, "/test_set") @nottest def test_get_set_authenticated(self): - """ INTEGRATION: connecting to server with mutual auth """ + """INTEGRATION: connecting to server with mutual auth""" # This gives an unexplicable ssl error, as connecting to the same # Etcd cluster where this fails with the exact same code this # doesn't fail client = etcd.Client( port=6001, - protocol='https', + protocol="https", cert=self.client_all_cert, - ca_cert=self.ca_cert_path + ca_cert=self.ca_cert_path, ) - set_result = client.set('/test_set', 'test-key') - self.assertEquals(u'set', set_result.action.lower()) - self.assertEquals(u'/test_set', set_result.key) - self.assertEquals(u'test-key', set_result.value) - get_result = client.get('/test_set') - self.assertEquals('get', get_result.action.lower()) - self.assertEquals('/test_set', get_result.key) - self.assertEquals('test-key', get_result.value) + set_result = client.set("/test_set", "test-key") + self.assertEquals("set", set_result.action.lower()) + self.assertEquals("/test_set", set_result.key) + self.assertEquals("test-key", set_result.value) + get_result = client.get("/test_set") + self.assertEquals("get", get_result.action.lower()) + self.assertEquals("/test_set", get_result.key) + self.assertEquals("test-key", get_result.value) diff --git a/src/etcd/tests/test_auth.py b/src/etcd/tests/test_auth.py index 5c8c0b07..e1d88dbb 100644 --- a/src/etcd/tests/test_auth.py +++ b/src/etcd/tests/test_auth.py @@ -8,18 +8,17 @@ class TestEtcdAuthBase(EtcdIntegrationTest): def setUp(self): # Sets up the root user, toggles auth - u = auth.EtcdUser(self.client, 'root') - u.password = 'testpass' + u = auth.EtcdUser(self.client, "root") + u.password = "testpass" u.write() - self.client = etcd.Client(port=6001, username='root', - password='testpass') + self.client = etcd.Client(port=6001, username="root", password="testpass") self.unauth_client = etcd.Client(port=6001) a = auth.Auth(self.client) a.active = True def tearDown(self): - u = auth.EtcdUser(self.client, 'test_user') - r = auth.EtcdRole(self.client, 'test_role') + u = auth.EtcdUser(self.client, "test_user") + r = auth.EtcdRole(self.client, "test_role") try: u.delete() except: @@ -34,11 +33,11 @@ def tearDown(self): class EtcdUserTest(TestEtcdAuthBase): def test_names(self): - u = auth.EtcdUser(self.client, 'test_user') - self.assertEquals(u.names, ['root']) + u = auth.EtcdUser(self.client, "test_user") + self.assertEquals(u.names, ["root"]) def test_read(self): - u = auth.EtcdUser(self.client, 'root') + u = auth.EtcdUser(self.client, "root") # Reading an existing user succeeds try: u.read() @@ -46,32 +45,33 @@ def test_read(self): self.fail("reading the root user raised an exception") # roles for said user are fetched - self.assertEquals(u.roles, set(['root'])) + self.assertEquals(u.roles, set(["root"])) # The user is correctly rendered out - self.assertEquals(u._to_net(), [{'user': 'root', 'password': None, - 'roles': ['root']}]) + self.assertEquals( + u._to_net(), [{"user": "root", "password": None, "roles": ["root"]}] + ) # An inexistent user raises the appropriate exception - u = auth.EtcdUser(self.client, 'user.does.not.exist') + u = auth.EtcdUser(self.client, "user.does.not.exist") self.assertRaises(etcd.EtcdKeyNotFound, u.read) # Reading with an unauthenticated client raises an exception - u = auth.EtcdUser(self.unauth_client, 'root') + u = auth.EtcdUser(self.unauth_client, "root") self.assertRaises(etcd.EtcdInsufficientPermissions, u.read) # Generic errors are caught c = etcd.Client(port=9999) - u = auth.EtcdUser(c, 'root') + u = auth.EtcdUser(c, "root") self.assertRaises(etcd.EtcdException, u.read) def test_write_and_delete(self): # Create an user - u = auth.EtcdUser(self.client, 'test_user') - u.roles.add('guest') - u.roles.add('root') + u = auth.EtcdUser(self.client, "test_user") + u.roles.add("guest") + u.roles.add("root") # directly from my suitcase - u.password = '123456' + u.password = "123456" try: u.write() except: @@ -81,35 +81,34 @@ def test_write_and_delete(self): u.read() # Verify we can log in as this user and access the auth (it has the # root role) - cl = etcd.Client(port=6001, username='test_user', - password='123456') - ul = auth.EtcdUser(cl, 'root') + cl = etcd.Client(port=6001, username="test_user", password="123456") + ul = auth.EtcdUser(cl, "root") try: ul.read() except etcd.EtcdInsufficientPermissions: self.fail("Reading auth with the new user is not possible") self.assertEquals(u.name, "test_user") - self.assertEquals(u.roles, set(['guest', 'root'])) + self.assertEquals(u.roles, set(["guest", "root"])) # set roles as a list, it works! - u.roles = ['guest', 'test_group'] + u.roles = ["guest", "test_group"] # We need this or the new API will return an internal error - r = auth.EtcdRole(self.client, 'test_group') - r.acls = {'*': 'R', '/test/*': 'RW'} + r = auth.EtcdRole(self.client, "test_group") + r.acls = {"*": "R", "/test/*": "RW"} r.write() try: u.write() except: self.fail("updating a user you previously created fails") u.read() - self.assertIn('test_group', u.roles) + self.assertIn("test_group", u.roles) # Unauthorized access is properly handled - ua = auth.EtcdUser(self.unauth_client, 'test_user') + ua = auth.EtcdUser(self.unauth_client, "test_user") self.assertRaises(etcd.EtcdInsufficientPermissions, ua.write) # now let's test deletion - du = auth.EtcdUser(self.client, 'user.does.not.exist') + du = auth.EtcdUser(self.client, "user.does.not.exist") self.assertRaises(etcd.EtcdKeyNotFound, du.delete) # Delete test_user @@ -121,45 +120,45 @@ def test_write_and_delete(self): class EtcdRoleTest(TestEtcdAuthBase): def test_names(self): - r = auth.EtcdRole(self.client, 'guest') - self.assertListEqual(r.names, [u'guest', u'root']) + r = auth.EtcdRole(self.client, "guest") + self.assertListEqual(r.names, ["guest", "root"]) def test_read(self): - r = auth.EtcdRole(self.client, 'guest') + r = auth.EtcdRole(self.client, "guest") try: r.read() except: - self.fail('Reading an existing role failed') + self.fail("Reading an existing role failed") # XXX The ACL path result changed from '*' to '/*' at some point # between etcd-2.2.2 and 2.2.5. They're equivalent so allow # for both. - if '/*' in r.acls: - self.assertEquals(r.acls, {'/*': 'RW'}) + if "/*" in r.acls: + self.assertEquals(r.acls, {"/*": "RW"}) else: - self.assertEquals(r.acls, {'*': 'RW'}) + self.assertEquals(r.acls, {"*": "RW"}) # We can actually skip most other read tests as they are common # with EtcdUser def test_write_and_delete(self): - r = auth.EtcdRole(self.client, 'test_role') - r.acls = {'*': 'R', '/test/*': 'RW'} + r = auth.EtcdRole(self.client, "test_role") + r.acls = {"*": "R", "/test/*": "RW"} try: r.write() except: self.fail("Writing a simple groups should not fail") - r1 = auth.EtcdRole(self.client, 'test_role') + r1 = auth.EtcdRole(self.client, "test_role") r1.read() self.assertEquals(r1.acls, r.acls) - r.revoke('/test/*', 'W') + r.revoke("/test/*", "W") r.write() r1.read() - self.assertEquals(r1.acls, {'*': 'R', '/test/*': 'R'}) - r.grant('/pub/*', 'RW') + self.assertEquals(r1.acls, {"*": "R", "/test/*": "R"}) + r.grant("/pub/*", "RW") r.write() r1.read() - self.assertEquals(r1.acls['/pub/*'], 'RW') + self.assertEquals(r1.acls["/pub/*"], "RW") # All other exceptions are tested by the user tests r1.name = None self.assertRaises(etcd.EtcdException, r1.write) diff --git a/src/etcd/tests/unit/__init__.py b/src/etcd/tests/unit/__init__.py index 9360b6bb..a1b95c44 100644 --- a/src/etcd/tests/unit/__init__.py +++ b/src/etcd/tests/unit/__init__.py @@ -2,6 +2,7 @@ import unittest import urllib3 import json + try: import mock except ImportError: @@ -9,15 +10,14 @@ class TestClientApiBase(unittest.TestCase): - def setUp(self): self.client = etcd.Client() def _prepare_response(self, s, d, cluster_id=None): if isinstance(d, dict): - data = json.dumps(d).encode('utf-8') + data = json.dumps(d).encode("utf-8") else: - data = d.encode('utf-8') + data = d.encode("utf-8") r = mock.create_autospec(urllib3.response.HTTPResponse)() r.status = s diff --git a/src/etcd/tests/unit/test_client.py b/src/etcd/tests/unit/test_client.py index d99de9bd..f8a51890 100644 --- a/src/etcd/tests/unit/test_client.py +++ b/src/etcd/tests/unit/test_client.py @@ -4,6 +4,7 @@ import dns.rdtypes.IN.SRV import dns.resolver from etcd.tests.unit import TestClientApiBase + try: import mock except ImportError: @@ -11,116 +12,111 @@ class TestClient(TestClientApiBase): - - def test_instantiate(self): - """ client can be instantiated""" + """client can be instantiated""" client = etcd.Client() assert client is not None def test_default_host(self): - """ default host is 127.0.0.1""" + """default host is 127.0.0.1""" client = etcd.Client() assert client.host == "127.0.0.1" def test_default_port(self): - """ default port is 4001""" + """default port is 4001""" client = etcd.Client() assert client.port == 4001 def test_default_prefix(self): client = etcd.Client() - assert client.version_prefix == '/v2' + assert client.version_prefix == "/v2" def test_default_protocol(self): - """ default protocol is http""" + """default protocol is http""" client = etcd.Client() - assert client.protocol == 'http' + assert client.protocol == "http" def test_default_read_timeout(self): - """ default read_timeout is 60""" + """default read_timeout is 60""" client = etcd.Client() assert client.read_timeout == 60 def test_default_allow_redirect(self): - """ default allow_redirect is True""" + """default allow_redirect is True""" client = etcd.Client() assert client.allow_redirect def test_default_username(self): - """ default username is None""" + """default username is None""" client = etcd.Client() assert client.username is None def test_default_password(self): - """ default username is None""" + """default username is None""" client = etcd.Client() assert client.password is None def test_set_host(self): - """ can change host """ - client = etcd.Client(host='192.168.1.1') - assert client.host == '192.168.1.1' + """can change host""" + client = etcd.Client(host="192.168.1.1") + assert client.host == "192.168.1.1" def test_set_port(self): - """ can change port """ + """can change port""" client = etcd.Client(port=4002) assert client.port == 4002 def test_set_prefix(self): - client = etcd.Client(version_prefix='/etcd') - assert client.version_prefix == '/etcd' + client = etcd.Client(version_prefix="/etcd") + assert client.version_prefix == "/etcd" def test_set_protocol(self): - """ can change protocol """ - client = etcd.Client(protocol='https') - assert client.protocol == 'https' + """can change protocol""" + client = etcd.Client(protocol="https") + assert client.protocol == "https" def test_set_read_timeout(self): - """ can set read_timeout """ + """can set read_timeout""" client = etcd.Client(read_timeout=45) assert client.read_timeout == 45 def test_set_allow_redirect(self): - """ can change allow_redirect """ + """can change allow_redirect""" client = etcd.Client(allow_redirect=False) assert not client.allow_redirect def test_default_base_uri(self): - """ default uri is http://127.0.0.1:4001 """ + """default uri is http://127.0.0.1:4001""" client = etcd.Client() - assert client.base_uri == 'http://127.0.0.1:4001' + assert client.base_uri == "http://127.0.0.1:4001" def test_set_base_uri(self): - """ can change base uri """ - client = etcd.Client( - host='192.168.1.1', - port=4003, - protocol='https') - assert client.base_uri == 'https://192.168.1.1:4003' + """can change base uri""" + client = etcd.Client(host="192.168.1.1", port=4003, protocol="https") + assert client.base_uri == "https://192.168.1.1:4003" def test_set_use_proxies(self): - """ can set the use_proxies flag """ - client = etcd.Client(use_proxies = True) + """can set the use_proxies flag""" + client = etcd.Client(use_proxies=True) assert client._use_proxies def test_set_username_only(self): - client = etcd.Client(username='username') + client = etcd.Client(username="username") assert client.username is None def test_set_password_only(self): - client = etcd.Client(password='password') + client = etcd.Client(password="password") assert client.password is None def test_set_username_password(self): - client = etcd.Client(username='username', password='password') - assert client.username == 'username' - assert client.password == 'password' + client = etcd.Client(username="username", password="password") + assert client.username == "username" + assert client.password == "password" def test_get_headers_with_auth(self): - client = etcd.Client(username='username', password='password') + client = etcd.Client(username="username", password="password") assert client._get_headers() == { - 'authorization': 'Basic dXNlcm5hbWU6cGFzc3dvcmQ=' + "authorization": "Basic dXNlcm5hbWU6cGFzc3dvcmQ=" } def test__set_version_info(self): @@ -132,13 +128,10 @@ def test__set_version_info(self): self.client._set_version_info() # Verify we call the proper endpoint - self.client.api_execute.assert_called_once_with( - '/version', - self.client._MGET - ) + self.client.api_execute.assert_called_once_with("/version", self.client._MGET) # Verify the properties while we are here - self.assertEqual('2.2.3', self.client.version) - self.assertEqual('2.3.0', self.client.cluster_version) + self.assertEqual("2.2.3", self.client.version) + self.assertEqual("2.3.0", self.client.cluster_version) def test_version_property(self): """Ensure the version property is set on first access.""" @@ -147,7 +140,7 @@ def test_version_property(self): self.client.api_execute.return_value.getheader.return_value = None # Verify the version property is set - self.assertEqual('2.2.3', self.client.version) + self.assertEqual("2.2.3", self.client.version) def test_cluster_version_property(self): """Ensure the cluster version property is set on first access.""" @@ -155,21 +148,21 @@ def test_cluster_version_property(self): self._mock_api(200, data) self.client.api_execute.return_value.getheader.return_value = None # Verify the cluster_version property is set - self.assertEqual('2.3.0', self.client.cluster_version) + self.assertEqual("2.3.0", self.client.cluster_version) def test_get_headers_without_auth(self): client = etcd.Client() assert client._get_headers() == {} def test_allow_reconnect(self): - """ Fails if allow_reconnect is false and a list of hosts is given""" + """Fails if allow_reconnect is false and a list of hosts is given""" with self.assertRaises(etcd.EtcdException): etcd.Client( - host=(('localhost', 4001), ('localhost', 4002)), + host=(("localhost", 4001), ("localhost", 4002)), ) # This doesn't raise an exception client = etcd.Client( - host=(('localhost', 4001), ('localhost', 4002)), + host=(("localhost", 4001), ("localhost", 4002)), allow_reconnect=True, use_proxies=True, ) @@ -177,21 +170,26 @@ def test_allow_reconnect(self): def test_discover(self): """Tests discovery.""" answers = [] - for i in range(1,3): + for i in range(1, 3): r = mock.create_autospec(dns.rdtypes.IN.SRV.SRV) r.port = 2379 try: method = dns.name.from_unicode except AttributeError: method = dns.name.from_text - r.target = method(u'etcd{}.example.com'.format(i)) + r.target = method("etcd{}.example.com".format(i)) answers.append(r) - dns.resolver.query = mock.create_autospec(dns.resolver.query, return_value=answers) + dns.resolver.query = mock.create_autospec( + dns.resolver.query, return_value=answers + ) self.machines = etcd.Client.machines - etcd.Client.machines = mock.create_autospec(etcd.Client.machines, return_value=[u'https://etcd2.example.com:2379']) - c = etcd.Client(srv_domain="example.com", allow_reconnect=True, protocol="https") + etcd.Client.machines = mock.create_autospec( + etcd.Client.machines, return_value=["https://etcd2.example.com:2379"] + ) + c = etcd.Client( + srv_domain="example.com", allow_reconnect=True, protocol="https" + ) etcd.Client.machines = self.machines - self.assertEqual(c.host, u'etcd1.example.com') + self.assertEqual(c.host, "etcd1.example.com") self.assertEqual(c.port, 2379) - self.assertEqual(c._machines_cache, - [u'https://etcd2.example.com:2379']) + self.assertEqual(c._machines_cache, ["https://etcd2.example.com:2379"]) diff --git a/src/etcd/tests/unit/test_lock.py b/src/etcd/tests/unit/test_lock.py index 7384084a..792690b1 100644 --- a/src/etcd/tests/unit/test_lock.py +++ b/src/etcd/tests/unit/test_lock.py @@ -1,4 +1,5 @@ import etcd + try: import mock except ImportError: @@ -7,66 +8,74 @@ class TestClientLock(TestClientApiBase): - def recursive_read(self): nodes = [ - {"key": "/_locks/test_lock/1", "value": "2qwwwq", - "modifiedIndex":33,"createdIndex":33}, - {"key": "/_locks/test_lock/34", "value": self.locker.uuid, - "modifiedIndex":34,"createdIndex":34}, + { + "key": "/_locks/test_lock/1", + "value": "2qwwwq", + "modifiedIndex": 33, + "createdIndex": 33, + }, + { + "key": "/_locks/test_lock/34", + "value": self.locker.uuid, + "modifiedIndex": 34, + "createdIndex": 34, + }, ] d = { "action": "get", - "node": {"dir": True, - "nodes": [{"key":"/_locks/test_lock", "dir": True, - "nodes": nodes}]} + "node": { + "dir": True, + "nodes": [{"key": "/_locks/test_lock", "dir": True, "nodes": nodes}], + }, } self._mock_api(200, d) def setUp(self): super(TestClientLock, self).setUp() - self.locker = etcd.Lock(self.client, 'test_lock') + self.locker = etcd.Lock(self.client, "test_lock") def test_initialization(self): """ Verify the lock gets initialized correctly """ - self.assertEqual(self.locker.name, u'test_lock') - self.assertEqual(self.locker.path, u'/_locks/test_lock') + self.assertEqual(self.locker.name, "test_lock") + self.assertEqual(self.locker.path, "/_locks/test_lock") self.assertEqual(self.locker.is_taken, False) def test_acquire(self): """ Acquiring a precedingly inexistent lock works. """ - l = etcd.Lock(self.client, 'test_lock') + l = etcd.Lock(self.client, "test_lock") l._find_lock = mock.MagicMock(spec=l._find_lock, return_value=False) l._acquired = mock.MagicMock(spec=l._acquired, return_value=True) # Mock the write d = { - u'action': u'set', - u'node': { - u'modifiedIndex': 190, - u'key': u'/_locks/test_lock/1', - u'value': l.uuid - } + "action": "set", + "node": { + "modifiedIndex": 190, + "key": "/_locks/test_lock/1", + "value": l.uuid, + }, } self._mock_api(200, d) self.assertEqual(l.acquire(), True) - self.assertEqual(l._sequence, '1') + self.assertEqual(l._sequence, "1") def test_is_acquired(self): """ Test is_acquired """ - self.locker._sequence = '1' + self.locker._sequence = "1" d = { - u'action': u'get', - u'node': { - u'modifiedIndex': 190, - u'key': u'/_locks/test_lock/1', - u'value': self.locker.uuid - } + "action": "get", + "node": { + "modifiedIndex": 190, + "key": "/_locks/test_lock/1", + "value": self.locker.uuid, + }, } self._mock_api(200, d) self.locker.is_taken = True @@ -76,7 +85,7 @@ def test_is_not_acquired(self): """ Test is_acquired failures """ - self.locker._sequence = '2' + self.locker._sequence = "2" self.locker.is_taken = False self.assertEqual(self.locker.is_acquired, False) self.locker.is_taken = True @@ -88,56 +97,67 @@ def test_acquired(self): """ Test the acquiring primitives """ - self.locker._sequence = '4' - retval = ('/_locks/test_lock/4', None) + self.locker._sequence = "4" + retval = ("/_locks/test_lock/4", None) self.locker._get_locker = mock.MagicMock( - spec=self.locker._get_locker, return_value=retval) + spec=self.locker._get_locker, return_value=retval + ) self.assertTrue(self.locker._acquired()) self.assertTrue(self.locker.is_taken) - retval = ('/_locks/test_lock/1', '/_locks/test_lock/4') + retval = ("/_locks/test_lock/1", "/_locks/test_lock/4") self.locker._get_locker = mock.MagicMock(return_value=retval) self.assertFalse(self.locker._acquired(blocking=False)) self.assertFalse(self.locker.is_taken) d = { - u'action': u'delete', - u'node': { - u'modifiedIndex': 190, - u'key': u'/_locks/test_lock/1', - u'value': self.locker.uuid - } + "action": "delete", + "node": { + "modifiedIndex": 190, + "key": "/_locks/test_lock/1", + "value": self.locker.uuid, + }, } self._mock_api(200, d) - returns = [('/_locks/test_lock/1', '/_locks/test_lock/4'), ('/_locks/test_lock/4', None)] + returns = [ + ("/_locks/test_lock/1", "/_locks/test_lock/4"), + ("/_locks/test_lock/4", None), + ] def side_effect(): return returns.pop() self.locker._get_locker = mock.MagicMock( - spec=self.locker._get_locker, side_effect=side_effect) + spec=self.locker._get_locker, side_effect=side_effect + ) self.assertTrue(self.locker._acquired()) def test_acquired_no_timeout(self): self.locker._sequence = 4 returns = [ - ('/_locks/test_lock/4', None), - ('/_locks/test_lock/1', etcd.EtcdResult(node={"key": '/_locks/test_lock/4', "modifiedIndex": 1})) + ("/_locks/test_lock/4", None), + ( + "/_locks/test_lock/1", + etcd.EtcdResult( + node={"key": "/_locks/test_lock/4", "modifiedIndex": 1} + ), + ), ] def side_effect(): return returns.pop() d = { - u'action': u'get', - u'node': { - u'modifiedIndex': 190, - u'key': u'/_locks/test_lock/4', - u'value': self.locker.uuid - } + "action": "get", + "node": { + "modifiedIndex": 190, + "key": "/_locks/test_lock/4", + "value": self.locker.uuid, + }, } self._mock_api(200, d) self.locker._get_locker = mock.create_autospec( - self.locker._get_locker, side_effect=side_effect) + self.locker._get_locker, side_effect=side_effect + ) self.assertTrue(self.locker._acquired()) def test_lock_key(self): @@ -146,24 +166,24 @@ def test_lock_key(self): """ with self.assertRaises(ValueError): self.locker.lock_key - self.locker._sequence = '5' - self.assertEqual(u'/_locks/test_lock/5',self.locker.lock_key) + self.locker._sequence = "5" + self.assertEqual("/_locks/test_lock/5", self.locker.lock_key) def test_set_sequence(self): - self.locker._set_sequence('/_locks/test_lock/10') - self.assertEqual('10', self.locker._sequence) + self.locker._set_sequence("/_locks/test_lock/10") + self.assertEqual("10", self.locker._sequence) def test_find_lock(self): d = { - u'action': u'get', - u'node': { - u'modifiedIndex': 190, - u'key': u'/_locks/test_lock/1', - u'value': self.locker.uuid - } + "action": "get", + "node": { + "modifiedIndex": 190, + "key": "/_locks/test_lock/1", + "value": self.locker.uuid, + }, } self._mock_api(200, d) - self.locker._sequence = '1' + self.locker._sequence = "1" self.assertTrue(self.locker._find_lock()) # Now let's pretend the lock is not there self._mock_exception(etcd.EtcdKeyNotFound, self.locker.lock_key) @@ -171,24 +191,42 @@ def test_find_lock(self): self.locker._sequence = None self.recursive_read() self.assertTrue(self.locker._find_lock()) - self.assertEqual(self.locker._sequence, '34') + self.assertEqual(self.locker._sequence, "34") def test_get_locker(self): self.recursive_read() - self.assertEqual((u'/_locks/test_lock/1', etcd.EtcdResult(node={'newKey': False, '_children': [], 'createdIndex': 33, 'modifiedIndex': 33, 'value': u'2qwwwq', 'expiration': None, 'key': u'/_locks/test_lock/1', 'ttl': None, 'action': None, 'dir': False})), - self.locker._get_locker()) + self.assertEqual( + ( + "/_locks/test_lock/1", + etcd.EtcdResult( + node={ + "newKey": False, + "_children": [], + "createdIndex": 33, + "modifiedIndex": 33, + "value": "2qwwwq", + "expiration": None, + "key": "/_locks/test_lock/1", + "ttl": None, + "action": None, + "dir": False, + } + ), + ), + self.locker._get_locker(), + ) with self.assertRaises(etcd.EtcdLockExpired): - self.locker._sequence = '35' + self.locker._sequence = "35" self.locker._get_locker() def test_release(self): d = { - u'action': u'delete', - u'node': { - u'modifiedIndex': 190, - u'key': u'/_locks/test_lock/1', - u'value': self.locker.uuid - } + "action": "delete", + "node": { + "modifiedIndex": 190, + "key": "/_locks/test_lock/1", + "value": self.locker.uuid, + }, } self._mock_api(200, d) self.locker._sequence = 1 diff --git a/src/etcd/tests/unit/test_old_request.py b/src/etcd/tests/unit/test_old_request.py index 5fb75581..105a90ae 100644 --- a/src/etcd/tests/unit/test_old_request.py +++ b/src/etcd/tests/unit/test_old_request.py @@ -1,5 +1,6 @@ import etcd import unittest + try: import mock except ImportError: @@ -9,10 +10,9 @@ class FakeHTTPResponse(object): - - def __init__(self, status, data='', headers=None): + def __init__(self, status, data="", headers=None): self.status = status - self.data = data.encode('utf-8') + self.data = data.encode("utf-8") self.headers = headers or { "x-etcd-cluster-id": "abdef12345", } @@ -25,340 +25,389 @@ def getheader(self, header): class TestClientRequest(unittest.TestCase): - def test_set(self): - """ Can set a value """ + """Can set a value""" client = etcd.Client() client.api_execute = mock.Mock( - return_value=FakeHTTPResponse(201, - '{"action":"SET",' - '"node": {' - '"key":"/testkey",' - '"value":"test",' - '"newKey":true,' - '"expiration":"2013-09-14T00:56:59.316195568+02:00",' - '"ttl":19,"modifiedIndex":183}}') + return_value=FakeHTTPResponse( + 201, + '{"action":"SET",' + '"node": {' + '"key":"/testkey",' + '"value":"test",' + '"newKey":true,' + '"expiration":"2013-09-14T00:56:59.316195568+02:00",' + '"ttl":19,"modifiedIndex":183}}', + ) ) - result = client.set('/testkey', 'test', ttl=19) + result = client.set("/testkey", "test", ttl=19) self.assertEqual( etcd.EtcdResult( - **{u'action': u'SET', - 'node': { - u'expiration': u'2013-09-14T00:56:59.316195568+02:00', - u'modifiedIndex': 183, - u'key': u'/testkey', - u'newKey': True, - u'ttl': 19, - u'value': u'test'}}), result) + **{ + "action": "SET", + "node": { + "expiration": "2013-09-14T00:56:59.316195568+02:00", + "modifiedIndex": 183, + "key": "/testkey", + "newKey": True, + "ttl": 19, + "value": "test", + }, + } + ), + result, + ) def test_test_and_set(self): - """ Can test and set a value """ + """Can test and set a value""" client = etcd.Client() client.api_execute = mock.Mock( - return_value=FakeHTTPResponse(200, - '{"action":"SET",' - '"node": {' - '"key":"/testkey",' - '"prevValue":"test",' - '"value":"newvalue",' - '"expiration":"2013-09-14T02:09:44.24390976+02:00",' - '"ttl":49,"modifiedIndex":203}}') + return_value=FakeHTTPResponse( + 200, + '{"action":"SET",' + '"node": {' + '"key":"/testkey",' + '"prevValue":"test",' + '"value":"newvalue",' + '"expiration":"2013-09-14T02:09:44.24390976+02:00",' + '"ttl":49,"modifiedIndex":203}}', + ) ) - result = client.test_and_set('/testkey', 'newvalue', 'test', ttl=19) + result = client.test_and_set("/testkey", "newvalue", "test", ttl=19) self.assertEqual( etcd.EtcdResult( - **{u'action': u'SET', - u'node': { - u'expiration': u'2013-09-14T02:09:44.24390976+02:00', - u'modifiedIndex': 203, - u'key': u'/testkey', - u'prevValue': u'test', - u'ttl': 49, - u'value': u'newvalue'} - }), result) + **{ + "action": "SET", + "node": { + "expiration": "2013-09-14T02:09:44.24390976+02:00", + "modifiedIndex": 203, + "key": "/testkey", + "prevValue": "test", + "ttl": 49, + "value": "newvalue", + }, + } + ), + result, + ) def test_test_and_test_failure(self): - """ Exception will be raised if prevValue != value in test_set """ + """Exception will be raised if prevValue != value in test_set""" client = etcd.Client() client.api_execute = mock.Mock( side_effect=ValueError( - 'The given PrevValue is not equal' - ' to the value of the key : TestAndSet: 1!=3')) + "The given PrevValue is not equal" + " to the value of the key : TestAndSet: 1!=3" + ) + ) try: - result = client.test_and_set( - '/testkey', - 'newvalue', - 'test', ttl=19) + result = client.test_and_set("/testkey", "newvalue", "test", ttl=19) except ValueError as e: - #from ipdb import set_trace; set_trace() + # from ipdb import set_trace; set_trace() self.assertEqual( - 'The given PrevValue is not equal' - ' to the value of the key : TestAndSet: 1!=3', str(e)) + "The given PrevValue is not equal" + " to the value of the key : TestAndSet: 1!=3", + str(e), + ) def test_delete(self): - """ Can delete a value """ + """Can delete a value""" client = etcd.Client() client.api_execute = mock.Mock( - return_value=FakeHTTPResponse(200, - '{"action":"DELETE",' - '"node": {' - '"key":"/testkey",' - '"prevValue":"test",' - '"expiration":"2013-09-14T01:06:35.5242587+02:00",' - '"modifiedIndex":189}}') + return_value=FakeHTTPResponse( + 200, + '{"action":"DELETE",' + '"node": {' + '"key":"/testkey",' + '"prevValue":"test",' + '"expiration":"2013-09-14T01:06:35.5242587+02:00",' + '"modifiedIndex":189}}', + ) + ) + result = client.delete("/testkey") + self.assertEqual( + etcd.EtcdResult( + **{ + "action": "DELETE", + "node": { + "expiration": "2013-09-14T01:06:35.5242587+02:00", + "modifiedIndex": 189, + "key": "/testkey", + "prevValue": "test", + }, + } + ), + result, ) - result = client.delete('/testkey') - self.assertEqual(etcd.EtcdResult( - **{u'action': u'DELETE', - u'node': { - u'expiration': u'2013-09-14T01:06:35.5242587+02:00', - u'modifiedIndex': 189, - u'key': u'/testkey', - u'prevValue': u'test'} - }), result) def test_get(self): - """ Can get a value """ + """Can get a value""" client = etcd.Client() client.api_execute = mock.Mock( - return_value=FakeHTTPResponse(200, - '{"action":"GET",' - '"node": {' - '"key":"/testkey",' - '"value":"test",' - '"modifiedIndex":190}}') + return_value=FakeHTTPResponse( + 200, + '{"action":"GET",' + '"node": {' + '"key":"/testkey",' + '"value":"test",' + '"modifiedIndex":190}}', + ) ) - result = client.get('/testkey') - self.assertEqual(etcd.EtcdResult( - **{u'action': u'GET', - u'node': { - u'modifiedIndex': 190, - u'key': u'/testkey', - u'value': u'test'} - }), result) + result = client.get("/testkey") + self.assertEqual( + etcd.EtcdResult( + **{ + "action": "GET", + "node": {"modifiedIndex": 190, "key": "/testkey", "value": "test"}, + } + ), + result, + ) def test_get_multi(self): """Can get multiple values""" pass def test_get_subdirs(self): - """ Can understand dirs in results """ + """Can understand dirs in results""" pass def test_not_in(self): - """ Can check if key is not in client """ + """Can check if key is not in client""" client = etcd.Client() client.get = mock.Mock(side_effect=etcd.EtcdKeyNotFound()) - result = '/testkey' not in client + result = "/testkey" not in client self.assertEqual(True, result) def test_in(self): - """ Can check if key is in client """ + """Can check if key is in client""" client = etcd.Client() client.api_execute = mock.Mock( - return_value=FakeHTTPResponse(200, - '{"action":"GET",' - '"node": {' - '"key":"/testkey",' - '"value":"test",' - '"modifiedIndex":190}}') + return_value=FakeHTTPResponse( + 200, + '{"action":"GET",' + '"node": {' + '"key":"/testkey",' + '"value":"test",' + '"modifiedIndex":190}}', + ) ) - result = '/testkey' in client + result = "/testkey" in client self.assertEqual(True, result) def test_simple_watch(self): - """ Can watch values """ + """Can watch values""" client = etcd.Client() client.api_execute = mock.Mock( - return_value=FakeHTTPResponse(200, - '{"action":"SET",' - '"node": {' - '"key":"/testkey",' - '"value":"test",' - '"newKey":true,' - '"expiration":"2013-09-14T01:35:07.623681365+02:00",' - '"ttl":19,' - '"modifiedIndex":192}}') + return_value=FakeHTTPResponse( + 200, + '{"action":"SET",' + '"node": {' + '"key":"/testkey",' + '"value":"test",' + '"newKey":true,' + '"expiration":"2013-09-14T01:35:07.623681365+02:00",' + '"ttl":19,' + '"modifiedIndex":192}}', + ) ) - result = client.watch('/testkey') + result = client.watch("/testkey") self.assertEqual( etcd.EtcdResult( - **{u'action': u'SET', - u'node': { - u'expiration': u'2013-09-14T01:35:07.623681365+02:00', - u'modifiedIndex': 192, - u'key': u'/testkey', - u'newKey': True, - u'ttl': 19, - u'value': u'test'} - }), result) + **{ + "action": "SET", + "node": { + "expiration": "2013-09-14T01:35:07.623681365+02:00", + "modifiedIndex": 192, + "key": "/testkey", + "newKey": True, + "ttl": 19, + "value": "test", + }, + } + ), + result, + ) def test_index_watch(self): - """ Can watch values from index """ + """Can watch values from index""" client = etcd.Client() client.api_execute = mock.Mock( - return_value=FakeHTTPResponse(200, - '{"action":"SET",' - '"node": {' - '"key":"/testkey",' - '"value":"test",' - '"newKey":true,' - '"expiration":"2013-09-14T01:35:07.623681365+02:00",' - '"ttl":19,' - '"modifiedIndex":180}}') + return_value=FakeHTTPResponse( + 200, + '{"action":"SET",' + '"node": {' + '"key":"/testkey",' + '"value":"test",' + '"newKey":true,' + '"expiration":"2013-09-14T01:35:07.623681365+02:00",' + '"ttl":19,' + '"modifiedIndex":180}}', + ) ) - result = client.watch('/testkey', index=180) + result = client.watch("/testkey", index=180) self.assertEqual( etcd.EtcdResult( - **{u'action': u'SET', - u'node': { - u'expiration': u'2013-09-14T01:35:07.623681365+02:00', - u'modifiedIndex': 180, - u'key': u'/testkey', - u'newKey': True, - u'ttl': 19, - u'value': u'test'} - }), result) + **{ + "action": "SET", + "node": { + "expiration": "2013-09-14T01:35:07.623681365+02:00", + "modifiedIndex": 180, + "key": "/testkey", + "newKey": True, + "ttl": 19, + "value": "test", + }, + } + ), + result, + ) class TestEventGenerator(object): - def check_watch(self, result): - assert etcd.EtcdResult( - **{u'action': u'SET', - u'node': { - u'expiration': u'2013-09-14T01:35:07.623681365+02:00', - u'modifiedIndex': 180, - u'key': u'/testkey', - u'newKey': True, - u'ttl': 19, - u'value': u'test'} - }) == result + assert ( + etcd.EtcdResult( + **{ + "action": "SET", + "node": { + "expiration": "2013-09-14T01:35:07.623681365+02:00", + "modifiedIndex": 180, + "key": "/testkey", + "newKey": True, + "ttl": 19, + "value": "test", + }, + } + ) + == result + ) def test_eternal_watch(self): - """ Can watch values from generator """ + """Can watch values from generator""" client = etcd.Client() client.api_execute = mock.Mock( - return_value=FakeHTTPResponse(200, - '{"action":"SET",' - '"node": {' - '"key":"/testkey",' - '"value":"test",' - '"newKey":true,' - '"expiration":"2013-09-14T01:35:07.623681365+02:00",' - '"ttl":19,' - '"modifiedIndex":180}}') + return_value=FakeHTTPResponse( + 200, + '{"action":"SET",' + '"node": {' + '"key":"/testkey",' + '"value":"test",' + '"newKey":true,' + '"expiration":"2013-09-14T01:35:07.623681365+02:00",' + '"ttl":19,' + '"modifiedIndex":180}}', + ) ) for result in range(1, 5): - result = next(client.eternal_watch('/testkey', index=180)) + result = next(client.eternal_watch("/testkey", index=180)) yield self.check_watch, result class TestClientApiExecutor(unittest.TestCase): - def test_get(self): - """ http get request """ + """http get request""" client = etcd.Client() - response = FakeHTTPResponse(status=200, data='arbitrary json data') + response = FakeHTTPResponse(status=200, data="arbitrary json data") client.http.request = mock.Mock(return_value=response) - result = client.api_execute('/v1/keys/testkey', client._MGET) - self.assertEqual('arbitrary json data'.encode('utf-8'), result.data) + result = client.api_execute("/v1/keys/testkey", client._MGET) + self.assertEqual("arbitrary json data".encode("utf-8"), result.data) def test_delete(self): - """ http delete request """ + """http delete request""" client = etcd.Client() - response = FakeHTTPResponse(status=200, data='arbitrary json data') + response = FakeHTTPResponse(status=200, data="arbitrary json data") client.http.request = mock.Mock(return_value=response) - result = client.api_execute('/v1/keys/testkey', client._MDELETE) - self.assertEqual('arbitrary json data'.encode('utf-8'), result.data) + result = client.api_execute("/v1/keys/testkey", client._MDELETE) + self.assertEqual("arbitrary json data".encode("utf-8"), result.data) def test_get_error(self): - """ http get error request 101""" + """http get error request 101""" client = etcd.Client() - response = FakeHTTPResponse(status=400, - data='{"message": "message",' - ' "cause": "cause",' - ' "errorCode": 100}') + response = FakeHTTPResponse( + status=400, + data='{"message": "message",' ' "cause": "cause",' ' "errorCode": 100}', + ) client.http.request = mock.Mock(return_value=response) try: - client.api_execute('/v2/keys/testkey', client._MGET) + client.api_execute("/v2/keys/testkey", client._MGET) assert False except etcd.EtcdKeyNotFound as e: - self.assertEqual(str(e), 'message : cause') + self.assertEqual(str(e), "message : cause") def test_put(self): - """ http put request """ + """http put request""" client = etcd.Client() - response = FakeHTTPResponse(status=200, data='arbitrary json data') + response = FakeHTTPResponse(status=200, data="arbitrary json data") client.http.request_encode_body = mock.Mock(return_value=response) - result = client.api_execute('/v2/keys/testkey', client._MPUT) - self.assertEqual('arbitrary json data'.encode('utf-8'), result.data) + result = client.api_execute("/v2/keys/testkey", client._MPUT) + self.assertEqual("arbitrary json data".encode("utf-8"), result.data) def test_test_and_set_error(self): - """ http post error request 101 """ + """http post error request 101""" client = etcd.Client() response = FakeHTTPResponse( status=400, - data='{"message": "message", "cause": "cause", "errorCode": 101}') + data='{"message": "message", "cause": "cause", "errorCode": 101}', + ) client.http.request_encode_body = mock.Mock(return_value=response) - payload = {'value': 'value', 'prevValue': 'oldValue', 'ttl': '60'} + payload = {"value": "value", "prevValue": "oldValue", "ttl": "60"} try: - client.api_execute('/v2/keys/testkey', client._MPUT, payload) + client.api_execute("/v2/keys/testkey", client._MPUT, payload) self.fail() except ValueError as e: - self.assertEqual('message : cause', str(e)) + self.assertEqual("message : cause", str(e)) def test_set_not_file_error(self): - """ http post error request 102 """ + """http post error request 102""" client = etcd.Client() response = FakeHTTPResponse( status=400, - data='{"message": "message", "cause": "cause", "errorCode": 102}') + data='{"message": "message", "cause": "cause", "errorCode": 102}', + ) client.http.request_encode_body = mock.Mock(return_value=response) - payload = {'value': 'value', 'prevValue': 'oldValue', 'ttl': '60'} + payload = {"value": "value", "prevValue": "oldValue", "ttl": "60"} try: - client.api_execute('/v2/keys/testkey', client._MPUT, payload) + client.api_execute("/v2/keys/testkey", client._MPUT, payload) self.fail() except etcd.EtcdNotFile as e: - self.assertEqual('message : cause', str(e)) + self.assertEqual("message : cause", str(e)) def test_get_error_unknown(self): - """ http get error request unknown """ + """http get error request unknown""" client = etcd.Client() - response = FakeHTTPResponse(status=400, - data='{"message": "message",' - ' "cause": "cause",' - ' "errorCode": 42}') + response = FakeHTTPResponse( + status=400, + data='{"message": "message",' ' "cause": "cause",' ' "errorCode": 42}', + ) client.http.request = mock.Mock(return_value=response) try: - client.api_execute('/v2/keys/testkey', client._MGET) + client.api_execute("/v2/keys/testkey", client._MGET) self.fail() except etcd.EtcdException as e: self.assertEqual(str(e), "message : cause") def test_get_error_request_invalid(self): - """ http get error request invalid """ + """http get error request invalid""" client = etcd.Client() - response = FakeHTTPResponse(status=400, - data='{)*garbage') + response = FakeHTTPResponse(status=400, data="{)*garbage") client.http.request = mock.Mock(return_value=response) try: - client.api_execute('/v2/keys/testkey', client._MGET) + client.api_execute("/v2/keys/testkey", client._MGET) self.fail() except etcd.EtcdException as e: - self.assertEqual(str(e), - "Bad response : {)*garbage") + self.assertEqual(str(e), "Bad response : {)*garbage") def test_get_error_invalid(self): - """ http get error request invalid """ + """http get error request invalid""" client = etcd.Client() - response = FakeHTTPResponse(status=400, - data='{){){)*garbage*') + response = FakeHTTPResponse(status=400, data="{){){)*garbage*") client.http.request = mock.Mock(return_value=response) - self.assertRaises(etcd.EtcdException, client.api_execute, - '/v2/keys/testkey', client._MGET) + self.assertRaises( + etcd.EtcdException, client.api_execute, "/v2/keys/testkey", client._MGET + ) diff --git a/src/etcd/tests/unit/test_request.py b/src/etcd/tests/unit/test_request.py index 9c0fcd64..6270a3c1 100644 --- a/src/etcd/tests/unit/test_request.py +++ b/src/etcd/tests/unit/test_request.py @@ -11,66 +11,54 @@ class TestClientApiInternals(TestClientApiBase): - def test_read_default_timeout(self): - """ Read timeout set to the default """ + """Read timeout set to the default""" d = { - u'action': u'get', - u'node': { - u'modifiedIndex': 190, - u'key': u'/testkey', - u'value': u'test' - } + "action": "get", + "node": {"modifiedIndex": 190, "key": "/testkey", "value": "test"}, } self._mock_api(200, d) - res = self.client.read('/testkey') - self.assertEqual(self.client.api_execute.call_args[1]['timeout'], None) + res = self.client.read("/testkey") + self.assertEqual(self.client.api_execute.call_args[1]["timeout"], None) def test_read_custom_timeout(self): - """ Read timeout set to the supplied value """ + """Read timeout set to the supplied value""" d = { - u'action': u'get', - u'node': { - u'modifiedIndex': 190, - u'key': u'/testkey', - u'value': u'test' - } + "action": "get", + "node": {"modifiedIndex": 190, "key": "/testkey", "value": "test"}, } self._mock_api(200, d) - self.client.read('/testkey', timeout=15) - self.assertEqual(self.client.api_execute.call_args[1]['timeout'], 15) + self.client.read("/testkey", timeout=15) + self.assertEqual(self.client.api_execute.call_args[1]["timeout"], 15) def test_read_no_timeout(self): - """ Read timeout disabled """ + """Read timeout disabled""" d = { - u'action': u'get', - u'node': { - u'modifiedIndex': 190, - u'key': u'/testkey', - u'value': u'test' - } + "action": "get", + "node": {"modifiedIndex": 190, "key": "/testkey", "value": "test"}, } self._mock_api(200, d) - self.client.read('/testkey', timeout=0) - self.assertEqual(self.client.api_execute.call_args[1]['timeout'], 0) + self.client.read("/testkey", timeout=0) + self.assertEqual(self.client.api_execute.call_args[1]["timeout"], 0) def test_write_no_params(self): - """ Calling `write` without a value argument will omit the `value` from - the API call params """ + """Calling `write` without a value argument will omit the `value` from + the API call params""" d = { - u'action': u'set', - u'node': { - u'createdIndex': 17, - u'dir': True, - u'key': u'/newdir', - u'modifiedIndex': 17 - } + "action": "set", + "node": { + "createdIndex": 17, + "dir": True, + "key": "/newdir", + "modifiedIndex": 17, + }, } self._mock_api(200, d) - self.client.write('/newdir', None, dir=True) - self.assertEqual(self.client.api_execute.call_args, - (('/v2/keys/newdir', 'PUT'), - dict(params={'dir': 'true'}))) + self.client.write("/newdir", None, dir=True) + self.assertEqual( + self.client.api_execute.call_args, + (("/v2/keys/newdir", "PUT"), dict(params={"dir": "true"})), + ) class TestClientApiInterface(TestClientApiBase): @@ -79,65 +67,72 @@ class TestClientApiInterface(TestClientApiBase): If a test should be run only in this class, please override the method there. """ - @mock.patch('urllib3.request.RequestMethods.request') + + @mock.patch("urllib3.request.RequestMethods.request") def test_machines(self, mocker): - """ Can request machines """ - data = ['http://127.0.0.1:4001', - 'http://127.0.0.1:4002', 'http://127.0.0.1:4003'] - d = ','.join(data) + """Can request machines""" + data = [ + "http://127.0.0.1:4001", + "http://127.0.0.1:4002", + "http://127.0.0.1:4003", + ] + d = ",".join(data) mocker.return_value = self._prepare_response(200, d) self.assertEqual(data, self.client.machines) - @mock.patch('etcd.Client.machines', new_callable=mock.PropertyMock) + @mock.patch("etcd.Client.machines", new_callable=mock.PropertyMock) def test_use_proxies(self, mocker): """Do not overwrite the machines cache when using proxies""" - mocker.return_value = ['https://10.0.0.2:4001', - 'https://10.0.0.3:4001', - 'https://10.0.0.4:4001'] + mocker.return_value = [ + "https://10.0.0.2:4001", + "https://10.0.0.3:4001", + "https://10.0.0.4:4001", + ] c = etcd.Client( - host=(('localhost', 4001), ('localproxy', 4001)), - protocol='https', + host=(("localhost", 4001), ("localproxy", 4001)), + protocol="https", allow_reconnect=True, - use_proxies=True + use_proxies=True, ) - self.assertEqual(c._machines_cache, ['https://localproxy:4001']) - self.assertEqual(c._base_uri, 'https://localhost:4001') + self.assertEqual(c._machines_cache, ["https://localproxy:4001"]) + self.assertEqual(c._base_uri, "https://localhost:4001") self.assertNotIn(c.base_uri, c._machines_cache) c = etcd.Client( - host=(('localhost', 4001), ('10.0.0.2', 4001)), - protocol='https', + host=(("localhost", 4001), ("10.0.0.2", 4001)), + protocol="https", allow_reconnect=True, - use_proxies=False + use_proxies=False, ) - self.assertIn('https://10.0.0.3:4001', c._machines_cache) + self.assertIn("https://10.0.0.3:4001", c._machines_cache) self.assertNotIn(c.base_uri, c._machines_cache) def test_members(self): - """ Can request machines """ + """Can request machines""" data = { - "members": - [ + "members": [ { "id": "ce2a822cea30bfca", "name": "default", "peerURLs": ["http://localhost:2380", "http://localhost:7001"], - "clientURLs": ["http://127.0.0.1:4001"] + "clientURLs": ["http://127.0.0.1:4001"], } ] } self._mock_api(200, data) - self.assertEqual(self.client.members["ce2a822cea30bfca"]["id"], "ce2a822cea30bfca") + self.assertEqual( + self.client.members["ce2a822cea30bfca"]["id"], "ce2a822cea30bfca" + ) def test_self_stats(self): - """ Request for stats """ + """Request for stats""" data = { "id": "eca0338f4ea31566", "leaderInfo": { "leader": "8a69d5f6b7814500", "startTime": "2014-10-24T13:15:51.186620747-07:00", - "uptime": "10m59.322358947s" + "uptime": "10m59.322358947s", }, "name": "node3", "recvAppendRequestCnt": 5944, @@ -145,285 +140,269 @@ def test_self_stats(self): "recvPkgRate": 9.00892789741075, "sendAppendRequestCnt": 0, "startTime": "2014-10-24T13:15:50.072007085-07:00", - "state": "StateFollower" + "state": "StateFollower", } - self._mock_api(200,data) - self.assertEqual(self.client.stats['name'], "node3") + self._mock_api(200, data) + self.assertEqual(self.client.stats["name"], "node3") def test_leader_stats(self): - """ Request for leader stats """ + """Request for leader stats""" data = {"leader": "924e2e83e93f2560", "followers": {}} - self._mock_api(200,data) - self.assertEqual(self.client.leader_stats['leader'], "924e2e83e93f2560") - + self._mock_api(200, data) + self.assertEqual(self.client.leader_stats["leader"], "924e2e83e93f2560") - @mock.patch('etcd.Client.members', new_callable=mock.PropertyMock) + @mock.patch("etcd.Client.members", new_callable=mock.PropertyMock) def test_leader(self, mocker): - """ Can request the leader """ + """Can request the leader""" members = {"ce2a822cea30bfca": {"id": "ce2a822cea30bfca", "name": "default"}} mocker.return_value = members - self._mock_api(200, {"leaderInfo":{"leader": "ce2a822cea30bfca", "followers": {}}}) + self._mock_api( + 200, {"leaderInfo": {"leader": "ce2a822cea30bfca", "followers": {}}} + ) self.assertEqual(self.client.leader, members["ce2a822cea30bfca"]) def test_set_plain(self): - """ Can set a value """ - d = {u'action': u'set', - u'node': { - u'expiration': u'2013-09-14T00:56:59.316195568+02:00', - u'modifiedIndex': 183, - u'key': u'/testkey', - u'ttl': 19, - u'value': u'test' - } - } + """Can set a value""" + d = { + "action": "set", + "node": { + "expiration": "2013-09-14T00:56:59.316195568+02:00", + "modifiedIndex": 183, + "key": "/testkey", + "ttl": 19, + "value": "test", + }, + } self._mock_api(200, d) - res = self.client.write('/testkey', 'test') + res = self.client.write("/testkey", "test") self.assertEqual(res, etcd.EtcdResult(**d)) def test_update(self): """Can update a result.""" - d = {u'action': u'set', - u'node': { - u'expiration': u'2013-09-14T00:56:59.316195568+02:00', - u'modifiedIndex': 6, - u'key': u'/testkey', - u'ttl': 19, - u'value': u'test' - } - } - self._mock_api(200,d) - res = self.client.get('/testkey') - res.value = 'ciao' - d['node']['value'] = 'ciao' - self._mock_api(200,d) + d = { + "action": "set", + "node": { + "expiration": "2013-09-14T00:56:59.316195568+02:00", + "modifiedIndex": 6, + "key": "/testkey", + "ttl": 19, + "value": "test", + }, + } + self._mock_api(200, d) + res = self.client.get("/testkey") + res.value = "ciao" + d["node"]["value"] = "ciao" + self._mock_api(200, d) newres = self.client.update(res) - self.assertEqual(newres.value, 'ciao') + self.assertEqual(newres.value, "ciao") def test_newkey(self): - """ Can set a new value """ + """Can set a new value""" d = { - u'action': u'set', - u'node': { - u'expiration': u'2013-09-14T00:56:59.316195568+02:00', - u'modifiedIndex': 183, - u'key': u'/testkey', - u'ttl': 19, - u'value': u'test' - } + "action": "set", + "node": { + "expiration": "2013-09-14T00:56:59.316195568+02:00", + "modifiedIndex": 183, + "key": "/testkey", + "ttl": 19, + "value": "test", + }, } self._mock_api(201, d) - res = self.client.write('/testkey', 'test') - d['node']['newKey'] = True + res = self.client.write("/testkey", "test") + d["node"]["newKey"] = True self.assertEqual(res, etcd.EtcdResult(**d)) def test_refresh(self): - """ Can refresh a new value """ + """Can refresh a new value""" d = { - u'action': u'update', - u'node': { - u'expiration': u'2016-05-31T08:27:54.660337Z', - u'modifiedIndex': 183, - u'key': u'/testkey', - u'ttl': 600, - u'value': u'test' - } + "action": "update", + "node": { + "expiration": "2016-05-31T08:27:54.660337Z", + "modifiedIndex": 183, + "key": "/testkey", + "ttl": 600, + "value": "test", + }, } self._mock_api(200, d) - res = self.client.refresh('/testkey', ttl=600) + res = self.client.refresh("/testkey", ttl=600) self.assertEqual(res, etcd.EtcdResult(**d)) def test_not_found_response(self): - """ Can handle server not found response """ - self._mock_api(404, 'Not found') - self.assertRaises(etcd.EtcdException, self.client.read, '/somebadkey') + """Can handle server not found response""" + self._mock_api(404, "Not found") + self.assertRaises(etcd.EtcdException, self.client.read, "/somebadkey") def test_compare_and_swap(self): - """ Can set compare-and-swap a value """ - d = {u'action': u'compareAndSwap', - u'node': { - u'expiration': u'2013-09-14T00:56:59.316195568+02:00', - u'modifiedIndex': 183, - u'key': u'/testkey', - u'ttl': 19, - u'value': u'test' - } - } + """Can set compare-and-swap a value""" + d = { + "action": "compareAndSwap", + "node": { + "expiration": "2013-09-14T00:56:59.316195568+02:00", + "modifiedIndex": 183, + "key": "/testkey", + "ttl": 19, + "value": "test", + }, + } self._mock_api(200, d) - res = self.client.write('/testkey', 'test', prevValue='test_old') + res = self.client.write("/testkey", "test", prevValue="test_old") self.assertEqual(res, etcd.EtcdResult(**d)) def test_compare_and_swap_failure(self): - """ Exception will be raised if prevValue != value in test_set """ - self._mock_exception(ValueError, 'Test Failed : [ 1!=3 ]') + """Exception will be raised if prevValue != value in test_set""" + self._mock_exception(ValueError, "Test Failed : [ 1!=3 ]") self.assertRaises( - ValueError, - self.client.write, - '/testKey', - 'test', - prevValue='oldbog' + ValueError, self.client.write, "/testKey", "test", prevValue="oldbog" ) def test_set_append(self): - """ Can append a new key """ + """Can append a new key""" d = { - u'action': u'create', - u'node': { - u'createdIndex': 190, - u'modifiedIndex': 190, - u'key': u'/testdir/190', - u'value': u'test' - } + "action": "create", + "node": { + "createdIndex": 190, + "modifiedIndex": 190, + "key": "/testdir/190", + "value": "test", + }, } self._mock_api(201, d) - res = self.client.write('/testdir', 'test') + res = self.client.write("/testdir", "test") self.assertEqual(res.createdIndex, 190) def test_set_dir_with_value(self): - """ Creating a directory with a value raises an error. """ - self.assertRaises(etcd.EtcdException, self.client.write, - '/bar', 'testvalye', dir=True) + """Creating a directory with a value raises an error.""" + self.assertRaises( + etcd.EtcdException, self.client.write, "/bar", "testvalye", dir=True + ) def test_delete(self): - """ Can delete a value """ + """Can delete a value""" d = { - u'action': u'delete', - u'node': { - u'key': u'/testkey', - "modifiedIndex": 3, - "createdIndex": 2 - } + "action": "delete", + "node": {"key": "/testkey", "modifiedIndex": 3, "createdIndex": 2}, } self._mock_api(200, d) - res = self.client.delete('/testKey') + res = self.client.delete("/testKey") self.assertEqual(res, etcd.EtcdResult(**d)) def test_pop(self): - """ Can pop a value """ + """Can pop a value""" d = { - u'action': u'delete', - u'node': { - u'key': u'/testkey', - u'modifiedIndex': 3, - u'createdIndex': 2 + "action": "delete", + "node": {"key": "/testkey", "modifiedIndex": 3, "createdIndex": 2}, + "prevNode": { + "newKey": False, + "createdIndex": None, + "modifiedIndex": 190, + "value": "test", + "expiration": None, + "key": "/testkey", + "ttl": None, + "dir": False, }, - u'prevNode': {u'newKey': False, u'createdIndex': None, - u'modifiedIndex': 190, u'value': u'test', u'expiration': None, - u'key': u'/testkey', u'ttl': None, u'dir': False} } self._mock_api(200, d) - res = self.client.pop(d['node']['key']) - self.assertEqual({attr: getattr(res, attr) for attr in dir(res) - if attr in etcd.EtcdResult._node_props}, d['prevNode']) - self.assertEqual(res.value, d['prevNode']['value']) + res = self.client.pop(d["node"]["key"]) + self.assertEqual( + { + attr: getattr(res, attr) + for attr in dir(res) + if attr in etcd.EtcdResult._node_props + }, + d["prevNode"], + ) + self.assertEqual(res.value, d["prevNode"]["value"]) def test_read(self): - """ Can get a value """ + """Can get a value""" d = { - u'action': u'get', - u'node': { - u'modifiedIndex': 190, - u'key': u'/testkey', - u'value': u'test' - } + "action": "get", + "node": {"modifiedIndex": 190, "key": "/testkey", "value": "test"}, } self._mock_api(200, d) - res = self.client.read('/testKey') + res = self.client.read("/testKey") self.assertEqual(res, etcd.EtcdResult(**d)) def test_get_dir(self): """Can get values in dirs""" d = { - u'action': u'get', - u'node': { - u'modifiedIndex': 190, - u'key': u'/testkey', - u'dir': True, - u'nodes': [ + "action": "get", + "node": { + "modifiedIndex": 190, + "key": "/testkey", + "dir": True, + "nodes": [ + {"key": "/testDir/testKey", "modifiedIndex": 150, "value": "test"}, { - u'key': u'/testDir/testKey', - u'modifiedIndex': 150, - u'value': 'test' + "key": "/testDir/testKey2", + "modifiedIndex": 190, + "value": "test2", }, - { - u'key': u'/testDir/testKey2', - u'modifiedIndex': 190, - u'value': 'test2' - } - ] - } + ], + }, } self._mock_api(200, d) - res = self.client.read('/testDir', recursive=True) + res = self.client.read("/testDir", recursive=True) self.assertEqual(res, etcd.EtcdResult(**d)) def test_not_in(self): - """ Can check if key is not in client """ - self._mock_exception(etcd.EtcdKeyNotFound, 'Key not Found : /testKey') - self.assertTrue('/testey' not in self.client) + """Can check if key is not in client""" + self._mock_exception(etcd.EtcdKeyNotFound, "Key not Found : /testKey") + self.assertTrue("/testey" not in self.client) def test_in(self): - """ Can check if key is not in client """ + """Can check if key is not in client""" d = { - u'action': u'get', - u'node': { - u'modifiedIndex': 190, - u'key': u'/testkey', - u'value': u'test' - } + "action": "get", + "node": {"modifiedIndex": 190, "key": "/testkey", "value": "test"}, } self._mock_api(200, d) - self.assertTrue('/testey' in self.client) + self.assertTrue("/testey" in self.client) def test_watch(self): - """ Can watch a key """ + """Can watch a key""" d = { - u'action': u'get', - u'node': { - u'modifiedIndex': 190, - u'key': u'/testkey', - u'value': u'test' - } + "action": "get", + "node": {"modifiedIndex": 190, "key": "/testkey", "value": "test"}, } self._mock_api(200, d) - res = self.client.read('/testkey', wait=True) + res = self.client.read("/testkey", wait=True) self.assertEqual(res, etcd.EtcdResult(**d)) def test_watch_index(self): - """ Can watch a key starting from the given Index """ + """Can watch a key starting from the given Index""" d = { - u'action': u'get', - u'node': { - u'modifiedIndex': 170, - u'key': u'/testkey', - u'value': u'testold' - } + "action": "get", + "node": {"modifiedIndex": 170, "key": "/testkey", "value": "testold"}, } self._mock_api(200, d) - res = self.client.read('/testkey', wait=True, waitIndex=True) + res = self.client.read("/testkey", wait=True, waitIndex=True) self.assertEqual(res, etcd.EtcdResult(**d)) class TestClientRequest(TestClientApiInterface): - def setUp(self): self.client = etcd.Client(expected_cluster_id="abcdef1234") def _mock_api(self, status, d, cluster_id=None): resp = self._prepare_response(status, d) resp.getheader.return_value = cluster_id or "abcdef1234" - self.client.http.request_encode_body = mock.MagicMock( - return_value=resp) + self.client.http.request_encode_body = mock.MagicMock(return_value=resp) self.client.http.request = mock.MagicMock(return_value=resp) - def _mock_error(self, error_code, msg, cause, method='PUT', fields=None, - cluster_id=None): + def _mock_error( + self, error_code, msg, cause, method="PUT", fields=None, cluster_id=None + ): resp = self._prepare_response( - 500, - {'errorCode': error_code, 'message': msg, 'cause': cause} + 500, {"errorCode": error_code, "message": msg, "cause": cause} ) resp.getheader.return_value = cluster_id or "abcdef1234" self.client.http.request_encode_body = mock.create_autospec( @@ -434,68 +413,63 @@ def _mock_error(self, error_code, msg, cause, method='PUT', fields=None, ) def test_compare_and_swap_failure(self): - """ Exception will be raised if prevValue != value in test_set """ - self._mock_error(200, 'Test Failed', - '[ 1!=3 ]', fields={'prevValue': 'oldbog'}) + """Exception will be raised if prevValue != value in test_set""" + self._mock_error(200, "Test Failed", "[ 1!=3 ]", fields={"prevValue": "oldbog"}) self.assertRaises( - ValueError, - self.client.write, - '/testKey', - 'test', - prevValue='oldbog' + ValueError, self.client.write, "/testKey", "test", prevValue="oldbog" ) def test_watch_timeout(self): - """ Exception will be raised if prevValue != value in test_set """ + """Exception will be raised if prevValue != value in test_set""" self.client.http.request = mock.create_autospec( self.client.http.request, - side_effect=urllib3.exceptions.ReadTimeoutError(self.client.http, - "foo", - "Read timed out") + side_effect=urllib3.exceptions.ReadTimeoutError( + self.client.http, "foo", "Read timed out" + ), ) self.assertRaises( etcd.EtcdWatchTimedOut, self.client.watch, - '/testKey', + "/testKey", ) def test_path_without_trailing_slash(self): - """ Exception will be raised if a path without a trailing slash is used """ - self.assertRaises(ValueError, self.client.api_execute, - 'testpath/bar', self.client._MPUT) + """Exception will be raised if a path without a trailing slash is used""" + self.assertRaises( + ValueError, self.client.api_execute, "testpath/bar", self.client._MPUT + ) def test_api_method_not_supported(self): - """ Exception will be raised if an unsupported HTTP method is used """ - self.assertRaises(etcd.EtcdException, - self.client.api_execute, '/testpath/bar', 'TRACE') + """Exception will be raised if an unsupported HTTP method is used""" + self.assertRaises( + etcd.EtcdException, self.client.api_execute, "/testpath/bar", "TRACE" + ) def test_read_cluster_id_changed(self): - """ Read timeout set to the default """ + """Read timeout set to the default""" d = { - u'action': u'set', - u'node': { - u'expiration': u'2013-09-14T00:56:59.316195568+02:00', - u'modifiedIndex': 6, - u'key': u'/testkey', - u'ttl': 19, - u'value': u'test', - } + "action": "set", + "node": { + "expiration": "2013-09-14T00:56:59.316195568+02:00", + "modifiedIndex": 6, + "key": "/testkey", + "ttl": 19, + "value": "test", + }, } self._mock_api(200, d, cluster_id="notabcd1234") - self.assertRaises(etcd.EtcdClusterIdChanged, - self.client.read, '/testkey') + self.assertRaises(etcd.EtcdClusterIdChanged, self.client.read, "/testkey") self.client.read("/testkey") def test_read_connection_error(self): self.client.http.request = mock.create_autospec( - self.client.http.request, - side_effect=socket.error() + self.client.http.request, side_effect=socket.error() ) - self.assertRaises(etcd.EtcdConnectionFailed, - self.client.read, '/something') + self.assertRaises(etcd.EtcdConnectionFailed, self.client.read, "/something") # Direct GET request - self.assertRaises(etcd.EtcdConnectionFailed, - self.client.api_execute, '/a', 'GET') + self.assertRaises( + etcd.EtcdConnectionFailed, self.client.api_execute, "/a", "GET" + ) def test_not_in(self): pass @@ -504,16 +478,16 @@ def test_in(self): pass def test_update_fails(self): - """ Non-atomic updates fail """ + """Non-atomic updates fail""" d = { - u'action': u'set', - u'node': { - u'expiration': u'2013-09-14T00:56:59.316195568+02:00', - u'modifiedIndex': 6, - u'key': u'/testkey', - u'ttl': 19, - u'value': u'test' - } + "action": "set", + "node": { + "expiration": "2013-09-14T00:56:59.316195568+02:00", + "modifiedIndex": 6, + "key": "/testkey", + "ttl": 19, + "value": "test", + }, } res = etcd.EtcdResult(**d) @@ -521,7 +495,8 @@ def test_update_fails(self): "errorCode": 101, "message": "Compare failed", "cause": "[ != bar] [7 != 6]", - "index": 6} + "index": 6, + } self._mock_api(412, error) - res.value = 'bar' + res.value = "bar" self.assertRaises(ValueError, self.client.update, res) diff --git a/src/etcd/tests/unit/test_result.py b/src/etcd/tests/unit/test_result.py index cb1414b1..372f10c3 100644 --- a/src/etcd/tests/unit/test_result.py +++ b/src/etcd/tests/unit/test_result.py @@ -8,22 +8,24 @@ except ImportError: from unittest import mock -class TestEtcdResult(unittest.TestCase): +class TestEtcdResult(unittest.TestCase): def test_get_subtree_1_level(self): """ Test get_subtree() for a read with tree 1 level deep. """ - response = {"node": { - 'key': "/test", - 'value': "hello", - 'expiration': None, - 'ttl': None, - 'modifiedIndex': 5, - 'createdIndex': 1, - 'newKey': False, - 'dir': False, - }} + response = { + "node": { + "key": "/test", + "value": "hello", + "expiration": None, + "ttl": None, + "modifiedIndex": 5, + "createdIndex": 1, + "newKey": False, + "dir": False, + } + } result = etcd.EtcdResult(**response) self.assertEqual(result.key, response["node"]["key"]) self.assertEqual(result.value, response["node"]["value"]) @@ -39,35 +41,37 @@ def test_get_subtree_2_level(self): Test get_subtree() for a read with tree 2 levels deep. """ leaf0 = { - 'key': "/test/leaf0", - 'value': "hello1", - 'expiration': None, - 'ttl': None, - 'modifiedIndex': 5, - 'createdIndex': 1, - 'newKey': False, - 'dir': False, + "key": "/test/leaf0", + "value": "hello1", + "expiration": None, + "ttl": None, + "modifiedIndex": 5, + "createdIndex": 1, + "newKey": False, + "dir": False, } leaf1 = { - 'key': "/test/leaf1", - 'value': "hello2", - 'expiration': None, - 'ttl': None, - 'modifiedIndex': 6, - 'createdIndex': 2, - 'newKey': False, - 'dir': False, + "key": "/test/leaf1", + "value": "hello2", + "expiration": None, + "ttl": None, + "modifiedIndex": 6, + "createdIndex": 2, + "newKey": False, + "dir": False, + } + testnode = { + "node": { + "key": "/test/", + "expiration": None, + "ttl": None, + "modifiedIndex": 6, + "createdIndex": 2, + "newKey": False, + "dir": True, + "nodes": [leaf0, leaf1], + } } - testnode = {"node": { - 'key': "/test/", - 'expiration': None, - 'ttl': None, - 'modifiedIndex': 6, - 'createdIndex': 2, - 'newKey': False, - 'dir': True, - 'nodes': [leaf0, leaf1] - }} result = etcd.EtcdResult(**testnode) self.assertEqual(result.key, "/test/") self.assertTrue(result.dir) @@ -90,36 +94,24 @@ def test_get_subtree_3_level(self): Test get_subtree() for a read with tree 3 levels deep. """ leaf0 = { - 'key': "/test/mid0/leaf0", - 'value': "hello1", + "key": "/test/mid0/leaf0", + "value": "hello1", } leaf1 = { - 'key': "/test/mid0/leaf1", - 'value': "hello2", + "key": "/test/mid0/leaf1", + "value": "hello2", } leaf2 = { - 'key': "/test/mid1/leaf2", - 'value': "hello1", + "key": "/test/mid1/leaf2", + "value": "hello1", } leaf3 = { - 'key': "/test/mid1/leaf3", - 'value': "hello2", - } - mid0 = { - 'key': "/test/mid0/", - 'dir': True, - 'nodes': [leaf0, leaf1] - } - mid1 = { - 'key': "/test/mid1/", - 'dir': True, - 'nodes': [leaf2, leaf3] + "key": "/test/mid1/leaf3", + "value": "hello2", } - testnode = {"node": { - 'key': "/test/", - 'dir': True, - 'nodes': [mid0, mid1] - }} + mid0 = {"key": "/test/mid0/", "dir": True, "nodes": [leaf0, leaf1]} + mid1 = {"key": "/test/mid1/", "dir": True, "nodes": [leaf2, leaf3]} + testnode = {"node": {"key": "/test/", "dir": True, "nodes": [mid0, mid1]}} result = etcd.EtcdResult(**testnode) self.assertEqual(result.key, "/test/") self.assertTrue(result.dir) From b8f3ad0d19626b21b08892ac910fc2e72e0001b6 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Mon, 30 Oct 2023 08:19:41 +0100 Subject: [PATCH 093/101] Convert unit tests to work with pytest --- src/etcd/tests/unit/test_lock.py | 16 ++----- src/etcd/tests/unit/test_old_request.py | 26 +++------- src/etcd/tests/unit/test_request.py | 63 +++++++------------------ 3 files changed, 26 insertions(+), 79 deletions(-) diff --git a/src/etcd/tests/unit/test_lock.py b/src/etcd/tests/unit/test_lock.py index 792690b1..0d70075b 100644 --- a/src/etcd/tests/unit/test_lock.py +++ b/src/etcd/tests/unit/test_lock.py @@ -99,9 +99,7 @@ def test_acquired(self): """ self.locker._sequence = "4" retval = ("/_locks/test_lock/4", None) - self.locker._get_locker = mock.MagicMock( - spec=self.locker._get_locker, return_value=retval - ) + self.locker._get_locker = mock.MagicMock(return_value=retval) self.assertTrue(self.locker._acquired()) self.assertTrue(self.locker.is_taken) retval = ("/_locks/test_lock/1", "/_locks/test_lock/4") @@ -125,9 +123,7 @@ def test_acquired(self): def side_effect(): return returns.pop() - self.locker._get_locker = mock.MagicMock( - spec=self.locker._get_locker, side_effect=side_effect - ) + self.locker._get_locker = mock.MagicMock(side_effect=side_effect) self.assertTrue(self.locker._acquired()) def test_acquired_no_timeout(self): @@ -136,9 +132,7 @@ def test_acquired_no_timeout(self): ("/_locks/test_lock/4", None), ( "/_locks/test_lock/1", - etcd.EtcdResult( - node={"key": "/_locks/test_lock/4", "modifiedIndex": 1} - ), + etcd.EtcdResult(node={"key": "/_locks/test_lock/4", "modifiedIndex": 1}), ), ] @@ -155,9 +149,7 @@ def side_effect(): } self._mock_api(200, d) - self.locker._get_locker = mock.create_autospec( - self.locker._get_locker, side_effect=side_effect - ) + self.locker._get_locker = mock.create_autospec(self.locker._get_locker, side_effect=side_effect) self.assertTrue(self.locker._acquired()) def test_lock_key(self): diff --git a/src/etcd/tests/unit/test_old_request.py b/src/etcd/tests/unit/test_old_request.py index 105a90ae..8f78cb33 100644 --- a/src/etcd/tests/unit/test_old_request.py +++ b/src/etcd/tests/unit/test_old_request.py @@ -98,18 +98,14 @@ def test_test_and_test_failure(self): client = etcd.Client() client.api_execute = mock.Mock( - side_effect=ValueError( - "The given PrevValue is not equal" - " to the value of the key : TestAndSet: 1!=3" - ) + side_effect=ValueError("The given PrevValue is not equal" " to the value of the key : TestAndSet: 1!=3") ) try: result = client.test_and_set("/testkey", "newvalue", "test", ttl=19) except ValueError as e: # from ipdb import set_trace; set_trace() self.assertEqual( - "The given PrevValue is not equal" - " to the value of the key : TestAndSet: 1!=3", + "The given PrevValue is not equal" " to the value of the key : TestAndSet: 1!=3", str(e), ) @@ -149,11 +145,7 @@ def test_get(self): client.api_execute = mock.Mock( return_value=FakeHTTPResponse( 200, - '{"action":"GET",' - '"node": {' - '"key":"/testkey",' - '"value":"test",' - '"modifiedIndex":190}}', + '{"action":"GET",' '"node": {' '"key":"/testkey",' '"value":"test",' '"modifiedIndex":190}}', ) ) @@ -189,11 +181,7 @@ def test_in(self): client.api_execute = mock.Mock( return_value=FakeHTTPResponse( 200, - '{"action":"GET",' - '"node": {' - '"key":"/testkey",' - '"value":"test",' - '"modifiedIndex":190}}', + '{"action":"GET",' '"node": {' '"key":"/testkey",' '"value":"test",' '"modifiedIndex":190}}', ) ) result = "/testkey" in client @@ -306,7 +294,7 @@ def test_eternal_watch(self): ) for result in range(1, 5): result = next(client.eternal_watch("/testkey", index=180)) - yield self.check_watch, result + self.check_watch(result) class TestClientApiExecutor(unittest.TestCase): @@ -408,6 +396,4 @@ def test_get_error_invalid(self): client = etcd.Client() response = FakeHTTPResponse(status=400, data="{){){)*garbage*") client.http.request = mock.Mock(return_value=response) - self.assertRaises( - etcd.EtcdException, client.api_execute, "/v2/keys/testkey", client._MGET - ) + self.assertRaises(etcd.EtcdException, client.api_execute, "/v2/keys/testkey", client._MGET) diff --git a/src/etcd/tests/unit/test_request.py b/src/etcd/tests/unit/test_request.py index 6270a3c1..ff355b03 100644 --- a/src/etcd/tests/unit/test_request.py +++ b/src/etcd/tests/unit/test_request.py @@ -68,8 +68,7 @@ class TestClientApiInterface(TestClientApiBase): If a test should be run only in this class, please override the method there. """ - @mock.patch("urllib3.request.RequestMethods.request") - def test_machines(self, mocker): + def test_machines(self): """Can request machines""" data = [ "http://127.0.0.1:4001", @@ -77,7 +76,7 @@ def test_machines(self, mocker): "http://127.0.0.1:4003", ] d = ",".join(data) - mocker.return_value = self._prepare_response(200, d) + self.client.http.request = mock.MagicMock(return_value=self._prepare_response(200, d)) self.assertEqual(data, self.client.machines) @mock.patch("etcd.Client.machines", new_callable=mock.PropertyMock) @@ -121,9 +120,7 @@ def test_members(self): ] } self._mock_api(200, data) - self.assertEqual( - self.client.members["ce2a822cea30bfca"]["id"], "ce2a822cea30bfca" - ) + self.assertEqual(self.client.members["ce2a822cea30bfca"]["id"], "ce2a822cea30bfca") def test_self_stats(self): """Request for stats""" @@ -156,9 +153,7 @@ def test_leader(self, mocker): """Can request the leader""" members = {"ce2a822cea30bfca": {"id": "ce2a822cea30bfca", "name": "default"}} mocker.return_value = members - self._mock_api( - 200, {"leaderInfo": {"leader": "ce2a822cea30bfca", "followers": {}}} - ) + self._mock_api(200, {"leaderInfo": {"leader": "ce2a822cea30bfca", "followers": {}}}) self.assertEqual(self.client.leader, members["ce2a822cea30bfca"]) def test_set_plain(self): @@ -257,9 +252,7 @@ def test_compare_and_swap(self): def test_compare_and_swap_failure(self): """Exception will be raised if prevValue != value in test_set""" self._mock_exception(ValueError, "Test Failed : [ 1!=3 ]") - self.assertRaises( - ValueError, self.client.write, "/testKey", "test", prevValue="oldbog" - ) + self.assertRaises(ValueError, self.client.write, "/testKey", "test", prevValue="oldbog") def test_set_append(self): """Can append a new key""" @@ -278,9 +271,7 @@ def test_set_append(self): def test_set_dir_with_value(self): """Creating a directory with a value raises an error.""" - self.assertRaises( - etcd.EtcdException, self.client.write, "/bar", "testvalye", dir=True - ) + self.assertRaises(etcd.EtcdException, self.client.write, "/bar", "testvalye", dir=True) def test_delete(self): """Can delete a value""" @@ -312,11 +303,7 @@ def test_pop(self): self._mock_api(200, d) res = self.client.pop(d["node"]["key"]) self.assertEqual( - { - attr: getattr(res, attr) - for attr in dir(res) - if attr in etcd.EtcdResult._node_props - }, + {attr: getattr(res, attr) for attr in dir(res) if attr in etcd.EtcdResult._node_props}, d["prevNode"], ) self.assertEqual(res.value, d["prevNode"]["value"]) @@ -398,34 +385,24 @@ def _mock_api(self, status, d, cluster_id=None): self.client.http.request_encode_body = mock.MagicMock(return_value=resp) self.client.http.request = mock.MagicMock(return_value=resp) - def _mock_error( - self, error_code, msg, cause, method="PUT", fields=None, cluster_id=None - ): - resp = self._prepare_response( - 500, {"errorCode": error_code, "message": msg, "cause": cause} - ) + def _mock_error(self, error_code, msg, cause, method="PUT", fields=None, cluster_id=None): + resp = self._prepare_response(500, {"errorCode": error_code, "message": msg, "cause": cause}) resp.getheader.return_value = cluster_id or "abcdef1234" self.client.http.request_encode_body = mock.create_autospec( self.client.http.request_encode_body, return_value=resp ) - self.client.http.request = mock.create_autospec( - self.client.http.request, return_value=resp - ) + self.client.http.request = mock.create_autospec(self.client.http.request, return_value=resp) def test_compare_and_swap_failure(self): """Exception will be raised if prevValue != value in test_set""" self._mock_error(200, "Test Failed", "[ 1!=3 ]", fields={"prevValue": "oldbog"}) - self.assertRaises( - ValueError, self.client.write, "/testKey", "test", prevValue="oldbog" - ) + self.assertRaises(ValueError, self.client.write, "/testKey", "test", prevValue="oldbog") def test_watch_timeout(self): """Exception will be raised if prevValue != value in test_set""" self.client.http.request = mock.create_autospec( self.client.http.request, - side_effect=urllib3.exceptions.ReadTimeoutError( - self.client.http, "foo", "Read timed out" - ), + side_effect=urllib3.exceptions.ReadTimeoutError(self.client.http, "foo", "Read timed out"), ) self.assertRaises( etcd.EtcdWatchTimedOut, @@ -435,15 +412,11 @@ def test_watch_timeout(self): def test_path_without_trailing_slash(self): """Exception will be raised if a path without a trailing slash is used""" - self.assertRaises( - ValueError, self.client.api_execute, "testpath/bar", self.client._MPUT - ) + self.assertRaises(ValueError, self.client.api_execute, "testpath/bar", self.client._MPUT) def test_api_method_not_supported(self): """Exception will be raised if an unsupported HTTP method is used""" - self.assertRaises( - etcd.EtcdException, self.client.api_execute, "/testpath/bar", "TRACE" - ) + self.assertRaises(etcd.EtcdException, self.client.api_execute, "/testpath/bar", "TRACE") def test_read_cluster_id_changed(self): """Read timeout set to the default""" @@ -462,14 +435,10 @@ def test_read_cluster_id_changed(self): self.client.read("/testkey") def test_read_connection_error(self): - self.client.http.request = mock.create_autospec( - self.client.http.request, side_effect=socket.error() - ) + self.client.http.request = mock.create_autospec(self.client.http.request, side_effect=socket.error()) self.assertRaises(etcd.EtcdConnectionFailed, self.client.read, "/something") # Direct GET request - self.assertRaises( - etcd.EtcdConnectionFailed, self.client.api_execute, "/a", "GET" - ) + self.assertRaises(etcd.EtcdConnectionFailed, self.client.api_execute, "/a", "GET") def test_not_in(self): pass From 531004dedd0569e7c3ffbfee96a49396bef7721b Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Mon, 30 Oct 2023 14:59:45 +0100 Subject: [PATCH 094/101] Convert integration tests to work with pytest --- .travis.yml | 6 +-- setup.py | 2 +- src/etcd/tests/integration/helpers.py | 61 +++++++++++++++++++---- src/etcd/tests/integration/test_simple.py | 19 ++----- src/etcd/tests/integration/test_ssl.py | 26 +++++----- 5 files changed, 74 insertions(+), 40 deletions(-) diff --git a/.travis.yml b/.travis.yml index b3db1220..f4d3a8f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,16 @@ language: python python: - - "2.7" - - "3.5" + - "3.7" before_install: - - ./download_etcd.sh 2.3.7 + - ./download_etcd.sh 3.4.0 - pip install --upgrade setuptools # command to install dependencies install: - pip install coveralls - pip install coverage + - pip install pytest - python bootstrap.py - bin/buildout diff --git a/setup.py b/setup.py index 8f6e9484..8bbc4d33 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ install_requires = ["urllib3>=1.7.1", "dnspython>=1.13.0"] -test_requires = ["mock", "nose", "pyOpenSSL>=0.14"] +test_requires = ["mock", "pytest", "pyOpenSSL>=0.14"] setup( name="python-etcd", diff --git a/src/etcd/tests/integration/helpers.py b/src/etcd/tests/integration/helpers.py index 1a0933df..99bc668e 100644 --- a/src/etcd/tests/integration/helpers.py +++ b/src/etcd/tests/integration/helpers.py @@ -1,3 +1,4 @@ +import re import shutil import subprocess import tempfile @@ -28,6 +29,24 @@ def __init__( self.schema = "http://" if tls: self.schema = "https://" + self.compat_args = self.check_compat_args() + + def check_compat_args(self): + version_re = re.compile(r"^etcd version:\s+(\d)\.(\d)", re.I) + version_data = subprocess.check_output( + [self.proc_name, "--version"] + ).decode("utf-8") + match = version_re.match(version_data) + if match is not None: + etcd_version = (int(match.group(1)), int(match.group(2))) + else: + etcd_version = (0, 0) + if etcd_version[0] < 3 or ( + etcd_version[0] == 3 and etcd_version[1] < 4 + ): + return [] + else: + return ["--enable-v2=true"] def run(self, number=1, proc_args=[]): if number > 1: @@ -40,7 +59,12 @@ def run(self, number=1, proc_args=[]): ] ) proc_args.extend( - ["-initial-cluster", initial_cluster, "-initial-cluster-state", "new"] + [ + "-initial-cluster", + initial_cluster, + "-initial-cluster-state", + "new", + ] ) else: proc_args.extend( @@ -70,7 +94,10 @@ def add_one(self, slot, proc_args=None): log.debug("Created directory %s" % directory) client = "%s127.0.0.1:%d" % (self.schema, self.port_range_start + slot) - peer = "%s127.0.0.1:%d" % ("http://", self.internal_port_range_start + slot) + peer = "%s127.0.0.1:%d" % ( + "http://", + self.internal_port_range_start + slot, + ) daemon_args = [ self.proc_name, "-data-dir", @@ -86,7 +113,7 @@ def add_one(self, slot, proc_args=None): "-listen-client-urls", client, ] - + daemon_args.extend(self.compat_args) if proc_args: daemon_args.extend(proc_args) @@ -134,7 +161,9 @@ def create_test_ca_certificate(cls, cert_path, key_path, cn=None): cert.add_extensions( [ crypto.X509Extension( - "basicConstraints".encode("ascii"), False, "CA:TRUE".encode("ascii") + "basicConstraints".encode("ascii"), + False, + "CA:TRUE".encode("ascii"), ), crypto.X509Extension( "keyUsage".encode("ascii"), @@ -164,10 +193,16 @@ def create_test_ca_certificate(cls, cert_path, key_path, cn=None): cert.sign(k, "sha1") with open(cert_path, "w") as f: - f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode("utf-8")) + f.write( + crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode( + "utf-8" + ) + ) with open(key_path, "w") as f: - f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, k).decode("utf-8")) + f.write( + crypto.dump_privatekey(crypto.FILETYPE_PEM, k).decode("utf-8") + ) return cert, k @@ -196,7 +231,9 @@ def create_test_certificate(cls, ca, ca_key, cert_path, key_path, cn=None): crypto.X509Extension( "keyUsage".encode("ascii"), False, - "nonRepudiation,digitalSignature,keyEncipherment".encode("ascii"), + "nonRepudiation,digitalSignature,keyEncipherment".encode( + "ascii" + ), ), crypto.X509Extension( "extendedKeyUsage".encode("ascii"), @@ -220,7 +257,13 @@ def create_test_certificate(cls, ca, ca_key, cert_path, key_path, cn=None): cert.sign(ca_key, "sha1") with open(cert_path, "w") as f: - f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode("utf-8")) + f.write( + crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode( + "utf-8" + ) + ) with open(key_path, "w") as f: - f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, k).decode("utf-8")) + f.write( + crypto.dump_privatekey(crypto.FILETYPE_PEM, k).decode("utf-8") + ) diff --git a/src/etcd/tests/integration/test_simple.py b/src/etcd/tests/integration/test_simple.py index 3faf80dd..6421b077 100644 --- a/src/etcd/tests/integration/test_simple.py +++ b/src/etcd/tests/integration/test_simple.py @@ -11,9 +11,6 @@ import etcd from . import helpers -from nose.tools import nottest - - log = logging.getLogger() @@ -149,9 +146,7 @@ def test_test_and_set(self): set_result = self.client.set("/test-key", "old-test-value") - set_result = self.client.test_and_set( - "/test-key", "test-value", "old-test-value" - ) + set_result = self.client.test_and_set("/test-key", "test-value", "old-test-value") self.assertRaises( ValueError, @@ -210,9 +205,7 @@ def test_reconnect_with_several_hosts_passed(self): """INTEGRATION: receive several hosts at connection setup.""" self.processHelper.stop() self.processHelper.run(number=3) - self.client = etcd.Client( - host=(("127.0.0.1", 6004), ("127.0.0.1", 6001)), allow_reconnect=True - ) + self.client = etcd.Client(host=(("127.0.0.1", 6004), ("127.0.0.1", 6001)), allow_reconnect=True) set_result = self.client.set("/test_set", "test-key1") get_result = self.client.get("/test_set") @@ -314,9 +307,7 @@ def watch_value(key, index, queue): ), ) - watcher = multiprocessing.Process( - target=watch_value, args=("/test-key", original_index, queue) - ) + watcher = multiprocessing.Process(target=watch_value, args=("/test-key", original_index, queue)) watcher.start() time.sleep(0.5) @@ -396,9 +387,7 @@ def watch_value(key, index, queue): ), ) - watcher = multiprocessing.Process( - target=watch_value, args=("/test-key", original_index, queue) - ) + watcher = multiprocessing.Process(target=watch_value, args=("/test-key", original_index, queue)) watcher.start() time.sleep(0.5) diff --git a/src/etcd/tests/integration/test_ssl.py b/src/etcd/tests/integration/test_ssl.py index 2819fc97..1328bacb 100644 --- a/src/etcd/tests/integration/test_ssl.py +++ b/src/etcd/tests/integration/test_ssl.py @@ -6,12 +6,12 @@ import multiprocessing import tempfile +import pytest import urllib3 import etcd from . import helpers from . import test_simple -from nose.tools import nottest log = logging.getLogger() @@ -67,11 +67,12 @@ def test_get_set_unauthenticated(self): # Since python 3 raises a MaxRetryError here, this gets caught in # different code blocks in python 2 and python 3, thus messages are # different. Python 3 does the right thing(TM), for the record - self.assertRaises(etcd.EtcdException, client.set, "/test_set", "test-key") + self.assertRaises( + etcd.EtcdException, client.set, "/test_set", "test-key" + ) self.assertRaises(etcd.EtcdException, client.get, "/test_set") - @nottest def test_get_set_unauthenticated_missing_ca(self): """INTEGRATION: try unauthenticated w/out validation (https->https)""" # This doesn't work for now and will need further inspection @@ -81,7 +82,9 @@ def test_get_set_unauthenticated_missing_ca(self): def test_get_set_unauthenticated_with_ca(self): """INTEGRATION: try unauthenticated with validation (https->https)""" - client = etcd.Client(protocol="https", port=6001, ca_cert=self.ca2_cert_path) + client = etcd.Client( + protocol="https", port=6001, ca_cert=self.ca2_cert_path + ) self.assertRaises( etcd.EtcdConnectionFailed, client.set, "/test-set", "test-key" @@ -91,7 +94,7 @@ def test_get_set_unauthenticated_with_ca(self): def test_get_set_authenticated(self): """INTEGRATION: set/get a new value authenticated""" - client = etcd.Client(port=6001, protocol="https", ca_cert=self.ca_cert_path) + client = etcd.Client(port=6001, protocol="https") set_result = client.set("/test_set", "test-key") get_result = client.get("/test_set") @@ -145,7 +148,8 @@ def setUpClass(cls): proc_args=[ "-cert-file=%s" % server_cert_path, "-key-file=%s" % server_key_path, - "-ca-file=%s" % cls.ca_cert_path, + "-trusted-ca-file", + cls.ca_cert_path, ], ) @@ -155,21 +159,19 @@ def test_get_set_unauthenticated(self): client = etcd.Client(port=6001) # See above for the reason of this change - self.assertRaises(etcd.EtcdException, client.set, "/test_set", "test-key") + self.assertRaises( + etcd.EtcdException, client.set, "/test_set", "test-key" + ) self.assertRaises(etcd.EtcdException, client.get, "/test_set") - @nottest + @pytest.mark.skip(reason="We need non SHA1-signed certs and I won't implement it now.") def test_get_set_authenticated(self): """INTEGRATION: connecting to server with mutual auth""" - # This gives an unexplicable ssl error, as connecting to the same - # Etcd cluster where this fails with the exact same code this - # doesn't fail client = etcd.Client( port=6001, protocol="https", cert=self.client_all_cert, - ca_cert=self.ca_cert_path, ) set_result = client.set("/test_set", "test-key") From e76a2e18c3c592cab16367e68c3ec4074dbb3efc Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Mon, 30 Oct 2023 15:18:06 +0100 Subject: [PATCH 095/101] fix formatting for black for tests --- src/etcd/tests/integration/helpers.py | 40 +++++------------------ src/etcd/tests/integration/test_simple.py | 12 +++++-- src/etcd/tests/integration/test_ssl.py | 20 +++--------- src/etcd/tests/test_auth.py | 4 +-- src/etcd/tests/unit/test_client.py | 12 ++----- src/etcd/tests/unit/test_lock.py | 4 ++- src/etcd/tests/unit/test_old_request.py | 16 +++++++-- src/etcd/tests/unit/test_request.py | 12 +++++-- 8 files changed, 52 insertions(+), 68 deletions(-) diff --git a/src/etcd/tests/integration/helpers.py b/src/etcd/tests/integration/helpers.py index 99bc668e..131e81d0 100644 --- a/src/etcd/tests/integration/helpers.py +++ b/src/etcd/tests/integration/helpers.py @@ -33,17 +33,13 @@ def __init__( def check_compat_args(self): version_re = re.compile(r"^etcd version:\s+(\d)\.(\d)", re.I) - version_data = subprocess.check_output( - [self.proc_name, "--version"] - ).decode("utf-8") + version_data = subprocess.check_output([self.proc_name, "--version"]).decode("utf-8") match = version_re.match(version_data) if match is not None: etcd_version = (int(match.group(1)), int(match.group(2))) else: etcd_version = (0, 0) - if etcd_version[0] < 3 or ( - etcd_version[0] == 3 and etcd_version[1] < 4 - ): + if etcd_version[0] < 3 or (etcd_version[0] == 3 and etcd_version[1] < 4): return [] else: return ["--enable-v2=true"] @@ -70,9 +66,7 @@ def run(self, number=1, proc_args=[]): proc_args.extend( [ "-initial-cluster", - "test-node-0=http://127.0.0.1:{}".format( - self.internal_port_range_start - ), + "test-node-0=http://127.0.0.1:{}".format(self.internal_port_range_start), "-initial-cluster-state", "new", ] @@ -88,9 +82,7 @@ def stop(self): def add_one(self, slot, proc_args=None): log = logging.getLogger() - directory = tempfile.mkdtemp( - dir=self.base_directory, prefix="python-etcd.%d-" % slot - ) + directory = tempfile.mkdtemp(dir=self.base_directory, prefix="python-etcd.%d-" % slot) log.debug("Created directory %s" % directory) client = "%s127.0.0.1:%d" % (self.schema, self.port_range_start + slot) @@ -193,16 +185,10 @@ def create_test_ca_certificate(cls, cert_path, key_path, cn=None): cert.sign(k, "sha1") with open(cert_path, "w") as f: - f.write( - crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode( - "utf-8" - ) - ) + f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode("utf-8")) with open(key_path, "w") as f: - f.write( - crypto.dump_privatekey(crypto.FILETYPE_PEM, k).decode("utf-8") - ) + f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, k).decode("utf-8")) return cert, k @@ -231,9 +217,7 @@ def create_test_certificate(cls, ca, ca_key, cert_path, key_path, cn=None): crypto.X509Extension( "keyUsage".encode("ascii"), False, - "nonRepudiation,digitalSignature,keyEncipherment".encode( - "ascii" - ), + "nonRepudiation,digitalSignature,keyEncipherment".encode("ascii"), ), crypto.X509Extension( "extendedKeyUsage".encode("ascii"), @@ -257,13 +241,7 @@ def create_test_certificate(cls, ca, ca_key, cert_path, key_path, cn=None): cert.sign(ca_key, "sha1") with open(cert_path, "w") as f: - f.write( - crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode( - "utf-8" - ) - ) + f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode("utf-8")) with open(key_path, "w") as f: - f.write( - crypto.dump_privatekey(crypto.FILETYPE_PEM, k).decode("utf-8") - ) + f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, k).decode("utf-8")) diff --git a/src/etcd/tests/integration/test_simple.py b/src/etcd/tests/integration/test_simple.py index 6421b077..631a14b6 100644 --- a/src/etcd/tests/integration/test_simple.py +++ b/src/etcd/tests/integration/test_simple.py @@ -205,7 +205,9 @@ def test_reconnect_with_several_hosts_passed(self): """INTEGRATION: receive several hosts at connection setup.""" self.processHelper.stop() self.processHelper.run(number=3) - self.client = etcd.Client(host=(("127.0.0.1", 6004), ("127.0.0.1", 6001)), allow_reconnect=True) + self.client = etcd.Client( + host=(("127.0.0.1", 6004), ("127.0.0.1", 6001)), allow_reconnect=True + ) set_result = self.client.set("/test_set", "test-key1") get_result = self.client.get("/test_set") @@ -307,7 +309,9 @@ def watch_value(key, index, queue): ), ) - watcher = multiprocessing.Process(target=watch_value, args=("/test-key", original_index, queue)) + watcher = multiprocessing.Process( + target=watch_value, args=("/test-key", original_index, queue) + ) watcher.start() time.sleep(0.5) @@ -387,7 +391,9 @@ def watch_value(key, index, queue): ), ) - watcher = multiprocessing.Process(target=watch_value, args=("/test-key", original_index, queue)) + watcher = multiprocessing.Process( + target=watch_value, args=("/test-key", original_index, queue) + ) watcher.start() time.sleep(0.5) diff --git a/src/etcd/tests/integration/test_ssl.py b/src/etcd/tests/integration/test_ssl.py index 1328bacb..ce1b0c97 100644 --- a/src/etcd/tests/integration/test_ssl.py +++ b/src/etcd/tests/integration/test_ssl.py @@ -67,9 +67,7 @@ def test_get_set_unauthenticated(self): # Since python 3 raises a MaxRetryError here, this gets caught in # different code blocks in python 2 and python 3, thus messages are # different. Python 3 does the right thing(TM), for the record - self.assertRaises( - etcd.EtcdException, client.set, "/test_set", "test-key" - ) + self.assertRaises(etcd.EtcdException, client.set, "/test_set", "test-key") self.assertRaises(etcd.EtcdException, client.get, "/test_set") @@ -82,13 +80,9 @@ def test_get_set_unauthenticated_missing_ca(self): def test_get_set_unauthenticated_with_ca(self): """INTEGRATION: try unauthenticated with validation (https->https)""" - client = etcd.Client( - protocol="https", port=6001, ca_cert=self.ca2_cert_path - ) + client = etcd.Client(protocol="https", port=6001, ca_cert=self.ca2_cert_path) - self.assertRaises( - etcd.EtcdConnectionFailed, client.set, "/test-set", "test-key" - ) + self.assertRaises(etcd.EtcdConnectionFailed, client.set, "/test-set", "test-key") self.assertRaises(etcd.EtcdConnectionFailed, client.get, "/test-set") def test_get_set_authenticated(self): @@ -117,9 +111,7 @@ def setUpClass(cls): cls.client_all_cert = os.path.join(cls.directory, "client-all.crt") - ca, ca_key = helpers.TestingCA.create_test_ca_certificate( - cls.ca_cert_path, ca_key_path - ) + ca, ca_key = helpers.TestingCA.create_test_ca_certificate(cls.ca_cert_path, ca_key_path) helpers.TestingCA.create_test_certificate( ca, ca_key, server_cert_path, server_key_path, "127.0.0.1" @@ -159,9 +151,7 @@ def test_get_set_unauthenticated(self): client = etcd.Client(port=6001) # See above for the reason of this change - self.assertRaises( - etcd.EtcdException, client.set, "/test_set", "test-key" - ) + self.assertRaises(etcd.EtcdException, client.set, "/test_set", "test-key") self.assertRaises(etcd.EtcdException, client.get, "/test_set") @pytest.mark.skip(reason="We need non SHA1-signed certs and I won't implement it now.") diff --git a/src/etcd/tests/test_auth.py b/src/etcd/tests/test_auth.py index e1d88dbb..f122882f 100644 --- a/src/etcd/tests/test_auth.py +++ b/src/etcd/tests/test_auth.py @@ -48,9 +48,7 @@ def test_read(self): self.assertEquals(u.roles, set(["root"])) # The user is correctly rendered out - self.assertEquals( - u._to_net(), [{"user": "root", "password": None, "roles": ["root"]}] - ) + self.assertEquals(u._to_net(), [{"user": "root", "password": None, "roles": ["root"]}]) # An inexistent user raises the appropriate exception u = auth.EtcdUser(self.client, "user.does.not.exist") diff --git a/src/etcd/tests/unit/test_client.py b/src/etcd/tests/unit/test_client.py index f8a51890..37cdee17 100644 --- a/src/etcd/tests/unit/test_client.py +++ b/src/etcd/tests/unit/test_client.py @@ -115,9 +115,7 @@ def test_set_username_password(self): def test_get_headers_with_auth(self): client = etcd.Client(username="username", password="password") - assert client._get_headers() == { - "authorization": "Basic dXNlcm5hbWU6cGFzc3dvcmQ=" - } + assert client._get_headers() == {"authorization": "Basic dXNlcm5hbWU6cGFzc3dvcmQ="} def test__set_version_info(self): """Verify _set_version_info makes the proper call to the server""" @@ -179,16 +177,12 @@ def test_discover(self): method = dns.name.from_text r.target = method("etcd{}.example.com".format(i)) answers.append(r) - dns.resolver.query = mock.create_autospec( - dns.resolver.query, return_value=answers - ) + dns.resolver.query = mock.create_autospec(dns.resolver.query, return_value=answers) self.machines = etcd.Client.machines etcd.Client.machines = mock.create_autospec( etcd.Client.machines, return_value=["https://etcd2.example.com:2379"] ) - c = etcd.Client( - srv_domain="example.com", allow_reconnect=True, protocol="https" - ) + c = etcd.Client(srv_domain="example.com", allow_reconnect=True, protocol="https") etcd.Client.machines = self.machines self.assertEqual(c.host, "etcd1.example.com") self.assertEqual(c.port, 2379) diff --git a/src/etcd/tests/unit/test_lock.py b/src/etcd/tests/unit/test_lock.py index 0d70075b..5996872e 100644 --- a/src/etcd/tests/unit/test_lock.py +++ b/src/etcd/tests/unit/test_lock.py @@ -149,7 +149,9 @@ def side_effect(): } self._mock_api(200, d) - self.locker._get_locker = mock.create_autospec(self.locker._get_locker, side_effect=side_effect) + self.locker._get_locker = mock.create_autospec( + self.locker._get_locker, side_effect=side_effect + ) self.assertTrue(self.locker._acquired()) def test_lock_key(self): diff --git a/src/etcd/tests/unit/test_old_request.py b/src/etcd/tests/unit/test_old_request.py index 8f78cb33..b660c24d 100644 --- a/src/etcd/tests/unit/test_old_request.py +++ b/src/etcd/tests/unit/test_old_request.py @@ -98,7 +98,9 @@ def test_test_and_test_failure(self): client = etcd.Client() client.api_execute = mock.Mock( - side_effect=ValueError("The given PrevValue is not equal" " to the value of the key : TestAndSet: 1!=3") + side_effect=ValueError( + "The given PrevValue is not equal" " to the value of the key : TestAndSet: 1!=3" + ) ) try: result = client.test_and_set("/testkey", "newvalue", "test", ttl=19) @@ -145,7 +147,11 @@ def test_get(self): client.api_execute = mock.Mock( return_value=FakeHTTPResponse( 200, - '{"action":"GET",' '"node": {' '"key":"/testkey",' '"value":"test",' '"modifiedIndex":190}}', + '{"action":"GET",' + '"node": {' + '"key":"/testkey",' + '"value":"test",' + '"modifiedIndex":190}}', ) ) @@ -181,7 +187,11 @@ def test_in(self): client.api_execute = mock.Mock( return_value=FakeHTTPResponse( 200, - '{"action":"GET",' '"node": {' '"key":"/testkey",' '"value":"test",' '"modifiedIndex":190}}', + '{"action":"GET",' + '"node": {' + '"key":"/testkey",' + '"value":"test",' + '"modifiedIndex":190}}', ) ) result = "/testkey" in client diff --git a/src/etcd/tests/unit/test_request.py b/src/etcd/tests/unit/test_request.py index ff355b03..7685dcaf 100644 --- a/src/etcd/tests/unit/test_request.py +++ b/src/etcd/tests/unit/test_request.py @@ -386,7 +386,9 @@ def _mock_api(self, status, d, cluster_id=None): self.client.http.request = mock.MagicMock(return_value=resp) def _mock_error(self, error_code, msg, cause, method="PUT", fields=None, cluster_id=None): - resp = self._prepare_response(500, {"errorCode": error_code, "message": msg, "cause": cause}) + resp = self._prepare_response( + 500, {"errorCode": error_code, "message": msg, "cause": cause} + ) resp.getheader.return_value = cluster_id or "abcdef1234" self.client.http.request_encode_body = mock.create_autospec( self.client.http.request_encode_body, return_value=resp @@ -402,7 +404,9 @@ def test_watch_timeout(self): """Exception will be raised if prevValue != value in test_set""" self.client.http.request = mock.create_autospec( self.client.http.request, - side_effect=urllib3.exceptions.ReadTimeoutError(self.client.http, "foo", "Read timed out"), + side_effect=urllib3.exceptions.ReadTimeoutError( + self.client.http, "foo", "Read timed out" + ), ) self.assertRaises( etcd.EtcdWatchTimedOut, @@ -435,7 +439,9 @@ def test_read_cluster_id_changed(self): self.client.read("/testkey") def test_read_connection_error(self): - self.client.http.request = mock.create_autospec(self.client.http.request, side_effect=socket.error()) + self.client.http.request = mock.create_autospec( + self.client.http.request, side_effect=socket.error() + ) self.assertRaises(etcd.EtcdConnectionFailed, self.client.read, "/something") # Direct GET request self.assertRaises(etcd.EtcdConnectionFailed, self.client.api_execute, "/a", "GET") From 468a5bffd2bdaeeb99484afdb91d790576563ed4 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Mon, 30 Oct 2023 15:19:05 +0100 Subject: [PATCH 096/101] Switch testing to use tox --- .travis.yml | 6 +- black.toml | 19 +++++ bootstrap.py | 188 -------------------------------------------------- build_etcd.sh | 27 -------- buildout.cfg | 41 ----------- tox.ini | 31 +++++++++ 6 files changed, 53 insertions(+), 259 deletions(-) create mode 100644 black.toml delete mode 100644 bootstrap.py delete mode 100755 build_etcd.sh delete mode 100644 buildout.cfg create mode 100644 tox.ini diff --git a/.travis.yml b/.travis.yml index f4d3a8f1..8143a4d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: python python: - "3.7" + - "3.11" before_install: - ./download_etcd.sh 3.4.0 @@ -11,12 +12,11 @@ install: - pip install coveralls - pip install coverage - pip install pytest - - python bootstrap.py + - pip install tox-travis - bin/buildout # command to run tests -script: - PATH=$PATH:./bin coverage run --source=src/etcd --omit="src/etcd/tests/*" bin/test +script: tox after_success: coveralls # Add env var to detect it during build diff --git a/black.toml b/black.toml new file mode 100644 index 00000000..11d9138c --- /dev/null +++ b/black.toml @@ -0,0 +1,19 @@ +[tool.black] +line-length = 100 +target-version = ['py37'] +include = '\.pyi?$' +exclude = ''' +( + \.eggs # exclude a few common directories in the + | \.git # root of the project + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | venv + | _build + | buck-out + | build + | dist +) +''' diff --git a/bootstrap.py b/bootstrap.py deleted file mode 100644 index f7d49e2f..00000000 --- a/bootstrap.py +++ /dev/null @@ -1,188 +0,0 @@ -############################################################################## -# -# Copyright (c) 2006 Zope Foundation and Contributors. -# All Rights Reserved. -# -# This software is subject to the provisions of the Zope Public License, -# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE. -# -############################################################################## -"""Bootstrap a buildout-based project - -Simply run this script in a directory containing a buildout.cfg. -The script accepts buildout command-line options, so you can -use the -c option to specify an alternate configuration file. -""" - -import os -import shutil -import sys -import tempfile - -from optparse import OptionParser - -tmpeggs = tempfile.mkdtemp() - -usage = """\ -[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options] - -Bootstraps a buildout-based project. - -Simply run this script in a directory containing a buildout.cfg, using the -Python that you want bin/buildout to use. - -Note that by using --find-links to point to local resources, you can keep -this script from going over the network. -""" - -parser = OptionParser(usage=usage) -parser.add_option("-v", "--version", help="use a specific zc.buildout version") - -parser.add_option( - "-t", - "--accept-buildout-test-releases", - dest="accept_buildout_test_releases", - action="store_true", - default=False, - help=( - "Normally, if you do not specify a --version, the " - "bootstrap script and buildout gets the newest " - "*final* versions of zc.buildout and its recipes and " - "extensions for you. If you use this flag, " - "bootstrap and buildout will get the newest releases " - "even if they are alphas or betas." - ), -) -parser.add_option( - "-c", - "--config-file", - help=("Specify the path to the buildout configuration " "file to be used."), -) -parser.add_option( - "-f", "--find-links", help=("Specify a URL to search for buildout releases") -) - - -options, args = parser.parse_args() - -###################################################################### -# load/install setuptools - -to_reload = False -try: - import pkg_resources - import setuptools -except ImportError: - ez = {} - - try: - from urllib.request import urlopen - except ImportError: - from urllib2 import urlopen - - # XXX use a more permanent ez_setup.py URL when available. - exec( - urlopen("https://bitbucket.org/pypa/setuptools/raw/0.7.2/ez_setup.py").read(), - ez, - ) - setup_args = dict(to_dir=tmpeggs, download_delay=0) - ez["use_setuptools"](**setup_args) - - if to_reload: - reload(pkg_resources) - import pkg_resources - - # This does not (always?) update the default working set. We will - # do it. - for path in sys.path: - if path not in pkg_resources.working_set.entries: - pkg_resources.working_set.add_entry(path) - -###################################################################### -# Install buildout - -ws = pkg_resources.working_set - -cmd = [ - sys.executable, - "-c", - "from setuptools.command.easy_install import main; main()", - "-mZqNxd", - tmpeggs, -] - -find_links = os.environ.get( - "bootstrap-testing-find-links", - options.find_links - or ( - "http://downloads.buildout.org/" - if options.accept_buildout_test_releases - else None - ), -) -if find_links: - cmd.extend(["-f", find_links]) - -setuptools_path = ws.find(pkg_resources.Requirement.parse("setuptools")).location - -requirement = "zc.buildout" -version = options.version -if version is None and not options.accept_buildout_test_releases: - # Figure out the most recent final version of zc.buildout. - import setuptools.package_index - - _final_parts = "*final-", "*final" - - def _final_version(parsed_version): - for part in parsed_version: - if (part[:1] == "*") and (part not in _final_parts): - return False - return True - - index = setuptools.package_index.PackageIndex(search_path=[setuptools_path]) - if find_links: - index.add_find_links((find_links,)) - req = pkg_resources.Requirement.parse(requirement) - if index.obtain(req) is not None: - best = [] - bestv = None - for dist in index[req.project_name]: - distv = dist.parsed_version - if _final_version(distv): - if bestv is None or distv > bestv: - best = [dist] - bestv = distv - elif distv == bestv: - best.append(dist) - if best: - best.sort() - version = best[-1].version -if version: - requirement = "==".join((requirement, version)) -cmd.append(requirement) - -import subprocess - -if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0: - raise Exception("Failed to execute command:\n%s", repr(cmd)[1:-1]) - -###################################################################### -# Import and run buildout - -ws.add_entry(tmpeggs) -ws.require(requirement) -import zc.buildout.buildout - -if not [a for a in args if "=" not in a]: - args.append("bootstrap") - -# if -c was provided, we push it back into args for buildout' main function -if options.config_file is not None: - args[0:0] = ["-c", options.config_file] - -zc.buildout.buildout.main(args) -shutil.rmtree(tmpeggs) diff --git a/build_etcd.sh b/build_etcd.sh deleted file mode 100755 index 5ce9d664..00000000 --- a/build_etcd.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -if [ $# -gt 0 ] - then - ETCD_VERSION="$1"; - else - ETCD_VERSION="master"; -fi - -echo "Using ETCD version $ETCD_VERSION" - -BASE=$PWD -mkdir -p gopath/src/coreos/ -export GOPATH=$BASE/gopath/ -cd $GOPATH/src/coreos -git clone https://github.com/coreos/etcd.git -cd etcd -git checkout -b buildout $ETCD_VERSION -./build -cd $BASE -cp -r $GOPATH/src/coreos/etcd/bin . - - -${TRAVIS:?"This is not a Travis build. All Done"} -#Temporal solution to travis issue #155 -sudo rm -rf /dev/shm && sudo ln -s /run/shm /dev/shm -echo "All Done" diff --git a/buildout.cfg b/buildout.cfg deleted file mode 100644 index 3a1e0baf..00000000 --- a/buildout.cfg +++ /dev/null @@ -1,41 +0,0 @@ -[buildout] -parts = python - sphinxbuilder - test - coverage -develop = . -eggs = - urllib3==1.19.1 - pyOpenSSL==16.2 - ${deps:extraeggs} - -[python] -recipe = zc.recipe.egg -interpreter = python -eggs = ${buildout:eggs} - -[test] -recipe = pbp.recipe.noserunner -eggs = ${python:eggs} - mock - -[coverage] -recipe = pbp.recipe.noserunner -eggs = ${test:eggs} - coverage -defaults = --with-coverage - --cover-package=etcd - -[sphinxbuilder] -recipe = collective.recipe.sphinxbuilder -source = ${buildout:directory}/docs-source -build = ${buildout:directory}/docs - - -[deps:python2] -extraeggs = - dnspython==1.12.0 - -[deps:python3] -extraeggs = - dnspython3==1.12.0 diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..c8b44115 --- /dev/null +++ b/tox.ini @@ -0,0 +1,31 @@ +[tox] +minversion = 2.5.0 +envlist = py{3,37}-{style,unit} +skip_missing_interpreters = True + +[testenv] +usedevelop = True +basepython = + py3: python3 + py37: python3.7 +description = + style: Style consistency checker + unit: Run unit tests. + py3: (Python 3.x) + py37: (Python 3.7) + +commands = +; style: flake8 + style: black --config black.toml --check src + unit: pytest --cov=etcd src/etcd/tests/ --cov-report=term-missing + +deps = + style: flake8 + style: black + unit: pytest-cov + unit: pyOpenSSL>=0.14 + +[flake8] +max-line-length = 100 +statistics = True +exclude = .venv,.eggs,.tox,build,venv From 51de50482d2cb7cda2e13f05dcafcc189511af25 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Mon, 30 Oct 2023 15:58:19 +0100 Subject: [PATCH 097/101] Add github actions, remove travis --- .github/workflows/python-package.yaml | 36 +++++++++++++++++++++++++++ .travis.yml | 23 ----------------- 2 files changed, 36 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/python-package.yaml delete mode 100644 .travis.yml diff --git a/.github/workflows/python-package.yaml b/.github/workflows/python-package.yaml new file mode 100644 index 00000000..96118dff --- /dev/null +++ b/.github/workflows/python-package.yaml @@ -0,0 +1,36 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Python package + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + ./download_etcd.sh 3.4.0 + python -m pip install --upgrade pip + python -m pip install tox coveralls + - name: Test with tox + run: | + tox + coveralls + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 8143a4d9..00000000 --- a/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -language: python -python: - - "3.7" - - "3.11" - -before_install: - - ./download_etcd.sh 3.4.0 - - pip install --upgrade setuptools - -# command to install dependencies -install: - - pip install coveralls - - pip install coverage - - pip install pytest - - pip install tox-travis - - bin/buildout - -# command to run tests -script: tox - -after_success: coveralls -# Add env var to detect it during build -env: TRAVIS=True From dcb22ef1492bb9bd4f5dd0098c6586abc53854a3 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Mon, 30 Oct 2023 16:57:17 +0100 Subject: [PATCH 098/101] Try to fix etcd path --- download_etcd.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/download_etcd.sh b/download_etcd.sh index bdd592de..f9e97dd4 100755 --- a/download_etcd.sh +++ b/download_etcd.sh @@ -4,3 +4,4 @@ VERSION=${1:-2.3.7} mkdir -p bin URL="https://github.com/coreos/etcd/releases/download/v${VERSION}/etcd-v${VERSION}-linux-amd64.tar.gz" curl -L $URL | tar -C ./bin --strip-components=1 -xzvf - "etcd-v${VERSION}-linux-amd64/etcd" +mv bin/etcd /usr/local/bin/ From f7c5c35d8e607a609e0a5a1ebc7b2c1a951557a6 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Tue, 31 Oct 2023 06:45:06 +0100 Subject: [PATCH 099/101] github action: add env variable for coveralls --- .github/workflows/python-package.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python-package.yaml b/.github/workflows/python-package.yaml index 96118dff..ee8141fc 100644 --- a/.github/workflows/python-package.yaml +++ b/.github/workflows/python-package.yaml @@ -30,6 +30,8 @@ jobs: python -m pip install --upgrade pip python -m pip install tox coveralls - name: Test with tox + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | tox coveralls From 5aea0fd4461bd05dd96e4ad637f6be7bceb1cee5 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Tue, 31 Oct 2023 07:46:53 +0100 Subject: [PATCH 100/101] Docs update --- NEWS.txt | 13 +++++++++++++ README.rst | 11 +++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/NEWS.txt b/NEWS.txt index 00d8cd25..39f8db71 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -1,5 +1,18 @@ News ==== +0.5.0 +----- +*Release date: 31-Oct-2023 + +* Drop python 2.x compatibility (should still work) +* Move to use pytest +* Support urllib3 v2, including support of self-signed certs +* Fix version check to avoid crashes with non-official releases +* Correctly handle watch timeouts in lock +* Allow trying more than one domain when looking up SRV records +* Support auth API both <= 2.2.5 and >= 2.3.0 +* Use github actions instead than travis + 0.4.5 ----- *Release date: 3-Mar-2017* diff --git a/README.rst b/README.rst index 0df9c602..8d68c00a 100644 --- a/README.rst +++ b/README.rst @@ -27,13 +27,13 @@ From source .. code:: bash $ python setup.py install - + From Pypi ~~~~~~~~~ .. code:: bash - $ python3.5 -m pip install python-etcd + $ python -m pip install python-etcd Usage ----- @@ -209,18 +209,17 @@ List contents of a directory Development setup ----------------- -To create a buildout, +To check your code, .. code:: bash - $ python bootstrap.py - $ bin/buildout + $ tox to test you should have etcd available in your system path: .. code:: bash - $ bin/test + $ command -v etcd to generate documentation, From d2889f7b23feee8797657b19c404f0d4034dd03c Mon Sep 17 00:00:00 2001 From: Karolina Surma <33810531+befeleme@users.noreply.github.com> Date: Wed, 24 Apr 2024 09:29:01 +0200 Subject: [PATCH 101/101] Support Python 3.13 What's new in Python 3.13 states: "logging: Remove undocumented and untested Logger.warn() and LoggerAdapter.warn() methods and logging.warn() function. Deprecated since Python 3.3, they were aliases to the logging.Logger.warning() method, logging.LoggerAdapter.warning() method and logging.warning() function. (Contributed by Victor Stinner in gh-105376.)" --- src/etcd/lock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/etcd/lock.py b/src/etcd/lock.py index 013bc234..2f790dd7 100644 --- a/src/etcd/lock.py +++ b/src/etcd/lock.py @@ -35,7 +35,7 @@ def uuid(self, value): old_uuid = self._uuid self._uuid = value if not self._find_lock(): - _log.warn("The hand-set uuid was not found, refusing") + _log.warning("The hand-set uuid was not found, refusing") self._uuid = old_uuid raise ValueError("Inexistent UUID") @@ -51,7 +51,7 @@ def is_acquired(self): self.client.read(self.lock_key) return True except etcd.EtcdKeyNotFound: - _log.warn("Lock was supposedly taken, but we cannot find it") + _log.warning("Lock was supposedly taken, but we cannot find it") self.is_taken = False return False