catch OperationTimedOut in consistency tests

kishkaru · kishkaru · commit 86436847aada · 2015-04-20T18:36:09.000-07:00
diff --git a/tests/integration/long/test_consistency.py b/tests/integration/long/test_consistency.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import struct, logging, sys, traceback
+import struct, logging, sys, traceback, time
 
 from cassandra import ConsistencyLevel, OperationTimedOut, ReadTimeout, WriteTimeout, Unavailable
 from cassandra.cluster import Cluster
@@ -65,7 +65,15 @@ def _insert(self, session, keyspace, count, consistency_level=ConsistencyLevel.O
         for i in range(count):
             ss = SimpleStatement('INSERT INTO cf(k, i) VALUES (0, 0)',
                                  consistency_level=consistency_level)
-            session.execute(ss)
+            while True:
+                try:
+                    session.execute(ss)
+                    break
+                except (OperationTimedOut, WriteTimeout):
+                    ex_type, ex, tb = sys.exc_info()
+                    log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
+                    del tb
+                    time.sleep(1)
 
     def _query(self, session, keyspace, count, consistency_level=ConsistencyLevel.ONE):
         routing_key = struct.pack('>i', 0)
@@ -81,6 +89,7 @@ def _query(self, session, keyspace, count, consistency_level=ConsistencyLevel.ON
                     ex_type, ex, tb = sys.exc_info()
                     log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
                     del tb
+                    time.sleep(1)
 
     def _assert_writes_succeed(self, session, keyspace, consistency_levels):
         for cl in consistency_levels:
diff --git a/tests/integration/long/utils.py b/tests/integration/long/utils.py
@@ -133,21 +133,21 @@ def ring(node):
 def wait_for_up(cluster, node, wait=True):
     while True:
         host = cluster.metadata.get_host(IP_FORMAT % node)
-        time.sleep(0.1)
         if host and host.is_up:
             log.debug("Done waiting for node %s to be up", node)
             return
         else:
             log.debug("Host is still marked down, waiting")
+            time.sleep(1)
 
 
 def wait_for_down(cluster, node, wait=True):
     log.debug("Waiting for node %s to be down", node)
     while True:
         host = cluster.metadata.get_host(IP_FORMAT % node)
-        time.sleep(0.1)
         if not host or not host.is_up:
             log.debug("Done waiting for node %s to be down", node)
             return
         else:
             log.debug("Host is still marked up, waiting")
+            time.sleep(1)
