Adding flaky tests plugin

Jon Wayne Parrott · Jon Wayne Parrott · commit a553b2246c19 · 2016-01-22T13:24:35.000-08:00
There were various datastore tests that were failing due to serviceunavailable
errors. Now, all the integration tests are marked with the flaky plugin but
will only retry for "gcloud.exceptions.ServiceUnavailable" exceptions.
diff --git a/2-structured-data/requirements-dev.txt b/2-structured-data/requirements-dev.txt
@@ -5,3 +5,4 @@ flake8
 coverage
 BeautifulSoup4
 mock
+flaky
diff --git a/2-structured-data/tests/test_crud.py b/2-structured-data/tests/test_crud.py
@@ -16,9 +16,16 @@
 
 import bookshelf
 import config
+from flaky import flaky
+from gcloud.exceptions import ServiceUnavailable
 from nose.plugins.attrib import attr
 
 
+def flaky_filter(e, *args):
+    return isinstance(e, ServiceUnavailable)
+
+
+@flaky(rerun_filter=flaky_filter)
 class IntegrationBase(unittest.TestCase):
 
     def createBooks(self, n=1):
diff --git a/2-structured-data/tox.ini b/2-structured-data/tox.ini
@@ -8,7 +8,12 @@ deps =
   -rrequirements.txt
   -rrequirements-dev.txt
 commands =
-  nosetests --with-coverage --cover-package bookshelf {posargs:-a '!e2e'}
+  nosetests \
+    --with-flaky \
+    --no-success-flaky-report \
+    --with-coverage \
+    --cover-package bookshelf \
+    {posargs:-a '!e2e'}
 passenv = GOOGLE_APPLICATION_CREDENTIALS
 
 [testenv:py34]
diff --git a/3-binary-data/requirements-dev.txt b/3-binary-data/requirements-dev.txt
@@ -5,3 +5,4 @@ flake8
 coverage
 BeautifulSoup4
 mock
+flaky
diff --git a/3-binary-data/tests/test_crud.py b/3-binary-data/tests/test_crud.py
@@ -16,9 +16,16 @@
 
 import bookshelf
 import config
+from flaky import flaky
+from gcloud.exceptions import ServiceUnavailable
 from nose.plugins.attrib import attr
 
 
+def flaky_filter(e, *args):
+    return isinstance(e, ServiceUnavailable)
+
+
+@flaky(rerun_filter=flaky_filter)
 class IntegrationBase(unittest.TestCase):
 
     def createBooks(self, n=1):
diff --git a/3-binary-data/tox.ini b/3-binary-data/tox.ini
@@ -8,7 +8,12 @@ deps =
   -rrequirements.txt
   -rrequirements-dev.txt
 commands =
-  nosetests --with-coverage --cover-package bookshelf {posargs:-a '!e2e'}
+  nosetests \
+    --with-flaky \
+    --no-success-flaky-report \
+    --with-coverage \
+    --cover-package bookshelf \
+    {posargs:-a '!e2e'}
 passenv = GOOGLE_APPLICATION_CREDENTIALS
 
 [testenv:py34]
diff --git a/4-auth/requirements-dev.txt b/4-auth/requirements-dev.txt
@@ -5,3 +5,4 @@ flake8
 coverage
 BeautifulSoup4
 mock
+flaky
diff --git a/4-auth/tests/test_crud.py b/4-auth/tests/test_crud.py
@@ -16,12 +16,17 @@
 
 import bookshelf
 import config
+from flaky import flaky
+from gcloud.exceptions import ServiceUnavailable
 from nose.plugins.attrib import attr
+from oauth2client.client import OAuth2Credentials
 
 
-from oauth2client.client import OAuth2Credentials
+def flaky_filter(e, *args):
+    return isinstance(e, ServiceUnavailable)
 
 
+@flaky(rerun_filter=flaky_filter)
 class IntegrationBase(unittest.TestCase):
 
     def createBooks(self, n=1):
diff --git a/4-auth/tox.ini b/4-auth/tox.ini
@@ -8,7 +8,12 @@ deps =
   -rrequirements.txt
   -rrequirements-dev.txt
 commands =
-  nosetests --with-coverage --cover-package bookshelf {posargs:-a '!e2e'}
+  nosetests \
+    --with-flaky \
+    --no-success-flaky-report \
+    --with-coverage \
+    --cover-package bookshelf \
+    {posargs:-a '!e2e'}
 passenv = GOOGLE_APPLICATION_CREDENTIALS
 
 [testenv:py34]
diff --git a/5-logging/requirements-dev.txt b/5-logging/requirements-dev.txt
@@ -5,3 +5,4 @@ flake8
 coverage
 BeautifulSoup4
 mock
+flaky
diff --git a/5-logging/tests/test_crud.py b/5-logging/tests/test_crud.py
@@ -16,12 +16,17 @@
 
 import bookshelf
 import config
+from flaky import flaky
+from gcloud.exceptions import ServiceUnavailable
 from nose.plugins.attrib import attr
+from oauth2client.client import OAuth2Credentials
 
 
-from oauth2client.client import OAuth2Credentials
+def flaky_filter(e, *args):
+    return isinstance(e, ServiceUnavailable)
 
 
+@flaky(rerun_filter=flaky_filter)
 class IntegrationBase(unittest.TestCase):
 
     def createBooks(self, n=1):
diff --git a/5-logging/tox.ini b/5-logging/tox.ini
@@ -8,7 +8,12 @@ deps =
   -rrequirements.txt
   -rrequirements-dev.txt
 commands =
-  nosetests --with-coverage --cover-package bookshelf {posargs:-a '!e2e'}
+  nosetests \
+    --with-flaky \
+    --no-success-flaky-report \
+    --with-coverage \
+    --cover-package bookshelf \
+    {posargs:-a '!e2e'}
 passenv = GOOGLE_APPLICATION_CREDENTIALS
 
 [testenv:py34]
diff --git a/6-pubsub/requirements-dev.txt b/6-pubsub/requirements-dev.txt
@@ -5,3 +5,4 @@ flake8
 coverage
 BeautifulSoup4
 mock
+flaky
diff --git a/6-pubsub/tests/test_crud.py b/6-pubsub/tests/test_crud.py
@@ -17,11 +17,18 @@
 import bookshelf
 from bookshelf import tasks
 import config
+from flaky import flaky
+from gcloud.exceptions import ServiceUnavailable
 import mock
 from nose.plugins.attrib import attr
 from oauth2client.client import OAuth2Credentials
 
 
+def flaky_filter(e, *args):
+    return isinstance(e, ServiceUnavailable)
+
+
+@flaky(rerun_filter=flaky_filter)
 class IntegrationBase(unittest.TestCase):
 
     def createBooks(self, n=1):
diff --git a/6-pubsub/tox.ini b/6-pubsub/tox.ini
@@ -8,7 +8,12 @@ deps =
   -rrequirements.txt
   -rrequirements-dev.txt
 commands =
-  nosetests --with-coverage --cover-package bookshelf {posargs:-a '!e2e'}
+  nosetests \
+    --with-flaky \
+    --no-success-flaky-report \
+    --with-coverage \
+    --cover-package bookshelf \
+    {posargs:-a '!e2e'}
 passenv = GOOGLE_APPLICATION_CREDENTIALS
 
 [testenv:py34]
diff --git a/7-gce/requirements-dev.txt b/7-gce/requirements-dev.txt
@@ -5,3 +5,4 @@ flake8
 coverage
 BeautifulSoup4
 mock
+flaky
diff --git a/7-gce/tests/test_crud.py b/7-gce/tests/test_crud.py
@@ -17,11 +17,18 @@
 import bookshelf
 from bookshelf import tasks
 import config
+from flaky import flaky
+from gcloud.exceptions import ServiceUnavailable
 import mock
 from nose.plugins.attrib import attr
 from oauth2client.client import OAuth2Credentials
 
 
+def flaky_filter(e, *args):
+    return isinstance(e, ServiceUnavailable)
+
+
+@flaky(rerun_filter=flaky_filter)
 class IntegrationBase(unittest.TestCase):
 
     def createBooks(self, n=1):
diff --git a/7-gce/tox.ini b/7-gce/tox.ini
@@ -8,7 +8,12 @@ deps =
   -rrequirements.txt
   -rrequirements-dev.txt
 commands =
-  nosetests --with-coverage --cover-package bookshelf {posargs:-a '!e2e'}
+  nosetests \
+    --with-flaky \
+    --no-success-flaky-report \
+    --with-coverage \
+    --cover-package bookshelf \
+    {posargs:-a '!e2e'}
 passenv = GOOGLE_APPLICATION_CREDENTIALS
 
 [testenv:py34]
diff --git a/optional-container-engine/requirements-dev.txt b/optional-container-engine/requirements-dev.txt
@@ -5,3 +5,4 @@ flake8
 coverage
 BeautifulSoup4
 mock
+flaky
diff --git a/optional-container-engine/tests/test_crud.py b/optional-container-engine/tests/test_crud.py
@@ -11,17 +11,23 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import unittest
 
 import bookshelf
 from bookshelf import tasks
 import config
+from flaky import flaky
+from gcloud.exceptions import ServiceUnavailable
 import mock
 from nose.plugins.attrib import attr
 from oauth2client.client import OAuth2Credentials
 
 
+def flaky_filter(e, *args):
+    return isinstance(e, ServiceUnavailable)
+
+
+@flaky(rerun_filter=flaky_filter)
 class IntegrationBase(unittest.TestCase):
 
     def createBooks(self, n=1):
diff --git a/optional-container-engine/tox.ini b/optional-container-engine/tox.ini
@@ -8,7 +8,12 @@ deps =
   -rrequirements.txt
   -rrequirements-dev.txt
 commands =
-  nosetests --with-coverage --cover-package bookshelf {posargs:-a '!e2e'}
+  nosetests \
+    --with-flaky \
+    --no-success-flaky-report \
+    --with-coverage \
+    --cover-package bookshelf \
+    {posargs:-a '!e2e'}
 passenv = GOOGLE_APPLICATION_CREDENTIALS
 
 [testenv:py34]
