Add coverage

pythonnet · vmuriart · Jan 31, 2017 · Jan 27, 2017 · Jan 27, 2017 · Jan 27, 2017
commit dee134c4939be63e1d4c9dbe407740c64e08f4bc
diff --git a/.travis.yml b/.travis.yml
@@ -25,14 +25,20 @@ addons:
       - nunit-console
 
 install:
-  - pip install six
-  - pip install pycparser
-  - python setup.py build_ext --inplace
+  - pip install pycparser coverage codecov six
+  - coverage run setup.py build_ext --inplace
 
 script:
   - export PYTHONPATH=`pwd`:$PYTHONPATH
   - python src/tests/runtests.py
   # - nunit-console src/embed_tests/bin/x64/ReleaseMono/Python.EmbeddingTest.dll
 
+after_success:
+  # Uncomment if need to geninterop, ie. py37 final
+  # - python tools/geninterop/geninterop.py
+
+  # Waiting on mono-cov support or SharpCover
+  - codecov
+
 notifications:
   email: false
diff --git a/README.md b/README.md
@@ -2,6 +2,7 @@
 
 [![appveyor shield][]](https://ci.appveyor.com/project/pythonnet/pythonnet/branch/master)
 [![travis shield][]](https://travis-ci.org/pythonnet/pythonnet)
+[![codecov shield][]](https://codecov.io/github/pythonnet/pythonnet)
 [![license shield][]](./LICENSE)
 [![pypi package version][]](https://pypi.python.org/pypi/pythonnet)
 [![python supported shield][]](https://pypi.python.org/pypi/pythonnet)
@@ -80,6 +81,8 @@ int32
 
 [appveyor shield]: https://img.shields.io/appveyor/ci/pythonnet/pythonnet/master.svg?label=AppVeyor
 
+[codecov shield]: https://img.shields.io/codecov/c/github/pythonnet/pythonnet/pytest.svg?label=codecov
+
 [license shield]: https://img.shields.io/badge/license-MIT-blue.svg
 
 [pypi package version]: https://img.shields.io/pypi/v/pythonnet.svg

diff --git a/appveyor.yml b/appveyor.yml
@@ -10,7 +10,6 @@ environment:
     PYTHONUNBUFFERED: True
     PYTHONWARNINGS: 'ignore:::wheel.pep425tags:'
     PYTHONPATH: C:\testdir
-    NUNIT: nunit-console
     CONDA_BLD: C:\conda
     CONDA_BLD_VERSION: 3.5
 
@@ -35,7 +34,6 @@ init:
   - set CONDA_BLD_ARCH=%PLATFORM:x=%
   - set PYTHON=C:\PYTHON%PYTHON_VERSION:.=%
   - if %PLATFORM%==x86 (set CONDA_BLD_ARCH=32)
-  - if %PLATFORM%==x86 (set NUNIT=%NUNIT%-x86)
   - if %PLATFORM%==x64 (set PYTHON=%PYTHON%-x64)
 
   # Prepend newly installed Python to the PATH of this build
@@ -47,21 +45,29 @@ init:
   - python -c "import ctypes; print(ctypes.sizeof(ctypes.c_wchar))"
 
 install:
-  # install for wheels
-  - pip install --upgrade pip wheel six
+  # install for wheels & coverage
+  - pip install --upgrade pip wheel coverage codecov six
+
+  # Install OpenCover. Can't put on packages.config; not Linux/Mono compatible
+  - .\tools\nuget\nuget.exe install OpenCover -OutputDirectory packages
 
 build_script:
-  # build clean sdist & wheel
-  - python setup.py sdist bdist_wheel
+  # build clean sdist & wheel with coverage of setup.py, install local wheel
+  - coverage run setup.py sdist bdist_wheel
 
 test_script:
   - pip install --no-index --find-links=.\dist\ pythonnet
   - ps: Copy-Item (Resolve-Path .\build\*\Python.Test.dll) C:\testdir
-  - python src\tests\runtests.py
-  # - "%NUNIT% src/embed_tests/bin/%PLATFORM%/ReleaseWin/Python.EmbeddingTest.dll"
+
+  # Test runner
+  - ps: .\ci\appveyor_run_tests.ps1
 
   # Build conda-recipe on Pull Requests
   - ps: .\ci\appveyor_build_recipe.ps1
 
+on_finish:
+  # Upload coverage
+  - codecov
+
 artifacts:
   - path: dist\*
diff --git a/ci/appveyor_run_tests.ps1 b/ci/appveyor_run_tests.ps1
@@ -0,0 +1,33 @@
+# Script to simplify appveyor configuration and resolve path to tools
+
+# Executable paths for OpenCover
+# Note if OpenCover fails, it won't affect the exit codes.
+$OPENCOVER = Resolve-Path .\packages\OpenCover.*\tools\OpenCover.Console.exe
+$NUNIT = Resolve-Path .\packages\NUnit.ConsoleRunner*\tools\nunit3-console.exe
+$PY = Get-Command python
+
+# Can't use ".\build\*\Python.EmbeddingTest.dll". Missing framework files.
+$CS_TESTS = Resolve-Path .\src\embed_tests\bin\*\*\Python.EmbeddingTest.dll
+$RUNTIME_DIR = Resolve-Path .\src\runtime\bin\*\ReleaseWin\
+
+# Run python tests with C# coverage
+# why `2>&1 | %{ "$_" }`? see: http://stackoverflow.com/a/20950421/5208670
+.$OPENCOVER -register:user -searchdirs:"$RUNTIME_DIR" -output:py.coverage -target:"$PY" -targetargs:src\tests\runtests.py -returntargetcode 2>&1 | %{ "$_" }
+$PYTHON_STATUS = $LastExitCode
+if ($PYTHON_STATUS -ne 0) {
+    Write-Host "Python tests failed, continuing to embedded tests" -ForegroundColor "Red"
+}
+
+# Run Embedded tests with C# coverage
+# .$OPENCOVER -register:user -searchdirs:"$RUNTIME_DIR" -output:cs.coverage -target:"$NUNIT" -targetargs:"$CS_TESTS" -returntargetcode
+# $NUNIT_STATUS = $LastExitCode
+# if ($NUNIT_STATUS -ne 0) {
+#     Write-Host "Embedded tests failed" -ForegroundColor "Red"
+# }
+
+# Embedded tests failing due to open issues, pass/fail only on Python exit code
+# if ($PYTHON_STATUS -ne 0 -or $NUNIT_STATUS -ne 0) {
+if ($PYTHON_STATUS -ne 0) {
+    Write-Host "Tests failed" -ForegroundColor "Red"
+    $host.SetShouldExit(1)
+}
diff --git a/src/embed_tests/packages.config b/src/embed_tests/packages.config
@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
   <package id="NUnit" version="3.5.0" targetFramework="net40" />
+  <package id="NUnit.ConsoleRunner" version="3.5.0" targetFramework="net40" />
 </packages>