Stability testing suite.

pythonnet · dmitriyse · Aug 31, 2017 · Sep 2, 2017 · Sep 2, 2017 · Sep 2, 2017
commit 36b23ef1f9af68015313b3bb482028d530ab2ef1
diff --git a/.travis.yml b/.travis.yml
@@ -97,7 +97,7 @@ install:
 
 script:
   - python -m pytest
-  - mono $NUNIT_PATH src/embed_tests/bin/Python.EmbeddingTest.dll
+  - mono $NUNIT_PATH src/embed_tests/bin/Python.EmbeddingTest.exe
   - if [[ $BUILD_OPTS == --xplat ]]; then dotnet src/embed_tests/bin/netcoreapp2.0_publish/Python.EmbeddingTest.dll; fi
 
 after_script:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,8 @@ This document follows the conventions laid out in [Keep a CHANGELOG][].
 ## [unreleased][]
 
 ### Added
+-   Added tool for debugging floating bugs. Stable tests are executed in the loop. ~100 cycles is enough to pop up any bugs.
+    Usage: Python.EmbeddingTest.exe --loop --where="cat != Unstable"
 -   Added support for embedding python into dotnet core 2.0 (NetStandard 2.0)
 -   Added new build system (pythonnet.15.sln) based on dotnetcore-sdk/xplat(crossplatform msbuild).
     Currently there two side-by-side build systems that produces the same output (net40) from the same sources.

diff --git a/ci/appveyor_run_tests.ps1 b/ci/appveyor_run_tests.ps1
@@ -20,7 +20,7 @@ else{
 $PY = Get-Command python
 
 # Can't use ".\build\*\Python.EmbeddingTest.dll". Missing framework files.
-$CS_TESTS = ".\src\embed_tests\bin\Python.EmbeddingTest.dll"
+$CS_TESTS = ".\src\embed_tests\bin\Python.EmbeddingTest.exe"
 $RUNTIME_DIR = ".\src\runtime\bin\"
 
 # Run python tests with C# coverage

diff --git a/src/embed_tests/Program.cs b/src/embed_tests/Program.cs
@@ -1,19 +1,54 @@
 using System;
-
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
 using NUnit.Common;
 
 using NUnitLite;
+using Python.Runtime;
 
 namespace Python.EmbeddingTest
 {
     public class Program
     {
         public static int Main(string[] args)
         {
-            return new AutoRun(typeof(Program).Assembly).Execute(
-                args,
-                new ExtendedTextWrapper(Console.Out),
-                Console.In);
+            if (args.Contains("--loop"))
+            {
+                args = args.Where(x => x != "--loop").ToArray();
+                int result;
+                int runNumber = 0;
+                string pathEnv = Environment.GetEnvironmentVariable("PATH");
+                do
+                {
+                    Console.ForegroundColor = ConsoleColor.Green;
+                    Console.Error.WriteLine($"----- Run = {++runNumber}, MemUsage = {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024} Mb -----");
+                    Console.ForegroundColor = ConsoleColor.Gray;
+
+                    result = new AutoRun(typeof(Program).Assembly).Execute(
+                        args,
+                        new ExtendedTextWrapper(Console.Out),
+                        Console.In);
+
+                    // Python does not see Environment.SetEnvironmentVariable changes.
+                    // So we needs restore PATH variable in a pythonic way.
+                    using (new PythonEngine())
+                    {
+                        dynamic os = PythonEngine.ImportModule("os");
+                        os.environ["PATH"] = new PyString(pathEnv);
+                    }
+                } while (true);
+
+                return result;
+            }
+            else
+            {
+                return new AutoRun(typeof(Program).Assembly).Execute(
+                    args,
+                    new ExtendedTextWrapper(Console.Out),
+                    Console.In);
+            }
         }
     }
 }
diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj
@@ -5,7 +5,7 @@
     <TargetFrameworks>net40;netcoreapp2.0</TargetFrameworks>
     <Platforms>x64;x86</Platforms>
     <Configurations>DebugMono;DebugMonoPY3;ReleaseMono;ReleaseMonoPY3;DebugWin;DebugWinPY3;ReleaseWin;ReleaseWinPY3</Configurations>
-    <OutputType Condition="'$(TargetFramework)' != 'net40' OR '$(PYTHONNET_VS_ENV)' == 'true'">Exe</OutputType>
+    <OutputType>Exe</OutputType>
     <GenerateProgramFile>false</GenerateProgramFile>
     <AssemblyName>Python.EmbeddingTest</AssemblyName>
     <RootNamespace>Python.EmbeddingTest</RootNamespace>

diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj
@@ -4,7 +4,7 @@
     <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
     <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
     <ProjectGuid>{4165C59D-2822-499F-A6DB-EACA4C331EB5}</ProjectGuid>
-    <OutputType>Library</OutputType>
+    <OutputType>Exe</OutputType>
     <AssemblyName>Python.EmbeddingTest</AssemblyName>
     <RootNamespace>Python.EmbeddingTest</RootNamespace>
     <DocumentationFile>bin\Python.EmbeddingTest.xml</DocumentationFile>
@@ -70,9 +70,12 @@
   </PropertyGroup>
   <ItemGroup>
     <Reference Include="Microsoft.CSharp" />
-    <Reference Include="nunit.framework, Version=3.7.8.0, Culture=neutral, PublicKeyToken=2638cd05610744eb, processorArchitecture=MSIL">
+    <Reference Include="nunit.framework, Version=3.8.1.0, Culture=neutral, PublicKeyToken=2638cd05610744eb, processorArchitecture=MSIL">
       <HintPath>..\..\packages\NUnit.3.8.1\lib\net40\nunit.framework.dll</HintPath>
     </Reference>
+    <Reference Include="nunitlite">
+      <HintPath>..\..\packages\NUnitLite.3.8.1\lib\net40\nunitlite.dll</HintPath>
+    </Reference>
     <Reference Include="System" />
   </ItemGroup>
   <ItemGroup>
@@ -84,6 +87,7 @@
     <Compile Include="pyimport.cs" />
     <Compile Include="pyinitialize.cs" />
     <Compile Include="pyrunstring.cs" />
+    <Compile Include="Program.cs" />
     <Compile Include="TestConverter.cs" />
     <Compile Include="TestCustomMarshal.cs" />
     <Compile Include="TestExample.cs" />

diff --git a/src/embed_tests/TestExample.cs b/src/embed_tests/TestExample.cs
@@ -5,6 +5,7 @@
 
 namespace Python.EmbeddingTest
 {
+    [Category("Unstable")]
     public class TestExample
     {
         [OneTimeSetUp]

diff --git a/src/embed_tests/packages.config b/src/embed_tests/packages.config
@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
   <package id="NUnit" version="3.8.1" targetFramework="net40" />
+  <package id="NUnitLite" version="3.8.1" targetFramework="net40" />
   <package id="NUnit.ConsoleRunner" version="3.7.0" targetFramework="net40" />
 </packages>