add codec tests

pythonnet · lostmsu · Feb 18, 2021 · Mar 10, 2020 · Mar 10, 2020 · Mar 11, 2020
commit f025cd1e8e6a6bbf1a22fcadee40d219dc0c32f8
diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs
@@ -6,28 +6,34 @@ namespace Python.EmbeddingTest {
     using Python.Runtime;
     using Python.Runtime.Codecs;
 
-    public class Codecs {
+    public class Codecs
+    {
         [SetUp]
-        public void SetUp() {
+        public void SetUp()
+        {
             PythonEngine.Initialize();
         }
 
         [TearDown]
-        public void Dispose() {
+        public void Dispose()
+        {
             PythonEngine.Shutdown();
         }
 
         [Test]
-        public void ConversionsGeneric() {
-            ConversionsGeneric<ValueTuple<int, string, object>, ValueTuple>();
+        public void TupleConversionsGeneric()
+        {
+            TupleConversionsGeneric<ValueTuple<int, string, object>, ValueTuple>();
         }
 
-        static void ConversionsGeneric<T, TTuple>() {
+        static void TupleConversionsGeneric<T, TTuple>()
+        {
             TupleCodec<TTuple>.Register();
             var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object());
             T restored = default;
             using (Py.GIL())
-            using (var scope = Py.CreateScope()) {
+            using (var scope = Py.CreateScope())
+            {
                 void Accept(T value) => restored = value;
                 var accept = new Action<T>(Accept).ToPython();
                 scope.Set(nameof(tuple), tuple);
@@ -38,15 +44,18 @@ static void ConversionsGeneric<T, TTuple>() {
         }
 
         [Test]
-        public void ConversionsObject() {
-            ConversionsObject<ValueTuple<int, string, object>, ValueTuple>();
+        public void TupleConversionsObject()
+        {
+            TupleConversionsObject<ValueTuple<int, string, object>, ValueTuple>();
         }
-        static void ConversionsObject<T, TTuple>() {
+        static void TupleConversionsObject<T, TTuple>()
+        {
             TupleCodec<TTuple>.Register();
             var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object());
             T restored = default;
             using (Py.GIL())
-            using (var scope = Py.CreateScope()) {
+            using (var scope = Py.CreateScope())
+            {
                 void Accept(object value) => restored = (T)value;
                 var accept = new Action<object>(Accept).ToPython();
                 scope.Set(nameof(tuple), tuple);
@@ -57,38 +66,42 @@ static void ConversionsObject<T, TTuple>() {
         }
 
         [Test]
-        public void TupleRoundtripObject() {
+        public void TupleRoundtripObject()
+        {
             TupleRoundtripObject<ValueTuple<int, string, object>, ValueTuple>();
         }
-        static void TupleRoundtripObject<T, TTuple>() {
+        static void TupleRoundtripObject<T, TTuple>()
+        {
             var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object());
-            using (Py.GIL()) {
+            using (Py.GIL())
+            {
                 var pyTuple = TupleCodec<TTuple>.Instance.TryEncode(tuple);
                 Assert.IsTrue(TupleCodec<TTuple>.Instance.TryDecode(pyTuple, out object restored));
                 Assert.AreEqual(expected: tuple, actual: restored);
             }
         }
 
         [Test]
-        public void TupleRoundtripGeneric() {
+        public void TupleRoundtripGeneric()
+        {
             TupleRoundtripGeneric<ValueTuple<int, string, object>, ValueTuple>();
         }
 
-        static void TupleRoundtripGeneric<T, TTuple>() {
+        static void TupleRoundtripGeneric<T, TTuple>()
+        {
             var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object());
-            using (Py.GIL()) {
+            using (Py.GIL())
+            {
                 var pyTuple = TupleCodec<TTuple>.Instance.TryEncode(tuple);
                 Assert.IsTrue(TupleCodec<TTuple>.Instance.TryDecode(pyTuple, out T restored));
                 Assert.AreEqual(expected: tuple, actual: restored);
             }
         }
 
-        static PyObject GetPythonIterable()
+        static string GetIntIterableCommands(string instanceName)
         {
-            var locals = new PyDict();
-            using (Py.GIL())
-            {
-                PythonEngine.Exec(@"
+            var builder = new System.Text.StringBuilder();
+            builder.AppendLine(@"
 class foo():
     def __init__(self):
         self.counter = 0
@@ -98,9 +111,18 @@ def __next__(self):
         if self.counter == 3:
             raise StopIteration
         self.counter = self.counter + 1
-        return self.counter
-foo_instance = foo()
-", null, locals.Handle);
+        return self.counter");
+
+            builder.AppendLine(instanceName + " = foo()");
+            return builder.ToString();
+        }
+
+        static PyObject GetPythonIterable()
+        {
+            var locals = new PyDict();
+            using (Py.GIL())
+            {
+                PythonEngine.Exec(GetIntIterableCommands("foo_instance"), null, locals.Handle);
             }
 
             return locals.GetItem("foo_instance");
@@ -113,16 +135,18 @@ public void ListDecoderTest()
             var items = new List<PyObject>() { new PyInt(1), new PyInt(2), new PyInt(3) };
 
             var pyList = new PyList(items.ToArray());
-            Assert.IsTrue(codec.CanDecode(pyList, typeof(IList<bool>)));
-            Assert.IsTrue(codec.CanDecode(pyList, typeof(IList<int>)));
-            Assert.IsFalse(codec.CanDecode(pyList, typeof(System.Collections.IEnumerable)));
-            Assert.IsFalse(codec.CanDecode(pyList, typeof(IEnumerable<int>)));
-            Assert.IsFalse(codec.CanDecode(pyList, typeof(ICollection<float>)));
-            Assert.IsFalse(codec.CanDecode(pyList, typeof(bool)));
+
+            var pyListType = pyList.GetPythonType();
+            Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList<bool>)));
+            Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList<int>)));
+            Assert.IsFalse(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable)));
+            Assert.IsFalse(codec.CanDecode(pyListType, typeof(IEnumerable<int>)));
+            Assert.IsFalse(codec.CanDecode(pyListType, typeof(ICollection<float>)));
+            Assert.IsFalse(codec.CanDecode(pyListType, typeof(bool)));
 
             //we'd have to copy into a list instance to do this, it would not be lossless.
             //lossy converters can be implemented outside of the python.net core library
-            Assert.IsFalse(codec.CanDecode(pyList, typeof(List<int>)));
+            Assert.IsFalse(codec.CanDecode(pyListType, typeof(List<int>)));
 
             //convert to list of int
             IList<int> intList = null;
@@ -137,11 +161,12 @@ public void ListDecoderTest()
             IList<string> stringList = null;
             Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringList); });
             Assert.AreEqual(stringList.Count, 3);
-            Assert.Throws(typeof(PythonException), ()=> { var x = stringList[0]; });
+            Assert.Throws(typeof(PythonException), () => { var x = stringList[0]; });
 
             //can't convert python iterable to list (this will require a copy which isn't lossless)
             var foo = GetPythonIterable();
-            Assert.IsFalse(codec.CanDecode(foo, typeof(IList<int>)));
+            var fooType = foo.GetPythonType();
+            Assert.IsFalse(codec.CanDecode(fooType, typeof(IList<int>)));
         }
 
         [Test]
@@ -189,18 +214,20 @@ public void SequenceDecoderTest()
             //can't convert python iterable to collection (this will require a copy which isn't lossless)
             //python iterables do not satisfy the python sequence protocol
             var foo = GetPythonIterable();
-            Assert.IsFalse(codec.CanDecode(foo, typeof(ICollection<int>)));
+            var fooType = foo.GetPythonType();
+            Assert.IsFalse(codec.CanDecode(fooType, typeof(ICollection<int>)));
 
             //python tuples do satisfy the python sequence protocol
             var pyTuple = new PyObject(Runtime.PyTuple_New(3));
+            var pyTupleType = pyTuple.GetPythonType();
 
             Runtime.PyTuple_SetItem(pyTuple.Handle, 0, items[0].Handle);
             Runtime.PyTuple_SetItem(pyTuple.Handle, 1, items[1].Handle);
             Runtime.PyTuple_SetItem(pyTuple.Handle, 2, items[2].Handle);
 
-            Assert.IsTrue(codec.CanDecode(pyTuple, typeof(ICollection<float>)));
-            Assert.IsTrue(codec.CanDecode(pyTuple, typeof(ICollection<int>)));
-            Assert.IsTrue(codec.CanDecode(pyTuple, typeof(ICollection<string>)));
+            Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection<float>)));
+            Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection<int>)));
+            Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection<string>)));
 
             //convert to collection of int
             ICollection<int> intCollection2 = null;
@@ -226,46 +253,6 @@ public void SequenceDecoderTest()
             Runtime.CheckExceptionOccurred();
 
         }
-    }
-
-    /// <summary>
-    /// "Decodes" only objects of exact type <typeparamref name="T"/>.
-    /// Result is just the raw proxy to the encoder instance itself.
-    /// </summary>
-    class ObjectToEncoderInstanceEncoder<T> : IPyObjectEncoder
-    {
-        public bool CanEncode(Type type) => type == typeof(T);
-        public PyObject TryEncode(object value) => PyObject.FromManagedObject(this);
-    }
-
-    /// <summary>
-    /// Decodes object of specified Python type to the predefined value <see cref="DecodeResult"/>
-    /// </summary>
-    /// <typeparam name="TTarget">Type of the <see cref="DecodeResult"/></typeparam>
-    class DecoderReturningPredefinedValue<TTarget> : IPyObjectDecoder
-    {
-        public PyObject TheOnlySupportedSourceType { get; }
-        public TTarget DecodeResult { get; }
-
-        public DecoderReturningPredefinedValue(PyObject objectType, TTarget decodeResult)
-        {
-            this.TheOnlySupportedSourceType = objectType;
-            this.DecodeResult = decodeResult;
-        }
-
-        public bool CanDecode(PyObject objectType, Type targetType)
-            => objectType.Handle == TheOnlySupportedSourceType.Handle
-               && targetType == typeof(TTarget);
-        public bool TryDecode<T>(PyObject pyObj, out T value)
-        {
-            if (typeof(T) != typeof(TTarget))
-                throw new ArgumentException(nameof(T));
-            value = (T)(object)DecodeResult;
-            return true;
-        }
-    }
-}
-
 
         [Test]
         public void IterableDecoderTest()
@@ -274,11 +261,12 @@ public void IterableDecoderTest()
             var items = new List<PyObject>() { new PyInt(1), new PyInt(2), new PyInt(3) };
 
             var pyList = new PyList(items.ToArray());
-            Assert.IsFalse(codec.CanDecode(pyList, typeof(IList<bool>)));
-            Assert.IsTrue(codec.CanDecode(pyList, typeof(System.Collections.IEnumerable)));
-            Assert.IsTrue(codec.CanDecode(pyList, typeof(IEnumerable<int>)));
-            Assert.IsFalse(codec.CanDecode(pyList, typeof(ICollection<float>)));
-            Assert.IsFalse(codec.CanDecode(pyList, typeof(bool)));
+            var pyListType = pyList.GetPythonType();
+            Assert.IsFalse(codec.CanDecode(pyListType, typeof(IList<bool>)));
+            Assert.IsTrue(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable)));
+            Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable<int>)));
+            Assert.IsFalse(codec.CanDecode(pyListType, typeof(ICollection<float>)));
+            Assert.IsFalse(codec.CanDecode(pyListType, typeof(bool)));
 
             //ensure a PyList can be converted to a plain IEnumerable
             System.Collections.IEnumerable plainEnumerable1 = null;
@@ -287,9 +275,9 @@ public void IterableDecoderTest()
 
             //can convert to any generic ienumerable.  If the type is not assignable from the python element
             //it will lead to an empty iterable when decoding.  TODO - should it throw?
-            Assert.IsTrue(codec.CanDecode(pyList, typeof(IEnumerable<int>)));
-            Assert.IsTrue(codec.CanDecode(pyList, typeof(IEnumerable<double>)));
-            Assert.IsTrue(codec.CanDecode(pyList, typeof(IEnumerable<string>)));
+            Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable<int>)));
+            Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable<double>)));
+            Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable<string>)));
 
             IEnumerable<int> intEnumerable = null;
             Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); });
@@ -320,18 +308,57 @@ public void IterableDecoderTest()
 
             //ensure a python class which implements the iterator protocol can be converter to a plain IEnumerable
             var foo = GetPythonIterable();
+            var fooType = foo.GetPythonType();
             System.Collections.IEnumerable plainEnumerable2 = null;
             Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable2); });
             CollectionAssert.AreEqual(plainEnumerable2, new List<object> { 1, 2, 3 });
 
             //can convert to any generic ienumerable.  If the type is not assignable from the python element
             //it will be an exception during TryDecode
-            Assert.IsTrue(codec.CanDecode(foo, typeof(IEnumerable<int>)));
-            Assert.IsTrue(codec.CanDecode(foo, typeof(IEnumerable<double>)));
-            Assert.IsTrue(codec.CanDecode(foo, typeof(IEnumerable<string>)));
+            Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable<int>)));
+            Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable<double>)));
+            Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable<string>)));
 
             Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); });
             CollectionAssert.AreEqual(intEnumerable, new List<object> { 1, 2, 3 });
         }
     }
+    }
+
+    /// <summary>
+    /// "Decodes" only objects of exact type <typeparamref name="T"/>.
+    /// Result is just the raw proxy to the encoder instance itself.
+    /// </summary>
+    class ObjectToEncoderInstanceEncoder<T> : IPyObjectEncoder
+    {
+        public bool CanEncode(Type type) => type == typeof(T);
+        public PyObject TryEncode(object value) => PyObject.FromManagedObject(this);
+    }
+
+    /// <summary>
+    /// Decodes object of specified Python type to the predefined value <see cref="DecodeResult"/>
+    /// </summary>
+    /// <typeparam name="TTarget">Type of the <see cref="DecodeResult"/></typeparam>
+    class DecoderReturningPredefinedValue<TTarget> : IPyObjectDecoder
+    {
+        public PyObject TheOnlySupportedSourceType { get; }
+        public TTarget DecodeResult { get; }
+
+        public DecoderReturningPredefinedValue(PyObject objectType, TTarget decodeResult)
+        {
+            this.TheOnlySupportedSourceType = objectType;
+            this.DecodeResult = decodeResult;
+        }
+
+        public bool CanDecode(PyObject objectType, Type targetType)
+            => objectType.Handle == TheOnlySupportedSourceType.Handle
+               && targetType == typeof(TTarget);
+        public bool TryDecode<T>(PyObject pyObj, out T value)
+        {
+            if (typeof(T) != typeof(TTarget))
+                throw new ArgumentException(nameof(T));
+            value = (T)(object)DecodeResult;
+            return true;
+        }
+    }
 }
diff --git a/src/runtime/Codecs/IterableDecoder.cs b/src/runtime/Codecs/IterableDecoder.cs
@@ -19,9 +19,7 @@ internal static bool IsIterable(Type targetType)
 
         internal static bool IsIterable(PyObject objectType)
         {
-            //TODO - do I need to decref iterObject?
-            IntPtr iterObject = Runtime.PyObject_GetIter(objectType.Handle);
-            return iterObject != IntPtr.Zero;
+            return objectType.HasAttr("__iter__");
         }
 
         public bool CanDecode(PyObject objectType, Type targetType)

diff --git a/src/runtime/Codecs/ListDecoder.cs b/src/runtime/Codecs/ListDecoder.cs
@@ -18,8 +18,8 @@ private static bool IsList(PyObject objectType)
             //must implement sequence protocol to fully implement list protocol
             if (!SequenceDecoder.IsSequence(objectType)) return false;
 
-            //returns wheter it implements the list protocol
-            return Runtime.PyList_Check(objectType.Handle);
+            //returns wheter the type is a list.
+            return objectType.Handle == Runtime.PyListType;
         }
 
         public bool CanDecode(PyObject objectType, Type targetType)

diff --git a/src/runtime/Codecs/SequenceDecoder.cs b/src/runtime/Codecs/SequenceDecoder.cs
@@ -18,8 +18,11 @@ internal static bool IsSequence(PyObject objectType)
             //must implement iterable protocol to fully implement sequence protocol
             if (!IterableDecoder.IsIterable(objectType)) return false;
 
-            //returns wheter it implements the sequence protocol
-            return Runtime.PySequence_Check(objectType.Handle);
+            //returns wheter it implements the sequence protocol
+            //according to python doc this needs to exclude dict subclasses
+            //but I don't know how to look for that given the objectType
+            //rather than the instance.
+            return objectType.HasAttr("__getitem__");
         }
 
         public bool CanDecode(PyObject objectType, Type targetType)