always explicitly specify the way strings are marshaled

pythonnet · lostmsu · Jan 28, 2021 · Nov 30, 2020 · Dec 2, 2020 · Dec 2, 2020
commit 2498d474a3ae8e84d505680de5a467be5b5c4e9f
diff --git a/src/embed_tests/References.cs b/src/embed_tests/References.cs
@@ -23,7 +23,7 @@ public void Dispose()
         public void MoveToPyObject_SetsNull()
         {
             var dict = new PyDict();
-            NewReference reference = Runtime.PyDict_Items(dict.Handle);
+            NewReference reference = Runtime.PyDict_Items(dict.Reference);
             try
             {
                 Assert.IsFalse(reference.IsNull());
@@ -41,7 +41,7 @@ public void MoveToPyObject_SetsNull()
         public void CanBorrowFromNewReference()
         {
             var dict = new PyDict();
-            NewReference reference = Runtime.PyDict_Items(dict.Handle);
+            NewReference reference = Runtime.PyDict_Items(dict.Reference);
             try
             {
                 PythonException.ThrowIfIsNotZero(Runtime.PyList_Reverse(reference));

diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs
@@ -332,7 +332,7 @@ static void RunAssemblyAndUnload(string domainName)
             // assembly (and Python .NET) to reside
             var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
 
-            theProxy.Call("InitPython", ShutdownMode.Soft);
+            theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Soft, PyRuntime.PythonDLL);
             // From now on use the Proxy to call into the new assembly
             theProxy.RunPython();
 
@@ -400,7 +400,7 @@ static void RunDomainReloadSteps<T1, T2>() where T1 : CrossCaller where T2 : Cro
                 try
                 {
                     var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
-                    theProxy.Call("InitPython", ShutdownMode.Reload);
+                    theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL);
 
                     var caller = CreateInstanceInstanceAndUnwrap<T1>(domain);
                     arg = caller.Execute(arg);
@@ -418,7 +418,7 @@ static void RunDomainReloadSteps<T1, T2>() where T1 : CrossCaller where T2 : Cro
                 try
                 {
                     var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
-                    theProxy.Call("InitPython", ShutdownMode.Reload);
+                    theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL);
 
                     var caller = CreateInstanceInstanceAndUnwrap<T2>(domain);
                     caller.Execute(arg);
@@ -478,8 +478,9 @@ public static void RunPython()
 
         private static IntPtr _state;
 
-        public static void InitPython(ShutdownMode mode)
+        public static void InitPython(ShutdownMode mode, string dllName)
         {
+            PyRuntime.PythonDLL = dllName;
             PythonEngine.Initialize(mode: mode);
             _state = PythonEngine.BeginAllowThreads();
         }

diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs
@@ -96,13 +96,15 @@ public static void PyCheck_Iter_PyObject_IsIterable_ThreadingLock_Test()
             // TypeFlags.HaveIter set in Python 2. This tests a different code path in PyObject_IsIterable and PyIter_Check.
             var threading = Runtime.Runtime.PyImport_ImportModule("threading");
             Exceptions.ErrorCheck(threading);
-            var threadingDict = Runtime.Runtime.PyModule_GetDict(threading);
+            var threadingDict = Runtime.Runtime.PyModule_GetDict(new BorrowedReference(threading));
             Exceptions.ErrorCheck(threadingDict);
             var lockType = Runtime.Runtime.PyDict_GetItemString(threadingDict, "Lock");
-            if (lockType == IntPtr.Zero)
+            if (lockType.IsNull)
                 throw new KeyNotFoundException("class 'Lock' was not found in 'threading'");
 
-            var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType, Runtime.Runtime.PyTuple_New(0));
+            var args = Runtime.Runtime.PyTuple_New(0);
+            var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType.DangerousGetAddress(), args);
+            Runtime.Runtime.XDecref(args);
             Exceptions.ErrorCheck(lockInstance);
 
             Assert.IsFalse(Runtime.Runtime.PyObject_IsIterable(lockInstance));

diff --git a/src/embed_tests/TestTypeManager.cs b/src/embed_tests/TestTypeManager.cs
diff --git a/src/runtime/CustomMarshaler.cs b/src/runtime/CustomMarshaler.cs
@@ -41,8 +41,9 @@ public int GetNativeDataSize()
     /// </summary>
     internal class UcsMarshaler : MarshalerBase
     {
+        internal static readonly int _UCS = Runtime.PyUnicode_GetMax() <= 0xFFFF ? 2 : 4;
+        internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32;
         private static readonly MarshalerBase Instance = new UcsMarshaler();
-        private static readonly Encoding PyEncoding = Runtime.PyEncoding;
 
         public override IntPtr MarshalManagedToNative(object managedObj)
         {
@@ -91,13 +92,13 @@ public static int GetUnicodeByteLength(IntPtr p)
             var len = 0;
             while (true)
             {
-                int c = Runtime._UCS == 2
+                int c = _UCS == 2
                     ? Marshal.ReadInt16(p, len * 2)
                     : Marshal.ReadInt32(p, len * 4);
 
                 if (c == 0)
                 {
-                    return len * Runtime._UCS;
+                    return len * _UCS;
                 }
                 checked
                 {
@@ -147,7 +148,7 @@ public static string PtrToPy3UnicodePy2String(IntPtr p)
     internal class StrArrayMarshaler : MarshalerBase
     {
         private static readonly MarshalerBase Instance = new StrArrayMarshaler();
-        private static readonly Encoding PyEncoding = Runtime.PyEncoding;
+        private static readonly Encoding PyEncoding = UcsMarshaler.PyEncoding;
 
         public override IntPtr MarshalManagedToNative(object managedObj)
         {
@@ -159,7 +160,7 @@ public override IntPtr MarshalManagedToNative(object managedObj)
             }
 
             int totalStrLength = argv.Sum(arg => arg.Length + 1);
-            int memSize = argv.Length * IntPtr.Size + totalStrLength * Runtime._UCS;
+            int memSize = argv.Length * IntPtr.Size + totalStrLength * UcsMarshaler._UCS;
 
             IntPtr mem = Marshal.AllocHGlobal(memSize);
             try

diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj
@@ -31,5 +31,9 @@
   <ItemGroup>
     <PackageReference Include="System.Security.Permissions" Version="4.4.0" />
     <PackageReference Include="System.Reflection.Emit" Version="4.3.0" />
+    <PackageReference Include="Lost.NonCopyableAnalyzer" Version="0.7.0-m04">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
   </ItemGroup>
 </Project>
diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs
@@ -194,14 +194,16 @@ internal static void SetArgsAndCause(IntPtr ob)
         /// Shortcut for (pointer == NULL) -&gt; throw PythonException
         /// </summary>
         /// <param name="pointer">Pointer to a Python object</param>
-        internal static void ErrorCheck(IntPtr pointer)
+        internal static void ErrorCheck(BorrowedReference pointer)
         {
-            if (pointer == IntPtr.Zero)
+            if (pointer.IsNull)
             {
                 throw new PythonException();
             }
         }
 
+        internal static void ErrorCheck(IntPtr pointer) => ErrorCheck(new BorrowedReference(pointer));
+
         /// <summary>
         /// Shortcut for (pointer == NULL or ErrorOccurred()) -&gt; throw PythonException
         /// </summary>

diff --git a/src/runtime/native/PyCompilerFlags.cs b/src/runtime/native/PyCompilerFlags.cs
@@ -0,0 +1,13 @@
+using System;
+
+namespace Python.Runtime.Native
+{
+    [Flags]
+    enum PyCompilerFlags
+    {
+        SOURCE_IS_UTF8 = 0x0100,
+        DONT_IMPLY_DEDENT = 0x0200,
+        ONLY_AST = 0x0400,
+        IGNORE_COOKIE = 0x0800,
+    }
+}
diff --git a/src/runtime/native/StrPtr.cs b/src/runtime/native/StrPtr.cs
@@ -0,0 +1,73 @@
+#nullable enable
+using System;
+using System.Runtime.InteropServices;
+using System.Text;
+
+namespace Python.Runtime.Native
+{
+    [StructLayout(LayoutKind.Sequential)]
+    struct StrPtr : IDisposable
+    {
+        public IntPtr RawPointer { get; set; }
+        unsafe byte* Bytes => (byte*)this.RawPointer;
+
+        public unsafe StrPtr(string value, Encoding encoding)
+        {
+            if (value is null) throw new ArgumentNullException(nameof(value));
+            if (encoding is null) throw new ArgumentNullException(nameof(encoding));
+
+            var bytes = encoding.GetBytes(value);
+            this.RawPointer = Marshal.AllocHGlobal(checked(bytes.Length + 1));
+            try
+            {
+                Marshal.Copy(bytes, 0, this.RawPointer, bytes.Length);
+                this.Bytes[bytes.Length] = 0;
+            }
+            catch
+            {
+                this.Dispose();
+                throw;
+            }
+        }
+
+        public unsafe string? ToString(Encoding encoding)
+        {
+            if (encoding is null) throw new ArgumentNullException(nameof(encoding));
+            if (this.RawPointer == IntPtr.Zero) return null;
+
+            return encoding.GetString((byte*)this.RawPointer, byteCount: checked((int)this.ByteCount));
+        }
+
+        public unsafe nuint ByteCount
+        {
+            get
+            {
+                if (this.RawPointer == IntPtr.Zero) throw new NullReferenceException();
+
+                nuint zeroIndex = 0;
+                while (this.Bytes[zeroIndex] != 0)
+                {
+                    zeroIndex++;
+                }
+                return zeroIndex;
+            }
+        }
+
+        public void Dispose()
+        {
+            if (this.RawPointer == IntPtr.Zero)
+                return;
+
+            Marshal.FreeHGlobal(this.RawPointer);
+            this.RawPointer = IntPtr.Zero;
+        }
+
+        internal static Encoding GetEncodingByPythonName(string pyEncodingName)
+        {
+            // https://stackoverflow.com/a/7798749/231238
+            if (pyEncodingName == "mbcs") return Encoding.Default;
+
+            return Encoding.GetEncoding(pyEncodingName);
+        }
+    }
+}
diff --git a/src/runtime/pyint.cs b/src/runtime/pyint.cs
@@ -149,7 +149,7 @@ public PyInt(sbyte value) : this((int)value)
 
         private static IntPtr FromString(string value)
         {
-            IntPtr val = Runtime.PyInt_FromString(value, IntPtr.Zero, 0);
+            IntPtr val = Runtime.PyLong_FromString(value, IntPtr.Zero, 0);
             PythonException.ThrowIfIsNull(val);
             return val;
         }

diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs
@@ -257,7 +257,7 @@ public bool HasAttr(string name)
         {
             if (name == null) throw new ArgumentNullException(nameof(name));
 
-            return Runtime.PyObject_HasAttrString(obj, name) != 0;
+            return Runtime.PyObject_HasAttrString(Reference, name) != 0;
         }
-Original file line number
+Diff line change
@@ Expand Up / @@ -257,7 +257,7 @@ public bool HasAttr(string name) @@
             {
                 if (name == null) throw new ArgumentNullException(nameof(name));
-                return Runtime.PyObject_HasAttrString(obj, name) != 0;
+                return Runtime.PyObject_HasAttrString(Reference, name) != 0;
             }
@@ Expand Down @@