Skip to content

Commit 624f7e3

Browse files
BadSingletonBadSingleton
committed
Merge remote-tracking branch 'upstream/master' into modernize-import-hook
2 parents 63de923 + 12027ad commit 624f7e3

File tree

10 files changed

+42
-49
lines changed

10 files changed

+42
-49
lines changed

‎AUTHORS.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
- Meinrad Recheis ([@henon](https://github.com/henon))
5656
- Mohamed Koubaa ([@koubaa](https://github.com/koubaa))
5757
- Patrick Stewart ([@patstew](https://github.com/patstew))
58+
- Peter Kese ([@pkese](https://github.com/pkese))
5859
- Raphael Nestler ([@rnestler](https://github.com/rnestler))
5960
- Rickard Holmberg ([@rickardraysearch](https://github.com/rickardraysearch))
6061
- Sam Winstanley ([@swinstanley](https://github.com/swinstanley))

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ One must now either use enum members (e.g. `MyEnum.Option`), or use enum constru
7272
- Exception stacktraces on `PythonException.StackTrace` are now properly formatted
7373
- Providing an invalid type parameter to a generic type or method produces a helpful Python error
7474
- Empty parameter names (as can be generated from F#) do not cause crashes
75+
- Unicode strings with surrogates were truncated when converting from Python
7576

7677
### Removed
7778

‎src/embed_tests/TestCustomMarshal.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ public static void GetManagedStringTwice()
2323
{
2424
const string expected = "FooBar";
2525

26-
IntPtr op = Runtime.Runtime.PyUnicode_FromString(expected);
26+
IntPtr op = Runtime.Runtime.PyString_FromString(expected);
2727
string s1 = Runtime.Runtime.GetManagedString(op);
2828
string s2 = Runtime.Runtime.GetManagedString(op);
2929

‎src/embed_tests/TestPyString.cs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,5 +94,24 @@ public void TestUnicode()
9494
PyObject actual = new PyString(expected);
9595
Assert.AreEqual(expected, actual.ToString());
9696
}
97+
98+
[Test]
99+
public void TestUnicodeSurrogateToString()
100+
{
101+
var expected = "foo\ud83d\udc3c";
102+
var actual = PythonEngine.Eval("'foo\ud83d\udc3c'");
103+
Assert.AreEqual(4, actual.Length());
104+
Assert.AreEqual(expected, actual.ToString());
105+
}
106+
107+
[Test]
108+
public void TestUnicodeSurrogate()
109+
{
110+
const string expected = "foo\ud83d\udc3c"; // "foo🐼"
111+
PyObject actual = new PyString(expected);
112+
// python treats "foo🐼" as 4 characters, dotnet as 5
113+
Assert.AreEqual(4, actual.Length());
114+
Assert.AreEqual(expected, actual.ToString());
115+
}
97116
}
98117
}

‎src/embed_tests/TestRuntime.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ public static void Py_IsInitializedValue()
3636
public static void RefCountTest()
3737
{
3838
Runtime.Runtime.Py_Initialize();
39-
IntPtr op = Runtime.Runtime.PyUnicode_FromString("FooBar");
39+
IntPtr op = Runtime.Runtime.PyString_FromString("FooBar");
4040

4141
// New object RefCount should be one
4242
Assert.AreEqual(1, Runtime.Runtime.Refcount(op));

‎src/runtime/converter.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ internal static IntPtr ToPython(object value, Type type)
229229
return CLRObject.GetInstHandle(value, type);
230230

231231
case TypeCode.String:
232-
return Runtime.PyUnicode_FromString((string)value);
232+
return Runtime.PyString_FromString((string)value);
233233

234234
case TypeCode.Int32:
235235
return Runtime.PyInt_FromInt32((int)value);

‎src/runtime/exceptions.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ internal static Exception ToException(BorrowedReference ob)
5050
{
5151
message = String.Format("{0}()", name);
5252
}
53-
return Runtime.PyUnicode_FromString(message);
53+
return Runtime.PyString_FromString(message);
5454
}
5555

5656
/// <summary>
@@ -75,7 +75,7 @@ internal static Exception ToException(BorrowedReference ob)
7575
{
7676
message = message.Substring(fullTypeName.Length);
7777
}
78-
return Runtime.PyUnicode_FromString(message);
78+
return Runtime.PyString_FromString(message);
7979
}
8080
}
8181

@@ -153,7 +153,7 @@ internal static void SetArgsAndCause(BorrowedReference ob, Exception e)
153153
if (!string.IsNullOrEmpty(e.Message))
154154
{
155155
args = Runtime.PyTuple_New(1);
156-
IntPtr msg = Runtime.PyUnicode_FromString(e.Message);
156+
IntPtr msg = Runtime.PyString_FromString(e.Message);
157157
Runtime.PyTuple_SetItem(args, 0, msg);
158158
}
159159
else

‎src/runtime/pystring.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public PyString(PyObject o) : base(FromObject(o))
5151

5252
private static IntPtr FromString(string s)
5353
{
54-
IntPtr val = Runtime.PyUnicode_FromUnicode(s, s.Length);
54+
IntPtr val = Runtime.PyString_FromString(s);
5555
PythonException.ThrowIfIsNull(val);
5656
return val;
5757
}

‎src/runtime/runtime.cs‎

Lines changed: 13 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,7 @@ private static void InitPyMembers()
230230
() => PyStringType = IntPtr.Zero);
231231
XDecref(op);
232232

233-
op = PyUnicode_FromString("unicode");
233+
op = PyString_FromString("unicode");
234234
SetPyMemberTypeOf(ref PyUnicodeType, op,
235235
() => PyUnicodeType = IntPtr.Zero);
236236
XDecref(op);
@@ -1529,7 +1529,12 @@ internal static bool PyString_Check(IntPtr ob)
15291529
internal static IntPtr PyString_FromString(string value)
15301530
{
15311531
fixed(char* ptr = value)
1532-
return PyUnicode_FromKindAndData(2, (IntPtr)ptr, value.Length);
1532+
return Delegates.PyUnicode_DecodeUTF16(
1533+
(IntPtr)ptr,
1534+
value.Length * sizeof(Char),
1535+
IntPtr.Zero,
1536+
IntPtr.Zero
1537+
).DangerousMoveToPointerOrNull();
15331538
}
15341539

15351540

@@ -1555,16 +1560,6 @@ internal static long PyBytes_Size(IntPtr op)
15551560

15561561
private static IntPtr _PyBytes_Size(IntPtr op) => Delegates._PyBytes_Size(op);
15571562

1558-
1559-
internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size)
1560-
{
1561-
return PyUnicode_FromStringAndSize(value, new IntPtr(size));
1562-
}
1563-
1564-
1565-
private static IntPtr PyUnicode_FromStringAndSize(IntPtr value, IntPtr size) => Delegates.PyUnicode_FromStringAndSize(value, size);
1566-
1567-
15681563
internal static IntPtr PyUnicode_AsUTF8(IntPtr unicode) => Delegates.PyUnicode_AsUTF8(unicode);
15691564

15701565
internal static bool PyUnicode_Check(IntPtr ob)
@@ -1578,22 +1573,6 @@ internal static bool PyUnicode_Check(IntPtr ob)
15781573

15791574
internal static IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err) => Delegates.PyUnicode_FromEncodedObject(ob, enc, err);
15801575

1581-
internal static IntPtr PyUnicode_FromKindAndData(int kind, IntPtr s, long size)
1582-
{
1583-
return PyUnicode_FromKindAndData(kind, s, new IntPtr(size));
1584-
}
1585-
1586-
1587-
private static IntPtr PyUnicode_FromKindAndData(int kind, IntPtr s, IntPtr size)
1588-
=> Delegates.PyUnicode_FromKindAndData(kind, s, size);
1589-
1590-
internal static IntPtr PyUnicode_FromUnicode(string s, long size)
1591-
{
1592-
fixed(char* ptr = s)
1593-
return PyUnicode_FromKindAndData(2, (IntPtr)ptr, size);
1594-
}
1595-
1596-
15971576
internal static int PyUnicode_GetMax() => Delegates.PyUnicode_GetMax();
15981577

15991578
internal static long PyUnicode_GetSize(IntPtr ob)
@@ -1612,12 +1591,6 @@ internal static long PyUnicode_GetSize(IntPtr ob)
16121591

16131592
internal static IntPtr PyUnicode_FromOrdinal(int c) => Delegates.PyUnicode_FromOrdinal(c);
16141593

1615-
internal static IntPtr PyUnicode_FromString(string s)
1616-
{
1617-
return PyUnicode_FromUnicode(s, s.Length);
1618-
}
1619-
1620-
16211594
internal static IntPtr PyUnicode_InternFromString(string s)
16221595
{
16231596
using var ptr = new StrPtr(s, Encoding.UTF8);
@@ -1648,11 +1621,12 @@ internal static string GetManagedString(IntPtr op)
16481621
if (type == PyUnicodeType)
16491622
{
16501623
using var p = PyUnicode_AsUTF16String(new BorrowedReference(op));
1651-
int length = (int)PyUnicode_GetSize(op);
1652-
char* codePoints = (char*)PyBytes_AsString(p.DangerousGetAddress());
1624+
var bytesPtr = p.DangerousGetAddress();
1625+
int bytesLength = (int)Runtime.PyBytes_Size(bytesPtr);
1626+
char* codePoints = (char*)PyBytes_AsString(bytesPtr);
16531627
return new string(codePoints,
16541628
startIndex: 1, // skip BOM
1655-
length: length);
1629+
length: bytesLength/2-1); // utf16 - BOM
16561630
}
16571631

16581632
return null;
@@ -2465,11 +2439,10 @@ static Delegates()
24652439
PyBytes_AsString = (delegate* unmanaged[Cdecl]<BorrowedReference, IntPtr>)GetFunctionByName(nameof(PyBytes_AsString), GetUnmanagedDll(_PythonDll));
24662440
PyBytes_FromString = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyBytes_FromString), GetUnmanagedDll(_PythonDll));
24672441
_PyBytes_Size = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName("PyBytes_Size", GetUnmanagedDll(_PythonDll));
2468-
PyUnicode_FromStringAndSize = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr, IntPtr>)GetFunctionByName(nameof(PyUnicode_FromStringAndSize), GetUnmanagedDll(_PythonDll));
24692442
PyUnicode_AsUTF8 = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyUnicode_AsUTF8), GetUnmanagedDll(_PythonDll));
24702443
PyUnicode_FromObject = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyUnicode_FromObject), GetUnmanagedDll(_PythonDll));
2444+
PyUnicode_DecodeUTF16 = (delegate* unmanaged[Cdecl]<IntPtr, nint, IntPtr, IntPtr, NewReference>)GetFunctionByName(nameof(PyUnicode_DecodeUTF16), GetUnmanagedDll(_PythonDll));
24712445
PyUnicode_FromEncodedObject = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr, IntPtr, IntPtr>)GetFunctionByName(nameof(PyUnicode_FromEncodedObject), GetUnmanagedDll(_PythonDll));
2472-
PyUnicode_FromKindAndData = (delegate* unmanaged[Cdecl]<int, IntPtr, IntPtr, IntPtr>)GetFunctionByName(nameof(PyUnicode_FromKindAndData), GetUnmanagedDll(_PythonDll));
24732446
PyUnicode_GetMax = (delegate* unmanaged[Cdecl]<int>)GetFunctionByName(nameof(PyUnicode_GetMax), GetUnmanagedDll(_PythonDll));
24742447
_PyUnicode_GetSize = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName("PyUnicode_GetSize", GetUnmanagedDll(_PythonDll));
24752448
PyUnicode_AsUnicode = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyUnicode_AsUnicode), GetUnmanagedDll(_PythonDll));
@@ -2762,11 +2735,10 @@ static Delegates()
27622735
internal static delegate* unmanaged[Cdecl]<BorrowedReference, IntPtr> PyBytes_AsString { get; }
27632736
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> PyBytes_FromString { get; }
27642737
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> _PyBytes_Size { get; }
2765-
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr, IntPtr> PyUnicode_FromStringAndSize { get; }
27662738
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> PyUnicode_AsUTF8 { get; }
27672739
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> PyUnicode_FromObject { get; }
27682740
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr, IntPtr, IntPtr> PyUnicode_FromEncodedObject { get; }
2769-
internal static delegate* unmanaged[Cdecl]<int, IntPtr, IntPtr, IntPtr> PyUnicode_FromKindAndData { get; }
2741+
internal static delegate* unmanaged[Cdecl]<IntPtr, nint, IntPtr, IntPtr, NewReference> PyUnicode_DecodeUTF16 { get; }
27702742
internal static delegate* unmanaged[Cdecl]<int> PyUnicode_GetMax { get; }
27712743
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> _PyUnicode_GetSize { get; }
27722744
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> PyUnicode_AsUnicode { get; }

‎src/runtime/typemanager.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -580,7 +580,7 @@ internal static IntPtr AllocateTypeObject(string name, IntPtr metatype)
580580
// Cheat a little: we'll set tp_name to the internal char * of
581581
// the Python version of the type name - otherwise we'd have to
582582
// allocate the tp_name and would have no way to free it.
583-
IntPtr temp = Runtime.PyUnicode_FromString(name);
583+
IntPtr temp = Runtime.PyString_FromString(name);
584584
IntPtr raw = Runtime.PyUnicode_AsUTF8(temp);
585585
Marshal.WriteIntPtr(type, TypeOffset.tp_name, raw);
586586
Marshal.WriteIntPtr(type, TypeOffset.name, temp);

0 commit comments

Comments
 (0)