🌐 AI搜索 & 代理 主页
Skip to content

Add a failing test for Unicode conversion #1467

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lostmsu merged 8 commits into from
Jun 11, 2021
Merged

Add a failing test for Unicode conversion #1467

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
da051a9
Add a failing test for Unicode conversion
pkese Jun 8, 2021
1b6c6c0
Fix Python -> .Net unicode string conversion
pkese Jun 8, 2021
4674b5c
Add separate test for the initial problem (passing); disable failing …
pkese Jun 8, 2021
1629116
Use exclusively PyUnicode_DecodeUTF16 for .NET->Python string convers…
filmor Jun 9, 2021
df5ebc2
Activate UnicodeSurrogate test
filmor Jun 9, 2021
d084b2e
Apply suggested changes
pkese Jun 9, 2021
f061d28
Revert stuff that I don't understand
pkese Jun 9, 2021
49ccc1e
Apply code review suggestions
filmor Jun 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Use exclusively PyUnicode_DecodeUTF16 for .NET->Python string convers… 
…ions
  • Loading branch information
@filmor
filmor committed Jun 9, 2021
commit 1629116c08ac90759b1c4283b156078c394bd710
2 changes: 1 addition & 1 deletion src/embed_tests/TestCustomMarshal.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public static void GetManagedStringTwice()
{
const string expected = "FooBar";

IntPtr op = Runtime.Runtime.PyUnicode_FromString(expected);
IntPtr op = Runtime.Runtime.PyString_FromString(expected);
string s1 = Runtime.Runtime.GetManagedString(op);
string s2 = Runtime.Runtime.GetManagedString(op);

Expand Down
2 changes: 1 addition & 1 deletion src/embed_tests/TestRuntime.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public static void Py_IsInitializedValue()
public static void RefCountTest()
{
Runtime.Runtime.Py_Initialize();
IntPtr op = Runtime.Runtime.PyUnicode_FromString("FooBar");
IntPtr op = Runtime.Runtime.PyString_FromString("FooBar");

// New object RefCount should be one
Assert.AreEqual(1, Runtime.Runtime.Refcount(op));
Expand Down
2 changes: 1 addition & 1 deletion src/runtime/converter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ internal static IntPtr ToPython(object value, Type type)
return CLRObject.GetInstHandle(value, type);

case TypeCode.String:
return Runtime.PyUnicode_FromString((string)value);
return Runtime.PyString_FromString((string)value);

case TypeCode.Int32:
return Runtime.PyInt_FromInt32((int)value);
Expand Down
6 changes: 3 additions & 3 deletions src/runtime/exceptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ internal static Exception ToException(BorrowedReference ob)
{
message = String.Format("{0}()", name);
}
return Runtime.PyUnicode_FromString(message);
return Runtime.PyString_FromString(message);
}

/// <summary>
Expand All @@ -75,7 +75,7 @@ internal static Exception ToException(BorrowedReference ob)
{
message = message.Substring(fullTypeName.Length);
}
return Runtime.PyUnicode_FromString(message);
return Runtime.PyString_FromString(message);
}
}

Expand Down Expand Up @@ -153,7 +153,7 @@ internal static void SetArgsAndCause(BorrowedReference ob, Exception e)
if (!string.IsNullOrEmpty(e.Message))
{
args = Runtime.PyTuple_New(1);
IntPtr msg = Runtime.PyUnicode_FromString(e.Message);
IntPtr msg = Runtime.PyString_FromString(e.Message);
Runtime.PyTuple_SetItem(args, 0, msg);
}
else
Expand Down
2 changes: 1 addition & 1 deletion src/runtime/pystring.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public PyString(PyObject o) : base(FromObject(o))

private static IntPtr FromString(string s)
{
IntPtr val = Runtime.PyUnicode_FromUnicode(s, s.Length);
IntPtr val = Runtime.PyString_FromString(s);
PythonException.ThrowIfIsNull(val);
return val;
}
Expand Down
47 changes: 9 additions & 38 deletions src/runtime/runtime.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,7 @@ private static void InitPyMembers()
() => PyStringType = IntPtr.Zero);
XDecref(op);

op = PyUnicode_FromString("unicode");
op = PyString_FromString("unicode");
SetPyMemberTypeOf(ref PyUnicodeType, op,
() => PyUnicodeType = IntPtr.Zero);
XDecref(op);
Expand Down Expand Up @@ -1527,7 +1527,12 @@ internal static bool PyString_Check(IntPtr ob)
internal static IntPtr PyString_FromString(string value)
{
fixed(char* ptr = value)
return PyUnicode_FromKindAndData(2, (IntPtr)ptr, value.Length);
return Delegates.PyUnicode_DecodeUTF16(
(IntPtr)ptr,
new IntPtr(value.Length * sizeof(Char)),
IntPtr.Zero,
IntPtr.Zero
).DangerousGetAddress();
}


Expand All @@ -1553,16 +1558,6 @@ internal static long PyBytes_Size(IntPtr op)

private static IntPtr _PyBytes_Size(IntPtr op) => Delegates._PyBytes_Size(op);


internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size)
{
return PyUnicode_FromStringAndSize(value, new IntPtr(size));
}


private static IntPtr PyUnicode_FromStringAndSize(IntPtr value, IntPtr size) => Delegates.PyUnicode_FromStringAndSize(value, size);


internal static IntPtr PyUnicode_AsUTF8(IntPtr unicode) => Delegates.PyUnicode_AsUTF8(unicode);

internal static bool PyUnicode_Check(IntPtr ob)
Expand All @@ -1576,22 +1571,6 @@ internal static bool PyUnicode_Check(IntPtr ob)

internal static IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err) => Delegates.PyUnicode_FromEncodedObject(ob, enc, err);

internal static IntPtr PyUnicode_FromKindAndData(int kind, IntPtr s, long size)
{
return PyUnicode_FromKindAndData(kind, s, new IntPtr(size));
}


private static IntPtr PyUnicode_FromKindAndData(int kind, IntPtr s, IntPtr size)
=> Delegates.PyUnicode_FromKindAndData(kind, s, size);

internal static IntPtr PyUnicode_FromUnicode(string s, long size)
{
fixed(char* ptr = s)
return PyUnicode_FromKindAndData(2, (IntPtr)ptr, size);
}


internal static int PyUnicode_GetMax() => Delegates.PyUnicode_GetMax();

internal static long PyUnicode_GetSize(IntPtr ob)
Expand All @@ -1610,12 +1589,6 @@ internal static long PyUnicode_GetSize(IntPtr ob)

internal static IntPtr PyUnicode_FromOrdinal(int c) => Delegates.PyUnicode_FromOrdinal(c);

internal static IntPtr PyUnicode_FromString(string s)
{
return PyUnicode_FromUnicode(s, s.Length);
}


internal static IntPtr PyUnicode_InternFromString(string s)
{
using var ptr = new StrPtr(s, Encoding.UTF8);
Expand Down Expand Up @@ -2443,11 +2416,10 @@ static Delegates()
PyBytes_AsString = (delegate* unmanaged[Cdecl]<BorrowedReference, IntPtr>)GetFunctionByName(nameof(PyBytes_AsString), GetUnmanagedDll(_PythonDll));
PyBytes_FromString = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyBytes_FromString), GetUnmanagedDll(_PythonDll));
_PyBytes_Size = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName("PyBytes_Size", GetUnmanagedDll(_PythonDll));
PyUnicode_FromStringAndSize = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr, IntPtr>)GetFunctionByName(nameof(PyUnicode_FromStringAndSize), GetUnmanagedDll(_PythonDll));
PyUnicode_AsUTF8 = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyUnicode_AsUTF8), GetUnmanagedDll(_PythonDll));
PyUnicode_FromObject = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyUnicode_FromObject), GetUnmanagedDll(_PythonDll));
PyUnicode_DecodeUTF16 = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr, IntPtr, IntPtr, NewReference>)GetFunctionByName(nameof(PyUnicode_DecodeUTF16), GetUnmanagedDll(_PythonDll));
PyUnicode_FromEncodedObject = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr, IntPtr, IntPtr>)GetFunctionByName(nameof(PyUnicode_FromEncodedObject), GetUnmanagedDll(_PythonDll));
PyUnicode_FromKindAndData = (delegate* unmanaged[Cdecl]<int, IntPtr, IntPtr, IntPtr>)GetFunctionByName(nameof(PyUnicode_FromKindAndData), GetUnmanagedDll(_PythonDll));
PyUnicode_GetMax = (delegate* unmanaged[Cdecl]<int>)GetFunctionByName(nameof(PyUnicode_GetMax), GetUnmanagedDll(_PythonDll));
_PyUnicode_GetSize = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName("PyUnicode_GetSize", GetUnmanagedDll(_PythonDll));
PyUnicode_AsUnicode = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyUnicode_AsUnicode), GetUnmanagedDll(_PythonDll));
Expand Down Expand Up @@ -2739,11 +2711,10 @@ static Delegates()
internal static delegate* unmanaged[Cdecl]<BorrowedReference, IntPtr> PyBytes_AsString { get; }
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> PyBytes_FromString { get; }
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> _PyBytes_Size { get; }
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr, IntPtr> PyUnicode_FromStringAndSize { get; }
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> PyUnicode_AsUTF8 { get; }
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> PyUnicode_FromObject { get; }
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr, IntPtr, IntPtr> PyUnicode_FromEncodedObject { get; }
internal static delegate* unmanaged[Cdecl]<int, IntPtr, IntPtr, IntPtr> PyUnicode_FromKindAndData { get; }
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr, IntPtr, IntPtr, NewReference> PyUnicode_DecodeUTF16 { get; }
internal static delegate* unmanaged[Cdecl]<int> PyUnicode_GetMax { get; }
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> _PyUnicode_GetSize { get; }
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> PyUnicode_AsUnicode { get; }
Expand Down
2 changes: 1 addition & 1 deletion src/runtime/typemanager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -580,7 +580,7 @@ internal static IntPtr AllocateTypeObject(string name, IntPtr metatype)
// Cheat a little: we'll set tp_name to the internal char * of
// the Python version of the type name - otherwise we'd have to
// allocate the tp_name and would have no way to free it.
IntPtr temp = Runtime.PyUnicode_FromString(name);
IntPtr temp = Runtime.PyString_FromString(name);
IntPtr raw = Runtime.PyUnicode_AsUTF8(temp);
Marshal.WriteIntPtr(type, TypeOffset.tp_name, raw);
Marshal.WriteIntPtr(type, TypeOffset.name, temp);
Expand Down