Simplify UTF8 StrPtr usage (#2374)

filmor · web-flow · commit f82aeea80cd4 · 2024-06-09T01:32:00.000+02:00
* Use non-BOM encodings

* Copy potential BOM to the output of PyString_FromString

The documentation of the used `PyUnicode_DecodeUTF16` states that not
passing `*byteorder` or passing a 0 results in the first two bytes, if
they are the BOM (U+FEFF, zero-width no-break space), to be interpreted
and skipped, which is incorrect when we convert a known "non BOM"
string, which all strings from C# are.

* Default to UTF8 for StrPtr
diff --git a/src/embed_tests/TestPyType.cs b/src/embed_tests/TestPyType.cs
@@ -28,7 +28,8 @@ public void CanCreateHeapType()
             const string name = "nÁmæ";
             const string docStr = "dÁcæ";
 
-            using var doc = new StrPtr(docStr, Encodings.UTF8);
+            using var doc = new StrPtr(docStr);
+
             var spec = new TypeSpec(
                 name: name,
                 basicSize: Util.ReadInt32(Runtime.Runtime.PyBaseObjectType, TypeOffset.tp_basicsize),
diff --git a/src/runtime/Native/NativeTypeSpec.cs b/src/runtime/Native/NativeTypeSpec.cs
@@ -17,7 +17,7 @@ public NativeTypeSpec(TypeSpec spec)
         {
             if (spec is null) throw new ArgumentNullException(nameof(spec));
 
-            this.Name = new StrPtr(spec.Name, Encodings.UTF8);
+            this.Name = new StrPtr(spec.Name);
             this.BasicSize = spec.BasicSize;
             this.ItemSize = spec.ItemSize;
             this.Flags = (int)spec.Flags;
diff --git a/src/runtime/Native/StrPtr.cs b/src/runtime/Native/StrPtr.cs
@@ -10,6 +10,8 @@ struct StrPtr : IDisposable
         public IntPtr RawPointer { get; set; }
         unsafe byte* Bytes => (byte*)this.RawPointer;
 
+        public unsafe StrPtr(string value) : this(value, Encodings.UTF8) {}
+
         public unsafe StrPtr(string value, Encoding encoding)
         {
             if (value is null) throw new ArgumentNullException(nameof(value));
diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs
@@ -795,13 +795,13 @@ public static int Py_Main(int argc, string[] argv)
 
         internal static int PyRun_SimpleString(string code)
         {
-            using var codePtr = new StrPtr(code, Encodings.UTF8);
+            using var codePtr = new StrPtr(code);
             return Delegates.PyRun_SimpleStringFlags(codePtr, Utf8String);
         }
 
         internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedReference globals, BorrowedReference locals)
         {
-            using var codePtr = new StrPtr(code, Encodings.UTF8);
+            using var codePtr = new StrPtr(code);
             return Delegates.PyRun_StringFlags(codePtr, st, globals, locals, Utf8String);
         }
 
@@ -813,14 +813,15 @@ internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedR
         /// </summary>
         internal static NewReference Py_CompileString(string str, string file, int start)
         {
-            using var strPtr = new StrPtr(str, Encodings.UTF8);
+            using var strPtr = new StrPtr(str);
+
             using var fileObj = new PyString(file);
             return Delegates.Py_CompileStringObject(strPtr, fileObj, start, Utf8String, -1);
         }
 
         internal static NewReference PyImport_ExecCodeModule(string name, BorrowedReference code)
         {
-            using var namePtr = new StrPtr(name, Encodings.UTF8);
+            using var namePtr = new StrPtr(name);
             return Delegates.PyImport_ExecCodeModule(namePtr, code);
         }
 
@@ -867,13 +868,13 @@ internal static bool PyObject_IsIterable(BorrowedReference ob)
 
         internal static int PyObject_HasAttrString(BorrowedReference pointer, string name)
         {
-            using var namePtr = new StrPtr(name, Encodings.UTF8);
+            using var namePtr = new StrPtr(name);
             return Delegates.PyObject_HasAttrString(pointer, namePtr);
         }
 
         internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, string name)
         {
-            using var namePtr = new StrPtr(name, Encodings.UTF8);
+            using var namePtr = new StrPtr(name);
             return Delegates.PyObject_GetAttrString(pointer, namePtr);
         }
 
@@ -884,12 +885,12 @@ internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, S
         internal static int PyObject_DelAttr(BorrowedReference @object, BorrowedReference name) => Delegates.PyObject_SetAttr(@object, name, null);
         internal static int PyObject_DelAttrString(BorrowedReference @object, string name)
         {
-            using var namePtr = new StrPtr(name, Encodings.UTF8);
+            using var namePtr = new StrPtr(name);
             return Delegates.PyObject_SetAttrString(@object, namePtr, null);
         }
         internal static int PyObject_SetAttrString(BorrowedReference @object, string name, BorrowedReference value)
         {
-            using var namePtr = new StrPtr(name, Encodings.UTF8);
+            using var namePtr = new StrPtr(name);
             return Delegates.PyObject_SetAttrString(@object, namePtr, value);
         }
 
@@ -1071,7 +1072,7 @@ internal static bool PyBool_CheckExact(BorrowedReference ob)
 
         internal static NewReference PyLong_FromString(string value, int radix)
         {
-            using var valPtr = new StrPtr(value, Encodings.UTF8);
+            using var valPtr = new StrPtr(value);
             return Delegates.PyLong_FromString(valPtr, IntPtr.Zero, radix);
         }
 
@@ -1274,7 +1275,7 @@ internal static NewReference EmptyPyBytes()
         internal static NewReference PyByteArray_FromStringAndSize(IntPtr strPtr, nint len) => Delegates.PyByteArray_FromStringAndSize(strPtr, len);
         internal static NewReference PyByteArray_FromStringAndSize(string s)
         {
-            using var ptr = new StrPtr(s, Encodings.UTF8);
+            using var ptr = new StrPtr(s);
             return PyByteArray_FromStringAndSize(ptr.RawPointer, checked((nint)ptr.ByteCount));
         }
 
@@ -1302,7 +1303,7 @@ internal static IntPtr PyBytes_AsString(BorrowedReference ob)
 
         internal static NewReference PyUnicode_InternFromString(string s)
         {
-            using var ptr = new StrPtr(s, Encodings.UTF8);
+            using var ptr = new StrPtr(s);
             return Delegates.PyUnicode_InternFromString(ptr);
         }
 
@@ -1377,7 +1378,7 @@ internal static bool PyDict_Check(BorrowedReference ob)
 
         internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer, string key)
         {
-            using var keyStr = new StrPtr(key, Encodings.UTF8);
+            using var keyStr = new StrPtr(key);
             return Delegates.PyDict_GetItemString(pointer, keyStr);
         }
 
@@ -1393,7 +1394,7 @@ internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer
         /// </summary>
         internal static int PyDict_SetItemString(BorrowedReference dict, string key, BorrowedReference value)
         {
-            using var keyPtr = new StrPtr(key, Encodings.UTF8);
+            using var keyPtr = new StrPtr(key);
             return Delegates.PyDict_SetItemString(dict, keyPtr, value);
         }
 
@@ -1402,7 +1403,7 @@ internal static int PyDict_SetItemString(BorrowedReference dict, string key, Bor
 
         internal static int PyDict_DelItemString(BorrowedReference pointer, string key)
         {
-            using var keyPtr = new StrPtr(key, Encodings.UTF8);
+            using var keyPtr = new StrPtr(key);
             return Delegates.PyDict_DelItemString(pointer, keyPtr);
         }
 
@@ -1517,7 +1518,7 @@ internal static bool PyIter_Check(BorrowedReference ob)
 
         internal static NewReference PyModule_New(string name)
         {
-            using var namePtr = new StrPtr(name, Encodings.UTF8);
+            using var namePtr = new StrPtr(name);
             return Delegates.PyModule_New(namePtr);
         }
 
@@ -1531,7 +1532,7 @@ internal static NewReference PyModule_New(string name)
         /// <returns>Return -1 on error, 0 on success.</returns>
         internal static int PyModule_AddObject(BorrowedReference module, string name, StolenReference value)
         {
-            using var namePtr = new StrPtr(name, Encodings.UTF8);
+            using var namePtr = new StrPtr(name);
             IntPtr valueAddr = value.DangerousGetAddressOrNull();
             int res = Delegates.PyModule_AddObject(module, namePtr, valueAddr);
             // We can't just exit here because the reference is stolen only on success.
@@ -1549,7 +1550,7 @@ internal static int PyModule_AddObject(BorrowedReference module, string name, St
 
         internal static NewReference PyImport_ImportModule(string name)
         {
-            using var namePtr = new StrPtr(name, Encodings.UTF8);
+            using var namePtr = new StrPtr(name);
             return Delegates.PyImport_ImportModule(namePtr);
         }
 
@@ -1558,7 +1559,7 @@ internal static NewReference PyImport_ImportModule(string name)
 
         internal static BorrowedReference PyImport_AddModule(string name)
         {
-            using var namePtr = new StrPtr(name, Encodings.UTF8);
+            using var namePtr = new StrPtr(name);
             return Delegates.PyImport_AddModule(namePtr);
         }
 
@@ -1586,13 +1587,13 @@ internal static void PySys_SetArgvEx(int argc, string[] argv, int updatepath)
 
         internal static BorrowedReference PySys_GetObject(string name)
         {
-            using var namePtr = new StrPtr(name, Encodings.UTF8);
+            using var namePtr = new StrPtr(name);
             return Delegates.PySys_GetObject(namePtr);
         }
 
         internal static int PySys_SetObject(string name, BorrowedReference ob)
         {
-            using var namePtr = new StrPtr(name, Encodings.UTF8);
+            using var namePtr = new StrPtr(name);
             return Delegates.PySys_SetObject(namePtr, ob);
         }
 
@@ -1691,7 +1692,7 @@ internal static IntPtr PyMem_Malloc(long size)
 
         internal static void PyErr_SetString(BorrowedReference ob, string message)
         {
-            using var msgPtr = new StrPtr(message, Encodings.UTF8);
+            using var msgPtr = new StrPtr(message);
             Delegates.PyErr_SetString(ob, msgPtr);
         }
 

Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,7 @@ public NativeTypeSpec(TypeSpec spec)`
`17`	`17`	`{`
`18`	`18`	`if (spec is null) throw new ArgumentNullException(nameof(spec));`
`19`	`19`
`20`		`- this.Name = new StrPtr(spec.Name, Encodings.UTF8);`
	`20`	`+ this.Name = new StrPtr(spec.Name);`
`21`	`21`	`this.BasicSize = spec.BasicSize;`
`22`	`22`	`this.ItemSize = spec.ItemSize;`
`23`	`23`	`this.Flags = (int)spec.Flags;`
Original file line number	Diff line number	Diff line change
`@@ -10,6 +10,8 @@ struct StrPtr : IDisposable`
`10`	`10`	`public IntPtr RawPointer { get; set; }`
`11`	`11`	`unsafe byte* Bytes => (byte*)this.RawPointer;`
`12`	`12`
	`13`	`+ public unsafe StrPtr(string value) : this(value, Encodings.UTF8) {}`
	`14`	`+`
`13`	`15`	`public unsafe StrPtr(string value, Encoding encoding)`
`14`	`16`	`{`
`15`	`17`	`if (value is null) throw new ArgumentNullException(nameof(value));`
Original file line number	Diff line number	Diff line change
`@@ -795,13 +795,13 @@ public static int Py_Main(int argc, string[] argv)`
`795`	`795`
`796`	`796`	`internal static int PyRun_SimpleString(string code)`
`797`	`797`	`{`
`798`		`- using var codePtr = new StrPtr(code, Encodings.UTF8);`
	`798`	`+ using var codePtr = new StrPtr(code);`
`799`	`799`	`return Delegates.PyRun_SimpleStringFlags(codePtr, Utf8String);`
`800`	`800`	`}`
`801`	`801`
`802`	`802`	`internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedReference globals, BorrowedReference locals)`
`803`	`803`	`{`
`804`		`- using var codePtr = new StrPtr(code, Encodings.UTF8);`
	`804`	`+ using var codePtr = new StrPtr(code);`
`805`	`805`	`return Delegates.PyRun_StringFlags(codePtr, st, globals, locals, Utf8String);`
`806`	`806`	`}`
`807`	`807`
`@@ -813,14 +813,15 @@ internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedR`
`813`	`813`	`/// </summary>`
`814`	`814`	`internal static NewReference Py_CompileString(string str, string file, int start)`
`815`	`815`	`{`
`816`		`- using var strPtr = new StrPtr(str, Encodings.UTF8);`
	`816`	`+ using var strPtr = new StrPtr(str);`
	`817`	`+`
`817`	`818`	`using var fileObj = new PyString(file);`
`818`	`819`	`return Delegates.Py_CompileStringObject(strPtr, fileObj, start, Utf8String, -1);`
`819`	`820`	`}`
`820`	`821`
`821`	`822`	`internal static NewReference PyImport_ExecCodeModule(string name, BorrowedReference code)`
`822`	`823`	`{`
`823`		`- using var namePtr = new StrPtr(name, Encodings.UTF8);`
	`824`	`+ using var namePtr = new StrPtr(name);`
`824`	`825`	`return Delegates.PyImport_ExecCodeModule(namePtr, code);`
`825`	`826`	`}`
`826`	`827`
`@@ -867,13 +868,13 @@ internal static bool PyObject_IsIterable(BorrowedReference ob)`
`867`	`868`
`868`	`869`	`internal static int PyObject_HasAttrString(BorrowedReference pointer, string name)`
`869`	`870`	`{`
`870`		`- using var namePtr = new StrPtr(name, Encodings.UTF8);`
	`871`	`+ using var namePtr = new StrPtr(name);`
`871`	`872`	`return Delegates.PyObject_HasAttrString(pointer, namePtr);`
`872`	`873`	`}`
`873`	`874`
`874`	`875`	`internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, string name)`
`875`	`876`	`{`
`876`		`- using var namePtr = new StrPtr(name, Encodings.UTF8);`
	`877`	`+ using var namePtr = new StrPtr(name);`
`877`	`878`	`return Delegates.PyObject_GetAttrString(pointer, namePtr);`
`878`	`879`	`}`
`879`	`880`
`@@ -884,12 +885,12 @@ internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, S`
`884`	`885`	`internal static int PyObject_DelAttr(BorrowedReference @object, BorrowedReference name) => Delegates.PyObject_SetAttr(@object, name, null);`
`885`	`886`	`internal static int PyObject_DelAttrString(BorrowedReference @object, string name)`
`886`	`887`	`{`
`887`		`- using var namePtr = new StrPtr(name, Encodings.UTF8);`
	`888`	`+ using var namePtr = new StrPtr(name);`
`888`	`889`	`return Delegates.PyObject_SetAttrString(@object, namePtr, null);`
`889`	`890`	`}`
`890`	`891`	`internal static int PyObject_SetAttrString(BorrowedReference @object, string name, BorrowedReference value)`
`891`	`892`	`{`
`892`		`- using var namePtr = new StrPtr(name, Encodings.UTF8);`
	`893`	`+ using var namePtr = new StrPtr(name);`
`893`	`894`	`return Delegates.PyObject_SetAttrString(@object, namePtr, value);`
`894`	`895`	`}`
`895`	`896`
`@@ -1071,7 +1072,7 @@ internal static bool PyBool_CheckExact(BorrowedReference ob)`
`1071`	`1072`
`1072`	`1073`	`internal static NewReference PyLong_FromString(string value, int radix)`
`1073`	`1074`	`{`
`1074`		`- using var valPtr = new StrPtr(value, Encodings.UTF8);`
	`1075`	`+ using var valPtr = new StrPtr(value);`
`1075`	`1076`	`return Delegates.PyLong_FromString(valPtr, IntPtr.Zero, radix);`
`1076`	`1077`	`}`
`1077`	`1078`
`@@ -1274,7 +1275,7 @@ internal static NewReference EmptyPyBytes()`
`1274`	`1275`	`internal static NewReference PyByteArray_FromStringAndSize(IntPtr strPtr, nint len) => Delegates.PyByteArray_FromStringAndSize(strPtr, len);`
`1275`	`1276`	`internal static NewReference PyByteArray_FromStringAndSize(string s)`
`1276`	`1277`	`{`
`1277`		`- using var ptr = new StrPtr(s, Encodings.UTF8);`
	`1278`	`+ using var ptr = new StrPtr(s);`
`1278`	`1279`	`return PyByteArray_FromStringAndSize(ptr.RawPointer, checked((nint)ptr.ByteCount));`
`1279`	`1280`	`}`
`1280`	`1281`
`@@ -1302,7 +1303,7 @@ internal static IntPtr PyBytes_AsString(BorrowedReference ob)`
`1302`	`1303`
`1303`	`1304`	`internal static NewReference PyUnicode_InternFromString(string s)`
`1304`	`1305`	`{`
`1305`		`- using var ptr = new StrPtr(s, Encodings.UTF8);`
	`1306`	`+ using var ptr = new StrPtr(s);`
`1306`	`1307`	`return Delegates.PyUnicode_InternFromString(ptr);`
`1307`	`1308`	`}`
`1308`	`1309`
`@@ -1377,7 +1378,7 @@ internal static bool PyDict_Check(BorrowedReference ob)`
`1377`	`1378`
`1378`	`1379`	`internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer, string key)`
`1379`	`1380`	`{`
`1380`		`- using var keyStr = new StrPtr(key, Encodings.UTF8);`
	`1381`	`+ using var keyStr = new StrPtr(key);`
`1381`	`1382`	`return Delegates.PyDict_GetItemString(pointer, keyStr);`
`1382`	`1383`	`}`
`1383`	`1384`
`@@ -1393,7 +1394,7 @@ internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer`
`1393`	`1394`	`/// </summary>`
`1394`	`1395`	`internal static int PyDict_SetItemString(BorrowedReference dict, string key, BorrowedReference value)`
`1395`	`1396`	`{`
`1396`		`- using var keyPtr = new StrPtr(key, Encodings.UTF8);`
	`1397`	`+ using var keyPtr = new StrPtr(key);`
`1397`	`1398`	`return Delegates.PyDict_SetItemString(dict, keyPtr, value);`
`1398`	`1399`	`}`
`1399`	`1400`
`@@ -1402,7 +1403,7 @@ internal static int PyDict_SetItemString(BorrowedReference dict, string key, Bor`
`1402`	`1403`
`1403`	`1404`	`internal static int PyDict_DelItemString(BorrowedReference pointer, string key)`
`1404`	`1405`	`{`
`1405`		`- using var keyPtr = new StrPtr(key, Encodings.UTF8);`
	`1406`	`+ using var keyPtr = new StrPtr(key);`
`1406`	`1407`	`return Delegates.PyDict_DelItemString(pointer, keyPtr);`
`1407`	`1408`	`}`
`1408`	`1409`
`@@ -1517,7 +1518,7 @@ internal static bool PyIter_Check(BorrowedReference ob)`
`1517`	`1518`
`1518`	`1519`	`internal static NewReference PyModule_New(string name)`
`1519`	`1520`	`{`
`1520`		`- using var namePtr = new StrPtr(name, Encodings.UTF8);`
	`1521`	`+ using var namePtr = new StrPtr(name);`
`1521`	`1522`	`return Delegates.PyModule_New(namePtr);`
`1522`	`1523`	`}`
`1523`	`1524`
`@@ -1531,7 +1532,7 @@ internal static NewReference PyModule_New(string name)`
`1531`	`1532`	`/// <returns>Return -1 on error, 0 on success.</returns>`
`1532`	`1533`	`internal static int PyModule_AddObject(BorrowedReference module, string name, StolenReference value)`
`1533`	`1534`	`{`
`1534`		`- using var namePtr = new StrPtr(name, Encodings.UTF8);`
	`1535`	`+ using var namePtr = new StrPtr(name);`
`1535`	`1536`	`IntPtr valueAddr = value.DangerousGetAddressOrNull();`
`1536`	`1537`	`int res = Delegates.PyModule_AddObject(module, namePtr, valueAddr);`
`1537`	`1538`	`// We can't just exit here because the reference is stolen only on success.`
`@@ -1549,7 +1550,7 @@ internal static int PyModule_AddObject(BorrowedReference module, string name, St`
`1549`	`1550`
`1550`	`1551`	`internal static NewReference PyImport_ImportModule(string name)`
`1551`	`1552`	`{`
`1552`		`- using var namePtr = new StrPtr(name, Encodings.UTF8);`
	`1553`	`+ using var namePtr = new StrPtr(name);`
`1553`	`1554`	`return Delegates.PyImport_ImportModule(namePtr);`
`1554`	`1555`	`}`
`1555`	`1556`
`@@ -1558,7 +1559,7 @@ internal static NewReference PyImport_ImportModule(string name)`
`1558`	`1559`
`1559`	`1560`	`internal static BorrowedReference PyImport_AddModule(string name)`
`1560`	`1561`	`{`
`1561`		`- using var namePtr = new StrPtr(name, Encodings.UTF8);`
	`1562`	`+ using var namePtr = new StrPtr(name);`
`1562`	`1563`	`return Delegates.PyImport_AddModule(namePtr);`
`1563`	`1564`	`}`
`1564`	`1565`
`@@ -1586,13 +1587,13 @@ internal static void PySys_SetArgvEx(int argc, string[] argv, int updatepath)`
`1586`	`1587`
`1587`	`1588`	`internal static BorrowedReference PySys_GetObject(string name)`
`1588`	`1589`	`{`
`1589`		`- using var namePtr = new StrPtr(name, Encodings.UTF8);`
	`1590`	`+ using var namePtr = new StrPtr(name);`
`1590`	`1591`	`return Delegates.PySys_GetObject(namePtr);`
`1591`	`1592`	`}`
`1592`	`1593`
`1593`	`1594`	`internal static int PySys_SetObject(string name, BorrowedReference ob)`
`1594`	`1595`	`{`
`1595`		`- using var namePtr = new StrPtr(name, Encodings.UTF8);`
	`1596`	`+ using var namePtr = new StrPtr(name);`
`1596`	`1597`	`return Delegates.PySys_SetObject(namePtr, ob);`
`1597`	`1598`	`}`
`1598`	`1599`
`@@ -1691,7 +1692,7 @@ internal static IntPtr PyMem_Malloc(long size)`
`1691`	`1692`
`1692`	`1693`	`internal static void PyErr_SetString(BorrowedReference ob, string message)`
`1693`	`1694`	`{`
`1694`		`- using var msgPtr = new StrPtr(message, Encodings.UTF8);`
	`1695`	`+ using var msgPtr = new StrPtr(message);`
`1695`	`1696`	`Delegates.PyErr_SetString(ob, msgPtr);`
`1696`	`1697`	`}`
`1697`	`1698`