detect the size of wchar_t (aka Runtime.UCS) at runtime using PyUnicode_GetMax

lostmsu · lostmsu · commit 4d43b54d8b5d · 2020-11-29T18:37:01.000-08:00
diff --git a/.editorconfig b/.editorconfig
@@ -34,6 +34,10 @@ csharp_new_line_before_finally = true
 [*.sln]
 indent_style = tab
 
+[*.csproj]
+charset = utf-8
+insert_final_newline = true
+
 # bumpversion reformats itself after every bump
 [.bumpversion.cfg]
 trim_trailing_whitespace = false
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][].
 
 ### Changed
 -   Drop support for Python 2, 3.4, and 3.5
+-   `wchar_t` size aka `Runtime.UCS` is now determined at runtime
 -   `clr.AddReference` may now throw errors besides `FileNotFoundException`, that provide more
 details about the cause of the failure
 -   `clr.AddReference` no longer adds ".dll" implicitly
diff --git a/setup.py b/setup.py
@@ -251,14 +251,8 @@ def build_extension(self, ext):
         if not os.path.exists(dest_dir):
             os.makedirs(dest_dir)
 
-        # Up to Python 3.2 sys.maxunicode is used to determine the size of
-        # Py_UNICODE, but from 3.3 onwards Py_UNICODE is a typedef of wchar_t.
-        import ctypes
-        unicode_width = ctypes.sizeof(ctypes.c_wchar)
-
         defines = [
             "PYTHON{0}{1}".format(PY_MAJOR, PY_MINOR),
-            "UCS{0}".format(unicode_width),
         ]
 
         if CONFIG == "Debug":
diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj
@@ -49,9 +49,8 @@
     <Python3Version>$(PYTHONNET_PY3_VERSION)</Python3Version>
     <Python3Version Condition="'$(Python3Version)'==''">PYTHON38</Python3Version>
     <PythonWinDefineConstants>$(PYTHONNET_WIN_DEFINE_CONSTANTS)</PythonWinDefineConstants>
-    <PythonWinDefineConstants Condition="'$(PythonWinDefineConstants)'==''">UCS2</PythonWinDefineConstants>
     <PythonMonoDefineConstants>$(PYTHONNET_MONO_DEFINE_CONSTANTS)</PythonMonoDefineConstants>
-    <PythonMonoDefineConstants Condition="'$(PythonMonoDefineConstants)'==''">UCS4;MONO_LINUX;PYTHON_WITH_PYMALLOC</PythonMonoDefineConstants>
+    <PythonMonoDefineConstants Condition="'$(PythonMonoDefineConstants)'==''">MONO_LINUX;PYTHON_WITH_PYMALLOC</PythonMonoDefineConstants>
     <PythonInteropFile Condition="'$(PythonInteropFile)'==''">$(PYTHONNET_INTEROP_FILE)</PythonInteropFile>
   </PropertyGroup>
   <PropertyGroup Condition="$(Configuration.Contains('Debug')) AND '$(TargetFramework)'=='net40'">
@@ -143,20 +142,20 @@
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="All" />
   </ItemGroup>
 
-  <ItemGroup>
-    <None Update="intern_.tt">
-      <Generator>TextTemplatingFileGenerator</Generator>
-      <LastGenOutput>intern_.cs</LastGenOutput>
-    </None>
+  <ItemGroup>
+    <None Update="intern_.tt">
+      <Generator>TextTemplatingFileGenerator</Generator>
+      <LastGenOutput>intern_.cs</LastGenOutput>
+    </None>
   </ItemGroup>
 
 
-  <ItemGroup>
-    <Compile Update="intern_.cs">
-      <DesignTime>True</DesignTime>
-      <AutoGen>True</AutoGen>
-      <DependentUpon>intern_.tt</DependentUpon>
-    </Compile>
+  <ItemGroup>
+    <Compile Update="intern_.cs">
+      <DesignTime>True</DesignTime>
+      <AutoGen>True</AutoGen>
+      <DependentUpon>intern_.tt</DependentUpon>
+    </Compile>
   </ItemGroup>
 
   <Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj
@@ -29,46 +29,46 @@
     <PlatformTarget>x64</PlatformTarget>
   </PropertyGroup>-->
   <PropertyGroup Condition=" '$(Configuration)' == 'ReleaseMono'">
-    <DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON2;PYTHON27;UCS4</DefineConstants>
+    <DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON2;PYTHON27</DefineConstants>
     <Optimize>true</Optimize>
     <DebugType>pdbonly</DebugType>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)' == 'ReleaseMonoPY3'">
-    <DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON3;PYTHON38;UCS4</DefineConstants>
+    <DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON3;PYTHON38</DefineConstants>
     <Optimize>true</Optimize>
     <DebugType>pdbonly</DebugType>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)' == 'DebugMono'">
     <DebugSymbols>true</DebugSymbols>
-    <DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON2;PYTHON27;UCS4;TRACE;DEBUG</DefineConstants>
+    <DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON2;PYTHON27;TRACE;DEBUG</DefineConstants>
     <Optimize>false</Optimize>
     <DebugType>full</DebugType>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)' == 'DebugMonoPY3'">
     <DebugSymbols>true</DebugSymbols>
-    <DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON3;PYTHON38;UCS4;TRACE;DEBUG</DefineConstants>
+    <DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON3;PYTHON38;TRACE;DEBUG</DefineConstants>
     <Optimize>false</Optimize>
     <DebugType>full</DebugType>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)' == 'ReleaseWin'">
-    <DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON2;PYTHON27;UCS2</DefineConstants>
+    <DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON2;PYTHON27</DefineConstants>
     <Optimize>true</Optimize>
     <DebugType>pdbonly</DebugType>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)' == 'ReleaseWinPY3'">
-    <DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON3;PYTHON38;UCS2</DefineConstants>
+    <DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON3;PYTHON38</DefineConstants>
     <Optimize>true</Optimize>
     <DebugType>pdbonly</DebugType>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)' == 'DebugWin'">
     <DebugSymbols>true</DebugSymbols>
-    <DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON2;PYTHON27;UCS2;TRACE;DEBUG</DefineConstants>
+    <DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON2;PYTHON27;TRACE;DEBUG</DefineConstants>
     <Optimize>false</Optimize>
     <DebugType>full</DebugType>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)' == 'DebugWinPY3'">
     <DebugSymbols>true</DebugSymbols>
-    <DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON3;PYTHON38;UCS2;TRACE;DEBUG</DefineConstants>
+    <DefineConstants Condition="'$(DefineConstants)' == ''">PYTHON3;PYTHON38;TRACE;DEBUG</DefineConstants>
     <Optimize>false</Optimize>
     <DebugType>full</DebugType>
   </PropertyGroup>
diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
@@ -18,31 +18,8 @@ namespace Python.Runtime
     /// </summary>
     public class Runtime
     {
-        // C# compiler copies constants to the assemblies that references this library.
-        // We needs to replace all public constants to static readonly fields to allow
-        // binary substitution of different Python.Runtime.dll builds in a target application.
-
         public static int UCS => _UCS;
-
-#if UCS4
-        internal const int _UCS = 4;
-
-        /// <summary>
-        /// EntryPoint to be used in DllImport to map to correct Unicode
-        /// methods prior to PEP393. Only used for PY27.
-        /// </summary>
-        private const string PyUnicodeEntryPoint = "PyUnicodeUCS4_";
-#elif UCS2
-        internal const int _UCS = 2;
-
-        /// <summary>
-        /// EntryPoint to be used in DllImport to map to correct Unicode
-        /// methods prior to PEP393. Only used for PY27.
-        /// </summary>
-        private const string PyUnicodeEntryPoint = "PyUnicodeUCS2_";
-#else
-#error You must define either UCS2 or UCS4!
-#endif
+        internal static readonly int _UCS = PyUnicode_GetMax() <= 0xFFFF ? 2 : 4;
 
 #if PYTHON36
         const string _minor = "6";
@@ -1537,17 +1514,6 @@ internal static IntPtr PyBytes_AS_STRING(IntPtr ob)
             return ob + BytesOffset.ob_sval;
         }
 
-        internal static IntPtr PyString_FromStringAndSize(string value, long size)
-        {
-            return _PyString_FromStringAndSize(value, new IntPtr(size));
-        }
-
-        [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl,
-            EntryPoint = "PyUnicode_FromStringAndSize")]
-        internal static extern IntPtr _PyString_FromStringAndSize(
-            [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string value,
-            IntPtr size
-        );
 
         internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size)
         {
@@ -1588,6 +1554,9 @@ internal static IntPtr PyUnicode_FromUnicode(string s, long size)
             return PyUnicode_FromKindAndData(_UCS, s, size);
         }
 
+        [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)]
+        internal static extern int PyUnicode_GetMax();
+
         internal static long PyUnicode_GetSize(IntPtr ob)
         {
             return (long)_PyUnicode_GetSize(ob);
