added a few mixins to reflected .NET collection types, that implement…

… corresponding pythonic interfaces
pythonnet · lostmsu · Sep 23, 2021 · May 24, 2020 · Jun 3, 2020 · Jun 17, 2020
commit 5cb300a7a83d2599ad1af921641a494324ed04c0
diff --git a/src/runtime/InteropConfiguration.cs b/src/runtime/InteropConfiguration.cs
@@ -3,6 +3,8 @@ namespace Python.Runtime
     using System;
     using System.Collections.Generic;
 
+    using Python.Runtime.Mixins;
+
     public sealed class InteropConfiguration
     {
         internal readonly PythonBaseTypeProviderGroup pythonBaseTypeProviders
@@ -18,6 +20,7 @@ public static InteropConfiguration MakeDefault()
                 PythonBaseTypeProviders =
                 {
                     DefaultBaseTypeProvider.Instance,
+                    new CollectionMixinsProvider(new Lazy<PyObject>(() => Py.Import("clr._extras.collections"))),
                 },
             };
         }

diff --git a/src/runtime/Mixins/CollectionMixinsProvider.cs b/src/runtime/Mixins/CollectionMixinsProvider.cs
@@ -0,0 +1,90 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Python.Runtime.Mixins
+{
+    class CollectionMixinsProvider : IPythonBaseTypeProvider
+    {
+        readonly Lazy<PyObject> mixinsModule;
+        public CollectionMixinsProvider(Lazy<PyObject> mixinsModule)
+        {
+            this.mixinsModule = mixinsModule ?? throw new ArgumentNullException(nameof(mixinsModule));
+        }
+
+        public PyObject Mixins => this.mixinsModule.Value;
+
+        public IEnumerable<PyType> GetBaseTypes(Type type, IList<PyType> existingBases)
+        {
+            if (type is null)
+                throw new ArgumentNullException(nameof(type));
+
+            if (existingBases is null)
+                throw new ArgumentNullException(nameof(existingBases));
+
+            var interfaces = NewInterfaces(type).Select(GetDefinition).ToArray();
+
+            var newBases = new List<PyType>();
+            newBases.AddRange(existingBases);
+
+            // dictionaries
+            if (interfaces.Contains(typeof(IDictionary<,>)))
+            {
+                newBases.Add(new PyType(this.Mixins.GetAttr("MutableMappingMixin")));
+            }
+            else if (interfaces.Contains(typeof(IReadOnlyDictionary<,>)))
+            {
+                newBases.Add(new PyType(this.Mixins.GetAttr("MappingMixin")));
+            }
+
+            // item collections
+            if (interfaces.Contains(typeof(IList<>))
+                || interfaces.Contains(typeof(System.Collections.IList)))
+            {
+                newBases.Add(new PyType(this.Mixins.GetAttr("MutableSequenceMixin")));
+            }
+            else if (interfaces.Contains(typeof(IReadOnlyList<>)))
+            {
+                newBases.Add(new PyType(this.Mixins.GetAttr("SequenceMixin")));
+            }
+            else if (interfaces.Contains(typeof(ICollection<>))
+                     || interfaces.Contains(typeof(System.Collections.ICollection)))
+            {
+                newBases.Add(new PyType(this.Mixins.GetAttr("CollectionMixin")));
+            }
+            else if (interfaces.Contains(typeof(System.Collections.IEnumerable)))
+            {
+                newBases.Add(new PyType(this.Mixins.GetAttr("IterableMixin")));
+            }
+
+            // enumerators
+            if (interfaces.Contains(typeof(System.Collections.IEnumerator)))
+            {
+                newBases.Add(new PyType(this.Mixins.GetAttr("IteratorMixin")));
+            }
+
+            if (newBases.Count == existingBases.Count)
+            {
+                return existingBases;
+            }
+
+            if (type.IsInterface && type.BaseType is null)
+            {
+                newBases.RemoveAll(@base => @base.Handle == Runtime.PyBaseObjectType);
+            }
+
+            return newBases;
+        }
+
+        static Type[] NewInterfaces(Type type)
+        {
+            var result = type.GetInterfaces();
+            return type.BaseType != null
+                ? result.Except(type.BaseType.GetInterfaces()).ToArray()
+                : result;
+        }
+
+        static Type GetDefinition(Type type)
+            => type.IsGenericType ? type.GetGenericTypeDefinition() : type;
+    }
+}
diff --git a/src/runtime/Mixins/collections.py b/src/runtime/Mixins/collections.py
@@ -0,0 +1,61 @@
+"""
+Implements collections.abc for common .NET types
+https://docs.python.org/3.6/library/collections.abc.html
+"""
+
+import collections.abc as col
+
+class IteratorMixin(col.Iterator):
+    def close(self):
+        self.Dispose()
+
+class IterableMixin(col.Iterable):
+    pass
+
+class SizedMixin(col.Sized):
+    def __len__(self): return self.Count
+
+class ContainerMixin(col.Container):
+    def __contains__(self, item): return self.Contains(item)
+
+try:
+    abc_Collection = col.Collection
+except AttributeError:
+    # Python 3.5- does not have collections.abc.Collection
+    abc_Collection = col.Container
+
+class CollectionMixin(SizedMixin, IterableMixin, ContainerMixin, abc_Collection):
+    pass
+
+class SequenceMixin(CollectionMixin, col.Sequence):
+    pass
+
+class MutableSequenceMixin(SequenceMixin, col.MutableSequence):
+    pass
+
+class MappingMixin(CollectionMixin, col.Mapping):
+    def keys(self): return self.Keys
+    def items(self): return self
+    def values(self): return self.Values
+    def __iter__(self): raise NotImplementedError
+    def get(self, key):
+        _, item = self.TryGetValue(key)
+        return item
+
+class MutableMappingMixin(MappingMixin, col.MutableMapping):
+    def __delitem__(self, key):
+        return self.Remove(key)
+    def clear(self):
+        self.Clear()
+    def pop(self, key):
+        return self.Remove(key)
+    def setdefault(self, key, value):
+        existed, item = self.TryGetValue(key)
+        if existed:
+            return item
+        else:
+            self[key] = value
+            return value
+    def update(self, items):
+        for key, value in items:
+            self[key] = value
diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj
@@ -39,10 +39,10 @@
   </ItemGroup>
 
   <ItemGroup>
-    <None Remove="resources\clr.py" />
     <EmbeddedResource Include="resources\clr.py">
       <LogicalName>clr.py</LogicalName>
     </EmbeddedResource>
+    <EmbeddedResource Include="Mixins\*.py" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs
@@ -10,6 +10,9 @@ internal static class Util
         internal const string MinimalPythonVersionRequired =
             "Only Python 3.5 or newer is supported";
 
+        internal const string UseOverloadWithReferenceTypes =
+            "This API is unsafe, and will be removed in the future. Use overloads working with *Reference types";
+
         internal static Int64 ReadCLong(IntPtr tp, int offset)
         {
             // On Windows, a C long is always 32 bits.

diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs
@@ -66,7 +66,7 @@ private PyScope(IntPtr ptr, PyScopeManager manager) : base(ptr)
             PythonException.ThrowIfIsNull(variables);
 
             int res = Runtime.PyDict_SetItem(
-                VarsRef, PyIdentifier.__builtins__,
+                VarsRef, new BorrowedReference(PyIdentifier.__builtins__),
                 Runtime.PyEval_GetBuiltins()
             );
             PythonException.ThrowIfIsNotZero(res);

diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs
@@ -223,10 +223,8 @@ public static void Initialize(IEnumerable<string> args, bool setSysArgv = true,
             var locals = new PyDict();
             try
             {
-                BorrowedReference module = Runtime.PyImport_AddModule("clr._extras");
+                BorrowedReference module = DefineModule("clr._extras");
                 BorrowedReference module_globals = Runtime.PyModule_GetDict(module);
-                BorrowedReference builtins = Runtime.PyEval_GetBuiltins();
-                Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins);
 
                 Assembly assembly = Assembly.GetExecutingAssembly();
                 using (Stream stream = assembly.GetManifestResourceStream("clr.py"))
@@ -237,6 +235,8 @@ public static void Initialize(IEnumerable<string> args, bool setSysArgv = true,
                     Exec(clr_py, module_globals, locals.Reference);
                 }
 
+                LoadExtraModules(module_globals);
+
                 // add the imported module to the clr module, and copy the API functions
                 // and decorators into the main clr module.
                 Runtime.PyDict_SetItemString(clr_dict, "_extras", module);
@@ -258,6 +258,34 @@ public static void Initialize(IEnumerable<string> args, bool setSysArgv = true,
             }
         }
 
+        static BorrowedReference DefineModule(string name)
+        {
+            var module = Runtime.PyImport_AddModule(name);
+            var module_globals = Runtime.PyModule_GetDict(module);
+            var builtins = Runtime.PyEval_GetBuiltins();
+            Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins);
+            return module;
+        }
+
+        static void LoadExtraModules(BorrowedReference targetModuleDict)
+        {
+            Assembly assembly = Assembly.GetExecutingAssembly();
+            foreach (string nested in new[] { "collections" })
+            {
+                var module = DefineModule("clr._extras." + nested);
+                var module_globals = Runtime.PyModule_GetDict(module);
+                string resourceName = typeof(PythonEngine).Namespace + ".Mixins." + nested + ".py";
+                using (var stream = assembly.GetManifestResourceStream(resourceName))
+                using (var reader = new StreamReader(stream))
+                {
+                    string pyCode = reader.ReadToEnd();
+                    Exec(pyCode, module_globals.DangerousGetAddress(), module_globals.DangerousGetAddress());
+                }
+
+                Runtime.PyDict_SetItemString(targetModuleDict, nested, module);
+            }
+        }
+
         static void OnDomainUnload(object _, EventArgs __)
         {
             Shutdown();
@@ -618,7 +646,7 @@ internal static PyObject RunString(string code, BorrowedReference globals, Borro
                 {
                     globals = tempGlobals = NewReference.DangerousFromPointer(Runtime.PyDict_New());
                     Runtime.PyDict_SetItem(
-                        globals, PyIdentifier.__builtins__,
+                        globals, new BorrowedReference(PyIdentifier.__builtins__),
                         Runtime.PyEval_GetBuiltins()
                     );
                 }

diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
@@ -1679,6 +1679,7 @@ internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer
         /// <summary>
         /// Return 0 on success or -1 on failure.
         /// </summary>
+        [Obsolete]
         internal static int PyDict_SetItem(BorrowedReference dict, IntPtr key, BorrowedReference value) => Delegates.PyDict_SetItem(dict, new BorrowedReference(key), value);
         /// <summary>
         /// Return 0 on success or -1 on failure.
@@ -2038,7 +2039,7 @@ internal static bool PyType_IsSameAsOrSubtype(BorrowedReference type, BorrowedRe
         internal static NewReference PyType_FromSpecWithBases(in NativeTypeSpec spec, BorrowedReference bases) => Delegates.PyType_FromSpecWithBases(in spec, bases);
 
         /// <summary>
-        /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error.
+        /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type�s base class. Return 0 on success, or return -1 and sets an exception on error.
         /// </summary>
 
         internal static int PyType_Ready(IntPtr type) => Delegates.PyType_Ready(type);