fixed ImportHook not seeing namespaces in assemblies loaded via direct call to Assembly.Load* methods

lostmsu · lostmsu · commit 9c07278effe4 · 2021-09-23T22:38:49.000-07:00
diff --git a/src/python_tests_runner/PythonTestRunner.cs b/src/python_tests_runner/PythonTestRunner.cs
@@ -34,7 +34,7 @@ static IEnumerable<string[]> PythonTestCases()
         {
             // Add the test that you want to debug here.
             yield return new[] { "test_enum", "test_enum_standard_attrs" };
-            yield return new[] { "test_module", "test_explicit_assembly_load" };
+            yield return new[] { "test_generic", "test_missing_generic_type" };
         }
 
         /// <summary>
diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs
@@ -290,7 +290,10 @@ internal static void ScanAssembly(Assembly assembly)
                     for (var n = 0; n < names.Length; n++)
                     {
                         s = n == 0 ? names[0] : s + "." + names[n];
-                        namespaces.TryAdd(s, new ConcurrentDictionary<Assembly, string>());
+                        if (namespaces.TryAdd(s, new ConcurrentDictionary<Assembly, string>()))
+                        {
+                            ImportHook.AddNamespace(s);
+                        }
                     }
                 }
 
diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs
@@ -1,6 +1,5 @@
 using System;
-using System.Collections.Generic;
-using System.Runtime.InteropServices;
+using System.Collections.Concurrent;
 
 namespace Python.Runtime
 {
@@ -37,6 +36,11 @@ def find_spec(klass, fullname, paths=None, target=None):
         if 'clr' not in sys.modules:
             return None
         clr = sys.modules['clr']
+
+        _add_pending = clr._add_pending_namespaces
+        if _add_pending:
+           _add_pending()
+
         if clr._available_namespaces and fullname in clr._available_namespaces:
             return importlib.machinery.ModuleSpec(fullname, DotNetLoader(), is_package=True)
         return None
@@ -169,12 +173,26 @@ static void TeardownNameSpaceTracking()
             Runtime.PyDict_SetItemString(root.dict, availableNsKey, Runtime.PyNone);
         }
 
-        public static void AddNamespace(string name)
+        static readonly ConcurrentQueue<string> addPending = new();
+        public static void AddNamespace(string name) => addPending.Enqueue(name);
+
+        internal static int AddPendingNamespaces()
+        {
+            int added = 0;
+            while (addPending.TryDequeue(out string ns))
+            {
+                AddNamespaceWithGIL(ns);
+                added++;
+            }
+            return added;
+        }
+
+        internal static void AddNamespaceWithGIL(string name)
         {
             var pyNs = Runtime.PyString_FromString(name);
             try
             {
-                var nsSet = Runtime.PyDict_GetItemString(new BorrowedReference(root.dict), availableNsKey);
+                var nsSet = Runtime.PyDict_GetItemString(root.DictRef, availableNsKey);
                 if (!(nsSet.IsNull  || nsSet.DangerousGetAddress() == Runtime.PyNone))
                 {
                     if (Runtime.PySet_Add(nsSet, new BorrowedReference(pyNs)) != 0)
diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs
@@ -535,8 +535,9 @@ public static Assembly AddReference(string name)
             // method because it may be called from other threads, leading to deadlocks
             // if it is called while Python code is executing.
             var currNs = AssemblyManager.GetNamespaces().Except(origNs);
-            foreach(var ns in currNs){
-                ImportHook.AddNamespace(ns);
+            foreach(var ns in currNs)
+            {
+                ImportHook.AddNamespaceWithGIL(ns);
             }
             return assembly;
         }
@@ -602,5 +603,9 @@ public static ModuleObject _load_clr_module(PyObject spec)
             mod = ImportHook.Import(modname.ToString());
             return mod;
         }
+
+        [ModuleFunction]
+        [ForbidPythonThreads]
+        public static int _add_pending_namespaces() => ImportHook.AddPendingNamespaces();
     }
 }
diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs
@@ -161,7 +161,8 @@ static void ValidateRequiredOffsetsPresent(PropertyInfo[] offsetProperties)
                 "Initialize",
                 "InitializeSlots",
                 "ListAssemblies",
-                "_load_clr_module",
+                nameof(CLRModule._load_clr_module),
+                nameof(CLRModule._add_pending_namespaces),
                 "Release",
                 "Reset",
                 "set_SuppressDocs",

Original file line number	Diff line number	Diff line change
`@@ -34,7 +34,7 @@ static IEnumerable<string[]> PythonTestCases()`
`34`	`34`	`{`
`35`	`35`	`// Add the test that you want to debug here.`
`36`	`36`	`yield return new[] { "test_enum", "test_enum_standard_attrs" };`
`37`		`- yield return new[] { "test_module", "test_explicit_assembly_load" };`
	`37`	`+ yield return new[] { "test_generic", "test_missing_generic_type" };`
`38`	`38`	`}`
`39`	`39`
`40`	`40`	`/// <summary>`
Original file line number	Diff line number	Diff line change
`@@ -290,7 +290,10 @@ internal static void ScanAssembly(Assembly assembly)`
`290`	`290`	`for (var n = 0; n < names.Length; n++)`
`291`	`291`	`{`
`292`	`292`	`s = n == 0 ? names[0] : s + "." + names[n];`
`293`		`- namespaces.TryAdd(s, new ConcurrentDictionary<Assembly, string>());`
	`293`	`+ if (namespaces.TryAdd(s, new ConcurrentDictionary<Assembly, string>()))`
	`294`	`+ {`
	`295`	`+ ImportHook.AddNamespace(s);`
	`296`	`+ }`
`294`	`297`	`}`
`295`	`298`	`}`
`296`	`299`
Original file line number	Diff line number	Diff line change
`@@ -535,8 +535,9 @@ public static Assembly AddReference(string name)`
`535`	`535`	`// method because it may be called from other threads, leading to deadlocks`
`536`	`536`	`// if it is called while Python code is executing.`
`537`	`537`	`var currNs = AssemblyManager.GetNamespaces().Except(origNs);`
`538`		`- foreach(var ns in currNs){`
`539`		`- ImportHook.AddNamespace(ns);`
	`538`	`+ foreach(var ns in currNs)`
	`539`	`+ {`
	`540`	`+ ImportHook.AddNamespaceWithGIL(ns);`
`540`	`541`	`}`
`541`	`542`	`return assembly;`
`542`	`543`	`}`
`@@ -602,5 +603,9 @@ public static ModuleObject _load_clr_module(PyObject spec)`
`602`	`603`	`mod = ImportHook.Import(modname.ToString());`
`603`	`604`	`return mod;`
`604`	`605`	`}`
	`606`	`+`
	`607`	`+ [ModuleFunction]`
	`608`	`+ [ForbidPythonThreads]`
	`609`	`+ public static int _add_pending_namespaces() => ImportHook.AddPendingNamespaces();`
`605`	`610`	`}`
`606`	`611`	`}`