From e896aa6714b9d736bea58fae7c9cbf0fd01337a2 Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 19 Dec 2019 02:49:46 +0800 Subject: [PATCH 1/2] * Decref the members of Runtime * Unified GetBuiltins method --- src/runtime/importhook.cs | 25 +----- src/runtime/runtime.cs | 170 ++++++++++++++++++++++++++------------ 2 files changed, 120 insertions(+), 75 deletions(-) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 94f0f7c94..aa3bbab6d 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -35,27 +35,6 @@ internal static void ReleaseModuleDef() } #endif - /// - /// Get a New reference to the builtins module. - /// - static IntPtr GetNewRefToBuiltins() - { - if (Runtime.IsPython3) - { - return Runtime.PyImport_ImportModule("builtins"); - } - else - { - // dict is a borrowed ref, no need to decref - IntPtr dict = Runtime.PyImport_GetModuleDict(); - - // GetItemString is a borrowed ref; incref to get a new ref - IntPtr builtins = Runtime.PyDict_GetItemString(dict, "__builtin__"); - Runtime.XIncref(builtins); - return builtins; - } - } - /// /// Initialize just the __import__ hook itself. /// @@ -64,7 +43,7 @@ static void InitImport() // We replace the built-in Python __import__ with our own: first // look in CLR modules, then if we don't find any call the default // Python __import__. - IntPtr builtins = GetNewRefToBuiltins(); + IntPtr builtins = Runtime.GetBuiltins(); py_import = Runtime.PyObject_GetAttrString(builtins, "__import__"); PythonException.ThrowIfIsNull(py_import); @@ -80,7 +59,7 @@ static void InitImport() /// static void RestoreImport() { - IntPtr builtins = GetNewRefToBuiltins(); + IntPtr builtins = Runtime.GetBuiltins(); int res = Runtime.PyObject_SetAttrString(builtins, "__import__", py_import); PythonException.ThrowIfIsNotZero(res); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 130d90c0a..2443e3edc 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -169,6 +169,8 @@ public class Runtime /// internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + private static PyReferenceCollection _pyRefs = new PyReferenceCollection(); + /// /// Initialize the runtime... /// @@ -194,99 +196,94 @@ internal static void Initialize(bool initSigs = false) TypeManager.Reset(); IntPtr op; - IntPtr dict; - if (IsPython3) - { - op = PyImport_ImportModule("builtins"); - dict = PyObject_GetAttrString(op, "__dict__"); - } - else // Python2 { - dict = PyImport_GetModuleDict(); - op = PyDict_GetItemString(dict, "__builtin__"); - } - PyNotImplemented = PyObject_GetAttrString(op, "NotImplemented"); - PyBaseObjectType = PyObject_GetAttrString(op, "object"); + var builtins = GetBuiltins(); + SetPyMember(ref PyNotImplemented, PyObject_GetAttrString(builtins, "NotImplemented")); - PyNone = PyObject_GetAttrString(op, "None"); - PyTrue = PyObject_GetAttrString(op, "True"); - PyFalse = PyObject_GetAttrString(op, "False"); + SetPyMember(ref PyBaseObjectType, PyObject_GetAttrString(builtins, "object")); - PyBoolType = PyObject_Type(PyTrue); - PyNoneType = PyObject_Type(PyNone); - PyTypeType = PyObject_Type(PyNoneType); + SetPyMember(ref PyNone, PyObject_GetAttrString(builtins, "None")); + SetPyMember(ref PyTrue, PyObject_GetAttrString(builtins, "True")); + SetPyMember(ref PyFalse, PyObject_GetAttrString(builtins, "False")); - op = PyObject_GetAttrString(dict, "keys"); - PyMethodType = PyObject_Type(op); - XDecref(op); + SetPyMember(ref PyBoolType, PyObject_Type(PyTrue)); + SetPyMember(ref PyNoneType, PyObject_Type(PyNone)); + SetPyMember(ref PyTypeType, PyObject_Type(PyNoneType)); - // For some arcane reason, builtins.__dict__.__setitem__ is *not* - // a wrapper_descriptor, even though dict.__setitem__ is. - // - // object.__init__ seems safe, though. - op = PyObject_GetAttrString(PyBaseObjectType, "__init__"); - PyWrapperDescriptorType = PyObject_Type(op); - XDecref(op); + op = PyObject_GetAttrString(builtins, "len"); + SetPyMember(ref PyMethodType, PyObject_Type(op)); + XDecref(op); -#if PYTHON3 - XDecref(dict); -#endif + // For some arcane reason, builtins.__dict__.__setitem__ is *not* + // a wrapper_descriptor, even though dict.__setitem__ is. + // + // object.__init__ seems safe, though. + op = PyObject_GetAttrString(PyBaseObjectType, "__init__"); + SetPyMember(ref PyWrapperDescriptorType, PyObject_Type(op)); + XDecref(op); + + SetPyMember(ref PySuper_Type, PyObject_GetAttrString(builtins, "super")); + + XDecref(builtins); + } op = PyString_FromString("string"); - PyStringType = PyObject_Type(op); + SetPyMember(ref PyStringType, PyObject_Type(op)); XDecref(op); op = PyUnicode_FromString("unicode"); - PyUnicodeType = PyObject_Type(op); + SetPyMember(ref PyUnicodeType, PyObject_Type(op)); XDecref(op); #if PYTHON3 op = PyBytes_FromString("bytes"); - PyBytesType = PyObject_Type(op); + SetPyMember(ref PyBytesType, PyObject_Type(op)); XDecref(op); #endif op = PyTuple_New(0); - PyTupleType = PyObject_Type(op); + SetPyMember(ref PyTupleType, PyObject_Type(op)); XDecref(op); op = PyList_New(0); - PyListType = PyObject_Type(op); + SetPyMember(ref PyListType, PyObject_Type(op)); XDecref(op); op = PyDict_New(); - PyDictType = PyObject_Type(op); + SetPyMember(ref PyDictType, PyObject_Type(op)); XDecref(op); op = PyInt_FromInt32(0); - PyIntType = PyObject_Type(op); + SetPyMember(ref PyIntType, PyObject_Type(op)); XDecref(op); op = PyLong_FromLong(0); - PyLongType = PyObject_Type(op); + SetPyMember(ref PyLongType, PyObject_Type(op)); XDecref(op); op = PyFloat_FromDouble(0); - PyFloatType = PyObject_Type(op); + SetPyMember(ref PyFloatType, PyObject_Type(op)); XDecref(op); -#if PYTHON3 +#if !PYTHON2 PyClassType = IntPtr.Zero; PyInstanceType = IntPtr.Zero; -#elif PYTHON2 - IntPtr s = PyString_FromString("_temp"); - IntPtr d = PyDict_New(); +#else + { + IntPtr s = PyString_FromString("_temp"); + IntPtr d = PyDict_New(); - IntPtr c = PyClass_New(IntPtr.Zero, d, s); - PyClassType = PyObject_Type(c); + IntPtr c = PyClass_New(IntPtr.Zero, d, s); + SetPyMember(ref PyClassType, PyObject_Type(c)); - IntPtr i = PyInstance_New(c, IntPtr.Zero, IntPtr.Zero); - PyInstanceType = PyObject_Type(i); + IntPtr i = PyInstance_New(c, IntPtr.Zero, IntPtr.Zero); + SetPyMember(ref PyInstanceType, PyObject_Type(i)); - XDecref(s); - XDecref(i); - XDecref(c); - XDecref(d); + XDecref(s); + XDecref(i); + XDecref(c); + XDecref(d); + } #endif Error = new IntPtr(-1); @@ -380,6 +377,9 @@ internal static void Shutdown() Exceptions.Shutdown(); ImportHook.Shutdown(); Finalizer.Shutdown(); + // TOOD: PyCLRMetaType's release operation still in #958 + PyCLRMetaType = IntPtr.Zero; + ResetPyMembers(); Py_Finalize(); } @@ -393,6 +393,19 @@ internal static int AtExit() return 0; } + private static void SetPyMember(ref IntPtr obj, IntPtr value) + { + // XXX: For current usages, value should not be null. + PythonException.ThrowIfIsNull(value); + obj = value; + _pyRefs.Add(ref obj); + } + + private static void ResetPyMembers() + { + _pyRefs.Release(); + } + internal static IntPtr Py_single_input = (IntPtr)256; internal static IntPtr Py_file_input = (IntPtr)257; internal static IntPtr Py_eval_input = (IntPtr)258; @@ -401,6 +414,7 @@ internal static int AtExit() internal static IntPtr PyModuleType; internal static IntPtr PyClassType; internal static IntPtr PyInstanceType; + internal static IntPtr PySuper_Type; internal static IntPtr PyCLRMetaType; internal static IntPtr PyMethodType; internal static IntPtr PyWrapperDescriptorType; @@ -1746,6 +1760,9 @@ internal static bool PyIter_Check(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyImport_Import(IntPtr name); + /// + /// Return value: New reference. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyImport_ImportModule(string name); @@ -1945,5 +1962,54 @@ internal static void SetNoSiteFlag() } } } + + /// + /// Return value: New reference. + /// + internal static IntPtr GetBuiltins() + { + return IsPython3 ? PyImport_ImportModule("builtins") + : PyImport_ImportModule("__builtin__"); + } + } + + + class PyReferenceCollection + { + public List _objects { get; private set; } + + public PyReferenceCollection() + { + _objects = new List(); + } + + /// + /// Record obj's address to release the obj in the future, + /// obj must alive before calling Release. + /// + public void Add(ref IntPtr obj) + { + unsafe + { + fixed (void* p = &obj) + { + _objects.Add((IntPtr)p); + } + } + } + + public void Release() + { + foreach (var objRef in _objects) + { + unsafe + { + var p = (void**)objRef; + Runtime.XDecref((IntPtr)(*p)); + *p = null; + } + } + _objects.Clear(); + } } } From ab0cb02fb8306f15beb9b734a8fbf56b2708ad26 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sat, 21 Dec 2019 11:56:30 +0800 Subject: [PATCH 2/2] Add explicit release action --- src/runtime/runtime.cs | 121 ++++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 57 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 2443e3edc..7748bafa9 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -198,20 +198,29 @@ internal static void Initialize(bool initSigs = false) IntPtr op; { var builtins = GetBuiltins(); - SetPyMember(ref PyNotImplemented, PyObject_GetAttrString(builtins, "NotImplemented")); - - SetPyMember(ref PyBaseObjectType, PyObject_GetAttrString(builtins, "object")); - - SetPyMember(ref PyNone, PyObject_GetAttrString(builtins, "None")); - SetPyMember(ref PyTrue, PyObject_GetAttrString(builtins, "True")); - SetPyMember(ref PyFalse, PyObject_GetAttrString(builtins, "False")); - - SetPyMember(ref PyBoolType, PyObject_Type(PyTrue)); - SetPyMember(ref PyNoneType, PyObject_Type(PyNone)); - SetPyMember(ref PyTypeType, PyObject_Type(PyNoneType)); + SetPyMember(ref PyNotImplemented, PyObject_GetAttrString(builtins, "NotImplemented"), + () => PyNotImplemented = IntPtr.Zero); + + SetPyMember(ref PyBaseObjectType, PyObject_GetAttrString(builtins, "object"), + () => PyBaseObjectType = IntPtr.Zero); + + SetPyMember(ref PyNone, PyObject_GetAttrString(builtins, "None"), + () => PyNone = IntPtr.Zero); + SetPyMember(ref PyTrue, PyObject_GetAttrString(builtins, "True"), + () => PyTrue = IntPtr.Zero); + SetPyMember(ref PyFalse, PyObject_GetAttrString(builtins, "False"), + () => PyFalse = IntPtr.Zero); + + SetPyMember(ref PyBoolType, PyObject_Type(PyTrue), + () => PyBoolType = IntPtr.Zero); + SetPyMember(ref PyNoneType, PyObject_Type(PyNone), + () => PyNoneType = IntPtr.Zero); + SetPyMember(ref PyTypeType, PyObject_Type(PyNoneType), + () => PyTypeType = IntPtr.Zero); op = PyObject_GetAttrString(builtins, "len"); - SetPyMember(ref PyMethodType, PyObject_Type(op)); + SetPyMember(ref PyMethodType, PyObject_Type(op), + () => PyMethodType = IntPtr.Zero); XDecref(op); // For some arcane reason, builtins.__dict__.__setitem__ is *not* @@ -219,50 +228,61 @@ internal static void Initialize(bool initSigs = false) // // object.__init__ seems safe, though. op = PyObject_GetAttrString(PyBaseObjectType, "__init__"); - SetPyMember(ref PyWrapperDescriptorType, PyObject_Type(op)); + SetPyMember(ref PyWrapperDescriptorType, PyObject_Type(op), + () => PyWrapperDescriptorType = IntPtr.Zero); XDecref(op); - SetPyMember(ref PySuper_Type, PyObject_GetAttrString(builtins, "super")); + SetPyMember(ref PySuper_Type, PyObject_GetAttrString(builtins, "super"), + () => PySuper_Type = IntPtr.Zero); XDecref(builtins); } op = PyString_FromString("string"); - SetPyMember(ref PyStringType, PyObject_Type(op)); + SetPyMember(ref PyStringType, PyObject_Type(op), + () => PyStringType = IntPtr.Zero); XDecref(op); op = PyUnicode_FromString("unicode"); - SetPyMember(ref PyUnicodeType, PyObject_Type(op)); + SetPyMember(ref PyUnicodeType, PyObject_Type(op), + () => PyUnicodeType = IntPtr.Zero); XDecref(op); #if PYTHON3 op = PyBytes_FromString("bytes"); - SetPyMember(ref PyBytesType, PyObject_Type(op)); + SetPyMember(ref PyBytesType, PyObject_Type(op), + () => PyBytesType = IntPtr.Zero); XDecref(op); #endif op = PyTuple_New(0); - SetPyMember(ref PyTupleType, PyObject_Type(op)); + SetPyMember(ref PyTupleType, PyObject_Type(op), + () => PyTupleType = IntPtr.Zero); XDecref(op); op = PyList_New(0); - SetPyMember(ref PyListType, PyObject_Type(op)); + SetPyMember(ref PyListType, PyObject_Type(op), + () => PyListType = IntPtr.Zero); XDecref(op); op = PyDict_New(); - SetPyMember(ref PyDictType, PyObject_Type(op)); + SetPyMember(ref PyDictType, PyObject_Type(op), + () => PyDictType = IntPtr.Zero); XDecref(op); op = PyInt_FromInt32(0); - SetPyMember(ref PyIntType, PyObject_Type(op)); + SetPyMember(ref PyIntType, PyObject_Type(op), + () => PyIntType = IntPtr.Zero); XDecref(op); op = PyLong_FromLong(0); - SetPyMember(ref PyLongType, PyObject_Type(op)); + SetPyMember(ref PyLongType, PyObject_Type(op), + () => PyLongType = IntPtr.Zero); XDecref(op); op = PyFloat_FromDouble(0); - SetPyMember(ref PyFloatType, PyObject_Type(op)); + SetPyMember(ref PyFloatType, PyObject_Type(op), + () => PyFloatType = IntPtr.Zero); XDecref(op); #if !PYTHON2 @@ -274,10 +294,12 @@ internal static void Initialize(bool initSigs = false) IntPtr d = PyDict_New(); IntPtr c = PyClass_New(IntPtr.Zero, d, s); - SetPyMember(ref PyClassType, PyObject_Type(c)); + SetPyMember(ref PyClassType, PyObject_Type(c), + () => PyClassType = IntPtr.Zero); IntPtr i = PyInstance_New(c, IntPtr.Zero, IntPtr.Zero); - SetPyMember(ref PyInstanceType, PyObject_Type(i)); + SetPyMember(ref PyInstanceType, PyObject_Type(i), + () => PyInstanceType = IntPtr.Zero); XDecref(s); XDecref(i); @@ -393,12 +415,12 @@ internal static int AtExit() return 0; } - private static void SetPyMember(ref IntPtr obj, IntPtr value) + private static void SetPyMember(ref IntPtr obj, IntPtr value, Action onRelease) { // XXX: For current usages, value should not be null. PythonException.ThrowIfIsNull(value); obj = value; - _pyRefs.Add(ref obj); + _pyRefs.Add(value, onRelease); } private static void ResetPyMembers() @@ -977,7 +999,7 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) internal static long PyObject_Size(IntPtr pointer) { - return (long) _PyObject_Size(pointer); + return (long)_PyObject_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyObject_Size")] @@ -1093,7 +1115,7 @@ internal static bool PyLong_Check(IntPtr ob) internal static IntPtr PyLong_FromUnsignedLong(object value) { - if(Is32Bit || IsWindows) + if (Is32Bit || IsWindows) return PyLong_FromUnsignedLong32(Convert.ToUInt32(value)); else return PyLong_FromUnsignedLong64(Convert.ToUInt64(value)); @@ -1283,7 +1305,7 @@ internal static int PySequence_DelSlice(IntPtr pointer, long i1, long i2) internal static long PySequence_Size(IntPtr pointer) { - return (long) _PySequence_Size(pointer); + return (long)_PySequence_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Size")] @@ -1308,7 +1330,7 @@ internal static IntPtr PySequence_Repeat(IntPtr pointer, long count) internal static long PySequence_Count(IntPtr pointer, IntPtr value) { - return (long) _PySequence_Count(pointer, value); + return (long)_PySequence_Count(pointer, value); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Count")] @@ -1351,7 +1373,7 @@ internal static IntPtr PyString_FromString(string value) internal static long PyBytes_Size(IntPtr op) { - return (long) _PyBytes_Size(op); + return (long)_PyBytes_Size(op); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyBytes_Size")] @@ -1582,7 +1604,7 @@ internal static bool PyDict_Check(IntPtr ob) internal static long PyDict_Size(IntPtr pointer) { - return (long) _PyDict_Size(pointer); + return (long)_PyDict_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyDict_Size")] @@ -1660,7 +1682,7 @@ internal static int PyList_SetSlice(IntPtr pointer, long start, long end, IntPtr internal static long PyList_Size(IntPtr pointer) { - return (long) _PyList_Size(pointer); + return (long)_PyList_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyList_Size")] @@ -1709,7 +1731,7 @@ internal static IntPtr PyTuple_GetSlice(IntPtr pointer, long start, long end) internal static long PyTuple_Size(IntPtr pointer) { - return (long) _PyTuple_Size(pointer); + return (long)_PyTuple_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyTuple_Size")] @@ -1976,40 +1998,25 @@ internal static IntPtr GetBuiltins() class PyReferenceCollection { - public List _objects { get; private set; } - - public PyReferenceCollection() - { - _objects = new List(); - } + private List> _actions = new List>(); /// /// Record obj's address to release the obj in the future, /// obj must alive before calling Release. /// - public void Add(ref IntPtr obj) + public void Add(IntPtr ob, Action onRelease) { - unsafe - { - fixed (void* p = &obj) - { - _objects.Add((IntPtr)p); - } - } + _actions.Add(new KeyValuePair(ob, onRelease)); } public void Release() { - foreach (var objRef in _objects) + foreach (var item in _actions) { - unsafe - { - var p = (void**)objRef; - Runtime.XDecref((IntPtr)(*p)); - *p = null; - } + Runtime.XDecref(item.Key); + item.Value?.Invoke(); } - _objects.Clear(); + _actions.Clear(); } } }