Implement support for DLR get/set

(cherry picked from commit 5dacfb4)

Author: filmor

src/runtime/InteropConfiguration.cs

@@ -20,9 +20,9 @@ public static InteropConfiguration MakeDefault()
             {
                 PythonBaseTypeProviders =
                 {
-                    DefaultBaseTypeProvider.Instance
-                    // see https://github.com/pythonnet/pythonnet/issues/1785
-                    // new CollectionMixinsProvider(new Lazy<PyObject>(() => Py.Import("clr._extras.collections"))),
+                    DefaultBaseTypeProvider.Instance,
+                    new CollectionMixinsProvider(new Lazy<PyObject>(() => Py.Import("clr._extras.collections"))),
+                    new DynamicObjectMixinsProvider(new Lazy<PyObject>(() => Py.Import("clr._extras.dlr"))),
                 },
             };
         }

src/runtime/Mixins/DynamicObjectMixinsProvider.cs

@@ -0,0 +1,47 @@
+using System;
+using System.Collections.Generic;
+using System.Dynamic;
+
+namespace Python.Runtime.Mixins;
+
+class DynamicObjectMixinsProvider : IPythonBaseTypeProvider, IDisposable
+{
+    readonly Lazy<PyObject> mixinsModule;
+
+    public DynamicObjectMixinsProvider(Lazy<PyObject> mixinsModule) =>
+		this.mixinsModule = mixinsModule ?? throw new ArgumentNullException(nameof(mixinsModule));
+
+    public PyObject Mixins => mixinsModule.Value;
+
+    public IEnumerable<PyType> GetBaseTypes(Type type, IList<PyType> existingBases)
+    {
+        if (type is null)
+            throw new ArgumentNullException(nameof(type));
+
+        if (existingBases is null)
+            throw new ArgumentNullException(nameof(existingBases));
+
+        if (!typeof(IDynamicMetaObjectProvider).IsAssignableFrom(type))
+            return existingBases;
+
+        var newBases = new List<PyType>(existingBases)
+        {
+            new(Mixins.GetAttr("DynamicMetaObjectProviderMixin"))
+        };
+
+        if (type.IsInterface && type.BaseType is null)
+        {
+            newBases.RemoveAll(@base => PythonReferenceComparer.Instance.Equals(@base, Runtime.PyBaseObjectType));
+        }
+
+        return newBases;
+    }
+
+    public void Dispose()
+    {
+        if (this.mixinsModule.IsValueCreated)
+        {
+            this.mixinsModule.Value.Dispose();
+        }
+    }
+}

src/runtime/Mixins/dlr.py

@@ -0,0 +1,18 @@
+"""
+Implements helpers for Dynamic Language Runtime (DLR) types.
+"""
+
+class DynamicMetaObjectProviderMixin:
+    def __dir__(self):
+        names = set(super().__dir__())
+
+        get_dynamic_member_names = getattr(self, "GetDynamicMemberNames", None)
+        if callable(get_dynamic_member_names):
+            try:
+                for name in get_dynamic_member_names():
+                    if isinstance(name, str):
+                        names.add(name)
+            except Exception:
+                pass
+
+        return list(sorted(names))

src/runtime/PythonEngine.cs

@@ -291,7 +291,7 @@ static void LoadSubmodule(BorrowedReference targetModuleDict, string fullName, s
 
         static void LoadMixins(BorrowedReference targetModuleDict)
         {
-            foreach (string nested in new[] { "collections" })
+            foreach (string nested in new[] { "collections", "dlr" })
             {
                 LoadSubmodule(targetModuleDict,
                     fullName: "clr._extras." + nested,

src/runtime/TypeManager.cs

@@ -1,9 +1,11 @@
 using System;
 using System.Collections.Generic;
+using System.Dynamic;
 using System.Linq;
 using System.Reflection;
 using System.Runtime.InteropServices;
 using System.Diagnostics;
+
 using Python.Runtime.Native;
 using Python.Runtime.StateSerialization;
 
@@ -37,10 +39,164 @@ internal class TypeManager
             "tp_clear",
         };
 
+        static readonly DynamicObjectMemberAccessor dynamicMemberAccessor = new();
+
+        static bool HasClrMember(object instance, string memberName) =>
+            instance.GetType().GetMember(memberName, BindingFlags.Public | BindingFlags.Instance).Length > 0;
+
+        static bool IsPythonSpecialAttributeName(string memberName) =>
+            memberName.Length > 4 && memberName.StartsWith("__") && memberName.EndsWith("__");
+
+        static bool TryGetDynamicInstance(BorrowedReference ob, out object instance, out IDynamicMetaObjectProvider dynamicObject)
+        {
+            if (ManagedType.GetManagedObject(ob) is CLRObject co && co.inst is IDynamicMetaObjectProvider coDynamic)
+            {
+                instance = co.inst;
+                dynamicObject = coDynamic;
+                return true;
+            }
+
+            if (Converter.ToManaged(ob, typeof(IDynamicMetaObjectProvider), out object? managedDynamic, false)
+                && managedDynamic is IDynamicMetaObjectProvider convertedDynamic)
+            {
+                instance = managedDynamic;
+                dynamicObject = convertedDynamic;
+                return true;
+            }
+
+            if (Converter.ToManaged(ob, typeof(object), out object? managedInstance, false)
+                && managedInstance is IDynamicMetaObjectProvider boxedDynamic)
+            {
+                instance = managedInstance;
+                dynamicObject = boxedDynamic;
+                return true;
+            }
+
+            instance = null!;
+            dynamicObject = null!;
+            return false;
+        }
+
+        public static NewReference tp_getattro_dlr_proxy(BorrowedReference ob, BorrowedReference key)
+        {
+            var isDynamic = TryGetDynamicInstance(ob, out object instance, out IDynamicMetaObjectProvider dynamicObject);
+
+            // The whole DLR machinery only makes sense with string keys and dynamic objects
+            if (!isDynamic || !Runtime.PyString_Check(key))
+            {
+                return Runtime.PyObject_GenericGetAttr(ob, key);
+            }
+
+            string memberName = Runtime.GetManagedString(key)!;
+
+            // Forward requests to GetDynamicMemberNames to the mixin implementation
+            if (memberName == nameof(DynamicObjectMemberAccessor.GetDynamicMemberNames)
+                && !HasClrMember(instance, memberName))
+            {
+                using var pyMemberNames = new Func<IReadOnlyCollection<string>>(
+                    () => dynamicMemberAccessor.GetDynamicMemberNames(dynamicObject)
+                ).ToPython();
+                return pyMemberNames.NewReferenceOrNull();
+            }
+
+            // Now, first try to access the Python attribute
+            var attr = Runtime.PyObject_GenericGetAttr(ob, key);
+            if (!attr.IsNull())
+                return attr;
+
+            // attr is null, so an exception must be set. If that exception is not an AttributeError,
+            // we return from this function immediately without clearing. All later returns until the
+            // very end will lead to the AttributeError getting raised.
+            if (Runtime.PyErr_ExceptionMatches(Exceptions.AttributeError) == 0)
+            {
+                return default;
+            }
+
+            if (HasClrMember(instance, memberName) || IsPythonSpecialAttributeName(memberName))
+            {
+                return default;
+            }
+
+            bool resolved = false;
+            object? value = null;
+            try
+            {
+                resolved = dynamicMemberAccessor.TryGetMember(dynamicObject, memberName, out value);
+            }
+            catch
+            {
+                return default;
+            }
+
+            if (!resolved)
+            {
+                return default;
+            }
+
+            Runtime.PyErr_Clear();
+
+            using var pyValue = value.ToPython();
+            return pyValue.NewReferenceOrNull();
+        }
+
+        public static int tp_setattro_dlr_proxy(BorrowedReference ob, BorrowedReference key, BorrowedReference val)
+        {
+            var isDynamic = TryGetDynamicInstance(ob, out object instance, out IDynamicMetaObjectProvider dynamicObject);
+
+            // The whole DLR machinery only makes sense with string keys and dynamic objects
+            if (!isDynamic || !Runtime.PyString_Check(key))
+            {
+                return Runtime.PyObject_GenericSetAttr(ob, key, val);
+            }
+
+            string memberName = Runtime.GetManagedString(key)!;
+
+            // For Python-derived types (IPythonDerivedType), the Python descriptor protocol
+            // (e.g. @property setters) takes priority over DLR member storage.
+            if (instance is IPythonDerivedType)
+            {
+                int pyResult = Runtime.PyObject_GenericSetAttr(ob, key, val);
+                if (pyResult == 0)
+                    return 0;
+
+                if (Runtime.PyErr_ExceptionMatches(Exceptions.AttributeError) == 0)
+                    return pyResult;
+
+                Runtime.PyErr_Clear();
+                // Fall through to DLR fallback below
+            }
+
+            if (!HasClrMember(instance, memberName) && !IsPythonSpecialAttributeName(memberName))
+            {
+                // Try DLR member storage first
+                bool handled = false;
+
+                if (val == null)
+                {
+                    handled = dynamicMemberAccessor.TryDeleteMember(dynamicObject, memberName);
+                }
+                else
+                {
+                    object? managedValue = null;
+                    if (val != Runtime.PyNone && !Converter.ToManaged(val, typeof(object), out managedValue, true))
+                        return -1;
+
+                    handled = dynamicMemberAccessor.TrySetMember(dynamicObject, memberName, managedValue);
+                }
+
+                if (handled)
+                    return 0;
+            }
+
+            // Fall back to Python attribute setting
+            return Runtime.PyObject_GenericSetAttr(ob, key, val);
+        }
+
         internal static void Initialize()
         {
             Debug.Assert(cache.Count == 0, "Cache should be empty",
                 "Some errors may occurred on last shutdown");
+            dynamicMemberAccessor.Clear();
             using (var plainType = SlotHelper.CreateObjectType())
             {
                 subtype_traverse = Util.ReadIntPtr(plainType.Borrow(), TypeOffset.tp_traverse);
@@ -64,6 +220,8 @@ internal static void RemoveTypes()
                 }
             }
 
+            dynamicMemberAccessor.Clear();
+
             foreach (var type in cache.Values)
             {
                 type.Dispose();
@@ -296,6 +454,13 @@ internal static void InitializeClass(PyType type, ClassBase impl, Type clrType)
                 throw PythonException.ThrowLastAsClrException();
             }
 
+            if (typeof(IDynamicMetaObjectProvider).IsAssignableFrom(clrType))
+            {
+                InitializeSlot(type, TypeOffset.tp_getattro, new Interop.BB_N(tp_getattro_dlr_proxy), slotsHolder);
+                InitializeSlot(type, TypeOffset.tp_setattro, new Interop.BBB_I32(tp_setattro_dlr_proxy), slotsHolder);
+                Runtime.PyType_Modified(type.Reference);
+            }
+
             var dict = Util.ReadRef(type, TypeOffset.tp_dict);
             string mn = clrType.Namespace ?? "";
             using (var mod = Runtime.PyString_FromString(mn))

src/runtime/Types/ClassDerived.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.ComponentModel;
+using System.Dynamic;
 using System.Diagnostics;
 using System.Linq;
 using System.Reflection;
@@ -227,6 +228,13 @@ internal static Type CreateDerivedType(string name,
                     continue;
                 }
 
+                // Avoid re-entrant DLR binder recursion when Python derives from
+                // DynamicObject-based types (including overrides in intermediate bases).
+                if (IsDynamicObjectHookMethod(method))
+                {
+                    continue;
+                }
+
                 // skip if this property has already been overridden
                 if ((method.Name.StartsWith("get_") || method.Name.StartsWith("set_"))
                     && pyProperties.Contains(method.Name.Substring(4)))
@@ -298,6 +306,35 @@ internal static Type CreateDerivedType(string name,
             return type;
         }
 
+        static bool IsDynamicObjectHookMethod(MethodInfo method)
+        {
+            MethodInfo origin = method.GetBaseDefinition();
+            Type? originType = origin.DeclaringType;
+            if (originType == typeof(DynamicObject))
+            {
+                return origin.Name switch
+                {
+                    nameof(DynamicObject.TryGetMember)
+                    or nameof(DynamicObject.TrySetMember)
+                    or nameof(DynamicObject.TryDeleteMember)
+                    or nameof(DynamicObject.TryInvokeMember)
+                    or nameof(DynamicObject.TryConvert)
+                    or nameof(DynamicObject.TryGetIndex)
+                    or nameof(DynamicObject.TrySetIndex)
+                    or nameof(DynamicObject.GetDynamicMemberNames)
+                    or nameof(IDynamicMetaObjectProvider.GetMetaObject) => true,
+                    _ => false,
+                };
+            }
+
+            if (originType == typeof(IDynamicMetaObjectProvider))
+            {
+                return origin.Name == nameof(IDynamicMetaObjectProvider.GetMetaObject);
+            }
+
+            return false;
+        }
+
         /// <summary>
         /// Add a constructor override that calls the python ctor after calling the base type constructor.
         /// </summary>