Example with this PropertyDrawer context class

using Scripts.Extensions;
using Scripts.Extensions.Serialized;
using System;
#if UNITY_EDITOR
using UnityEditor;
using UnityEditorInternal;
#endif
using UnityEngine;

[SelectionBase]
public class Interactable : MonoBehaviour
{
    [field:SerializeField]
    public InteractionPoint Interaction { get; private set; }

    private void Start() => Initialize();
    private void OnEnable()
    {
        this.OnAssemblyReload(Initialize);
        TransformUpdated();
    }

    private void TransformUpdated()
    {
        Interaction.TransformUpdated(transform);
    }

    private void Initialize()
    {

    }

    [Serializable]
    public class InteractionPoint
    {
        private Matrix4x4 _localToWorld;
        private Matrix4x4 _worldToLocal;
        [SerializeField]
        private Vector3 _position;
        public Vector3 Position
        {
            get => _localToWorld.MultiplyPoint3x4(_position);
            set => _position = _worldToLocal.MultiplyPoint3x4(value);
        }
        [SerializeField]
        private Quaternion _rotation = Quaternion.identity;
        public Quaternion Rotation
        {
            get => _localToWorld.rotation * _rotation;
            set => _rotation = _worldToLocal.rotation * value;
        }

        internal void TransformUpdated(Transform transform)
        {
            _localToWorld = transform.localToWorldMatrix;
            _worldToLocal = transform.worldToLocalMatrix;
        }

#if UNITY_EDITOR
        [CustomPropertyDrawer(typeof(InteractionPoint))]
        public class InteractionPointInspector : PropertyDrawer
        {
            private Context _ctx;

            public override void OnGUI(Rect position, SerializedProperty property, GUIContent label)
            {
                var obj = property.serializedObject.targetObject as MonoBehaviour;
                var val = property.GetValue<InteractionPoint>();
                if (obj)
                    val?.TransformUpdated(obj.transform);

                position.height = 18;
                GUI.Label(position, label);
                var button = new Rect(
                    position.x + EditorGUIUtility.labelWidth, position.y,
                    position.width - EditorGUIUtility.labelWidth, position.height);
                if (!Context.TrySubscribing(property, out var ctx)) return;
                ctx.GUIDrawEditButton(button);

                EditorGUI.indentLevel++;
                position.y += 20;
                EditorGUI.PropertyField(position, property.FindPropertyRelative(nameof(InteractionPoint._position)));
                position.y += 20;
                EditorGUI.PropertyField(position, property.FindPropertyRelative(nameof(InteractionPoint._rotation)));
                EditorGUI.indentLevel--;
            }
            public override float GetPropertyHeight(SerializedProperty property, GUIContent label)
            {
                return 18 * 3 + 2 * 2;
            }
            private class Context : SceneViewDrawerExtensions.Context<InteractionPoint, Context>
            {
                private Tool _lastTool;
                private InteractionPoint _point => _field;
                protected override void OnSceneGUI()
                {
                    if (!InternalEditorUtility.GetIsInspectorExpanded(_object))
                        return;
                    if (IsEditing && Tools.current != Tool.None)
                    {
                        _lastTool = Tools.current;
                        Tools.current = Tool.None;
                    }

                    DrawEmpty(_point.Position, _point.Rotation);
                    if (IsEditing && _lastTool == Tool.Move)
                    {
                        var newPos = Handles.PositionHandle(_point.Position, _point.Rotation);
                        if (newPos != _point.Position)
                        {
                            Undo.RecordObject(_object, "HandPose MoveTool");
                            _point.Position = newPos;
                        }
                    }
                    else if (IsEditing && _lastTool == Tool.Rotate)
                    {
                        var newRot = Handles.RotationHandle(_point.Rotation, _point.Position);
                        if (newRot != _point.Rotation)
                        {
                            Undo.RecordObject(_object, "HandPose MoveTool");
                            _point.Rotation = newRot;
                        }
                    }
                }

                protected override void OnStopped()
                {
                    Tools.current = _lastTool;
                }
                private void DrawEmpty(Vector3 pos, Quaternion rot)
                {
                    var size = HandleUtility.GetHandleSize(pos);
                    Handles.color = Color.red;
                    Handles.DrawAAPolyLine(pos, pos + rot * Vector3.right * size);
                    Handles.color = Color.green;
                    Handles.DrawAAPolyLine(pos, pos + rot * Vector3.up * size);
                    Handles.color = Color.blue;
                    Handles.DrawAAPolyLine(pos, pos + rot * Vector3.forward * size);
                }
            }
        }
#endif
    }
}