btrace, python: Enable ptwrite filter registration.

By default GDB will be printing the hex payload of the ptwrite package as
auxiliary information.  To customize this, the user can register a ptwrite
filter function in python, that takes the payload and the PC as arguments and
returns a string which will be printed instead.  Registering the filter
function is done using a factory pattern to make per-thread filtering easier.

Approved-By: Markus Metzger <markus.t.metzger@intel.com>

gdb/python/lib/gdb/ptwrite.py

@@ -0,0 +1,77 @@
+# Ptwrite utilities.
+# Copyright (C) 2023 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Utilities for working with ptwrite filters."""
+
+import gdb
+
+# _ptwrite_filter contains the per thread copies of the filter function.
+# The keys are tuples of inferior id and thread id.
+# The filter functions are created for each thread by calling the
+# _ptwrite_filter_factory.
+_ptwrite_filter = {}
+_ptwrite_filter_factory = None
+
+
+def _ptwrite_exit_handler(event):
+    """Exit handler to prune _ptwrite_filter on thread exit."""
+    _ptwrite_filter.pop(event.inferior_thread.ptid, None)
+
+
+gdb.events.thread_exited.connect(_ptwrite_exit_handler)
+
+
+def _clear_traces():
+    """Helper function to clear the trace of all threads."""
+    current_thread = gdb.selected_thread()
+
+    for inferior in gdb.inferiors():
+        for thread in inferior.threads():
+            thread.switch()
+            recording = gdb.current_recording()
+            if recording is not None:
+                recording.clear()
+
+    current_thread.switch()
+
+
+def register_filter_factory(filter_factory_):
+    """Register the ptwrite filter factory."""
+    if filter_factory_ is not None and not callable(filter_factory_):
+        raise TypeError("The filter factory must be callable or 'None'.")
+
+    # Clear the traces of all threads of all inferiors to force
+    # re-decoding with the new filter.
+    _clear_traces()
+
+    _ptwrite_filter.clear()
+    global _ptwrite_filter_factory
+    _ptwrite_filter_factory = filter_factory_
+
+
+def get_filter():
+    """Returns the filter of the current thread."""
+    thread = gdb.selected_thread()
+    key = thread.ptid
+
+    # Create a new filter for new threads.
+    if key not in _ptwrite_filter:
+        if _ptwrite_filter_factory is not None:
+            _ptwrite_filter[key] = _ptwrite_filter_factory(thread)
+        else:
+            return None
+
+    return _ptwrite_filter[key]

gdb/python/py-record-btrace.c

@@ -808,6 +808,109 @@ recpy_bt_function_call_history (PyObject *self, void *closure)
   return btpy_list_new (tinfo, first, last, 1, &recpy_func_type);
 }
 
+/* Helper function that calls PTW_FILTER with PAYLOAD and IP as arguments.
+   Returns the string that will be printed, if there is a filter to call.  */
+static std::optional<std::string>
+recpy_call_filter (const uint64_t payload, const uint64_t ip,
+		   const void *ptw_filter)
+{
+  std::optional<std::string> result;
+
+  gdb_assert (ptw_filter != nullptr);
+  if ((PyObject *) ptw_filter == Py_None)
+    return result;
+
+  gdbpy_enter enter_py;
+
+  gdbpy_ref<> py_payload = gdb_py_object_from_ulongest (payload);
+
+  gdbpy_ref<> py_ip;
+  if (ip == 0)
+    py_ip = gdbpy_ref<>::new_reference (Py_None);
+  else
+    py_ip = gdb_py_object_from_ulongest (ip);
+
+  gdbpy_ref<> py_result (PyObject_CallFunctionObjArgs ((PyObject *) ptw_filter,
+							py_payload.get (),
+							py_ip.get (),
+							nullptr));
+
+  if (py_result == nullptr)
+    {
+      gdbpy_print_stack ();
+      gdbpy_error (_("Couldn't call the ptwrite filter."));
+    }
+
+  /* Py_None is valid and results in no output.  */
+  if (py_result == Py_None)
+    {
+      result = "";
+      return result;
+    }
+
+  gdb::unique_xmalloc_ptr<char> user_string
+    = gdbpy_obj_to_string (py_result.get ());
+
+  if (user_string == nullptr)
+    {
+      gdbpy_print_stack ();
+      gdbpy_error (_("The ptwrite filter didn't return a string."));
+    }
+  else
+    result = user_string.get ();
+
+  return result;
+}
+
+/* Helper function returning the current ptwrite filter.  */
+
+static PyObject *
+get_ptwrite_filter ()
+{
+  gdbpy_ref<> module (PyImport_ImportModule ("gdb.ptwrite"));
+
+  if (PyErr_Occurred ())
+  {
+    gdbpy_print_stack ();
+    gdbpy_error (_("Couldn't import gdb.ptwrite."));
+  }
+
+  /* We need to keep the reference count.  */
+  gdbpy_ref<> ptw_filter (gdbpy_call_method (module.get (), "get_filter"));
+
+  if (PyErr_Occurred ())
+    {
+      gdbpy_print_stack ();
+      gdbpy_error (_("Couldn't get the ptwrite filter."));
+    }
+
+  return ptw_filter.get();
+}
+
+/* Used for registering any python ptwrite filter to the current thread.  A
+   pointer to this function is stored in the python extension interface.  */
+
+void
+gdbpy_load_ptwrite_filter (const struct extension_language_defn *extlang,
+			   struct btrace_thread_info *btinfo)
+{
+  gdb_assert (btinfo != nullptr);
+
+  gdbpy_enter enter_py;
+
+  btinfo->ptw_context = get_ptwrite_filter ();
+
+#if defined (HAVE_STRUCT_PT_EVENT_VARIANT_PTWRITE)
+  if (!btinfo->target->conf.pt.ptwrite && btinfo->ptw_context != Py_None)
+    warning (_("The target doesn't support decoding ptwrite events."));
+#else
+  if (btinfo->ptw_context != Py_None)
+    warning (_("Libipt doesn't support decoding ptwrite events."));
+#endif /* defined (HAVE_STRUCT_PT_EVENT_VARIANT_PTWRITE) */
+
+  btinfo->ptw_callback_fun = &recpy_call_filter;
+}
+
 /* Implementation of BtraceRecord.goto (self, BtraceInstruction) -> None.  */
 
 PyObject *

gdb/python/python-internal.h

@@ -464,6 +464,9 @@ extern enum ext_lang_rc gdbpy_apply_val_pretty_printer
    struct ui_file *stream, int recurse,
    const struct value_print_options *options,
    const struct language_defn *language);
+extern void gdbpy_load_ptwrite_filter
+  (const struct extension_language_defn *extlang,
+   struct btrace_thread_info *btinfo);
 extern enum ext_lang_bt_status gdbpy_apply_frame_filter
   (const struct extension_language_defn *,
    const frame_info_ptr &frame, frame_filter_flags flags,

gdb/python/python.c

@@ -159,6 +159,8 @@ static const struct extension_language_ops python_extension_ops =
 
   gdbpy_apply_frame_filter,
 
+  gdbpy_load_ptwrite_filter,
+
   gdbpy_preserve_values,
 
   gdbpy_breakpoint_has_cond,