Merge remote-tracking branch 'living180/stacktrace_rework'

* living180/stacktrace_rework:
  Update change log
  Add deprecation warnings to old stack trace functions
  Use new stack trace functionality for CachePanel
  Use new stack trace functionality for SQLPanel
  Add new stack trace functionality

Author: matthiask

debug_toolbar/middleware.py

@@ -10,6 +10,7 @@
 
 from debug_toolbar import settings as dt_settings
 from debug_toolbar.toolbar import DebugToolbar
+from debug_toolbar.utils import clear_stack_trace_caches
 
 _HTML_TYPES = ("text/html", "application/xhtml+xml")
 
@@ -56,6 +57,7 @@ def __call__(self, request):
             # Run panels like Django middleware.
             response = toolbar.process_request(request)
         finally:
+            clear_stack_trace_caches()
             # Deactivate instrumentation ie. monkey-unpatch. This must run
             # regardless of the response. Keep 'return' clauses below.
             for panel in reversed(toolbar.enabled_panels):

debug_toolbar/panels/cache.py

@@ -6,14 +6,8 @@
 from django.core.cache import CacheHandler, caches
 from django.utils.translation import gettext_lazy as _, ngettext
 
-from debug_toolbar import settings as dt_settings
 from debug_toolbar.panels import Panel
-from debug_toolbar.utils import (
-    get_stack,
-    get_template_info,
-    render_stacktrace,
-    tidy_stacktrace,
-)
+from debug_toolbar.utils import get_stack_trace, get_template_info, render_stacktrace
 
 # The order of the methods in this list determines the order in which they are listed in
 # the Commands table in the panel content.
@@ -135,18 +129,13 @@ def _record_call(self, cache, name, original_method, args, kwargs):
         t = time.time() - t
         cache._djdt_recording = False
 
-        if dt_settings.get_config()["ENABLE_STACKTRACES"]:
-            stacktrace = tidy_stacktrace(reversed(get_stack()))
-        else:
-            stacktrace = []
-
         self._store_call_info(
             name=name,
             time_taken=t,
             return_value=value,
             args=args,
             kwargs=kwargs,
-            trace=stacktrace,
+            trace=get_stack_trace(),
             template_info=get_template_info(),
             backend=cache,
         )

debug_toolbar/panels/sql/tracking.py

@@ -6,7 +6,7 @@
 from django.utils.encoding import force_str
 
 from debug_toolbar import settings as dt_settings
-from debug_toolbar.utils import get_stack, get_template_info, tidy_stacktrace
+from debug_toolbar.utils import get_stack_trace, get_template_info
 
 try:
     from psycopg2._json import Json as PostgresJson
@@ -155,10 +155,6 @@ def _record(self, method, sql, params):
         finally:
             stop_time = time()
             duration = (stop_time - start_time) * 1000
-            if dt_settings.get_config()["ENABLE_STACKTRACES"]:
-                stacktrace = tidy_stacktrace(reversed(get_stack()))
-            else:
-                stacktrace = []
             _params = ""
             try:
                 _params = json.dumps(self._decode(params))
@@ -180,7 +176,7 @@ def _record(self, method, sql, params):
                 "raw_sql": sql,
                 "params": _params,
                 "raw_params": params,
-                "stacktrace": stacktrace,
+                "stacktrace": get_stack_trace(),
                 "start_time": start_time,
                 "stop_time": stop_time,
                 "is_slow": duration > dt_settings.get_config()["SQL_WARNING_THRESHOLD"],

debug_toolbar/utils.py

@@ -1,10 +1,13 @@
 import inspect
+import linecache
 import os.path
 import sys
+import warnings
 from importlib import import_module
 from pprint import pformat
 
 import django
+from asgiref.local import Local
 from django.core.exceptions import ImproperlyConfigured
 from django.template import Node
 from django.utils.html import format_html
@@ -18,6 +21,9 @@
     threading = None
 
 
+_local_data = Local()
+
+
 # Figure out some paths
 django_path = os.path.realpath(os.path.dirname(django.__file__))
 
@@ -44,6 +50,15 @@ def omit_path(path):
     return any(path.startswith(hidden_path) for hidden_path in hidden_paths)
 
 
+def _stack_trace_deprecation_warning():
+    warnings.warn(
+        "get_stack() and tidy_stacktrace() are deprecated in favor of"
+        " get_stack_trace()",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
+
 def tidy_stacktrace(stack):
     """
     Clean up stacktrace and remove all entries that are excluded by the
@@ -52,6 +67,8 @@ def tidy_stacktrace(stack):
     ``stack`` should be a list of frame tuples from ``inspect.stack()`` or
     ``debug_toolbar.utils.get_stack()``.
     """
+    _stack_trace_deprecation_warning()
+
     trace = []
     for frame, path, line_no, func_name, text in (f[:5] for f in stack):
         if omit_path(os.path.realpath(path)):
@@ -234,6 +251,8 @@ def get_stack(context=1):
 
     Modified version of ``inspect.stack()`` which calls our own ``getframeinfo()``
     """
+    _stack_trace_deprecation_warning()
+
     frame = sys._getframe(1)
     framelist = []
     while frame:
@@ -242,6 +261,99 @@ def get_stack(context=1):
     return framelist
 
 
+def _stack_frames(depth=1):
+    frame = inspect.currentframe()
+    while frame is not None:
+        if depth > 0:
+            depth -= 1
+        else:
+            yield frame
+        frame = frame.f_back
+
+
+class _StackTraceRecorder:
+    def __init__(self, excluded_paths):
+        self.excluded_paths = excluded_paths
+        self.filename_cache = {}
+        self.is_excluded_cache = {}
+
+    def get_source_file(self, frame):
+        frame_filename = frame.f_code.co_filename
+
+        value = self.filename_cache.get(frame_filename)
+        if value is None:
+            filename = inspect.getsourcefile(frame)
+            if filename is None:
+                is_source = False
+                filename = frame_filename
+            else:
+                is_source = True
+                # Ensure linecache validity the first time this recorder
+                # encounters the filename in this frame.
+                linecache.checkcache(filename)
+            value = (filename, is_source)
+            self.filename_cache[frame_filename] = value
+
+        return value
+
+    def is_excluded_path(self, path):
+        excluded = self.is_excluded_cache.get(path)
+        if excluded is None:
+            resolved_path = os.path.realpath(path)
+            excluded = any(
+                resolved_path.startswith(excluded_path)
+                for excluded_path in self.excluded_paths
+            )
+            self.is_excluded_cache[path] = excluded
+        return excluded
+
+    def get_stack_trace(self, include_locals=False, depth=1):
+        trace = []
+        for frame in _stack_frames(depth=depth + 1):
+            filename, is_source = self.get_source_file(frame)
+
+            if self.is_excluded_path(filename):
+                continue
+
+            line_no = frame.f_lineno
+            func_name = frame.f_code.co_name
+
+            if is_source:
+                module = inspect.getmodule(frame, filename)
+                module_globals = module.__dict__ if module is not None else None
+                source_line = linecache.getline(
+                    filename, line_no, module_globals
+                ).strip()
+            else:
+                source_line = ""
+
+            frame_locals = frame.f_locals if include_locals else None
+
+            trace.append((filename, line_no, func_name, source_line, frame_locals))
+        trace.reverse()
+        return trace
+
+
+def get_stack_trace(depth=1):
+    config = dt_settings.get_config()
+    if config["ENABLE_STACKTRACES"]:
+        stack_trace_recorder = getattr(_local_data, "stack_trace_recorder", None)
+        if stack_trace_recorder is None:
+            stack_trace_recorder = _StackTraceRecorder(hidden_paths)
+            _local_data.stack_trace_recorder = stack_trace_recorder
+        return stack_trace_recorder.get_stack_trace(
+            include_locals=config["ENABLE_STACKTRACES_LOCALS"],
+            depth=depth,
+        )
+    else:
+        return []
+
+
+def clear_stack_trace_caches():
+    if hasattr(_local_data, "stack_trace_recorder"):
+        del _local_data.stack_trace_recorder
+
+
 class ThreadCollector:
     def __init__(self):
         if threading is None:

docs/changes.rst

@@ -11,6 +11,17 @@ Change log
 * Fixed the cache panel to correctly count cache misses from the get_many()
   cache method.
 * Removed some obsolete compatibility code from the stack trace recording code.
+* Added a new mechanism for capturing stack traces which includes per-request
+  caching to reduce expensive file system operations.  Updated the cache and
+  SQL panels to record stack traces using this new mechanism.
+
+Deprecated features
+~~~~~~~~~~~~~~~~~~~
+
+* The ``debug_toolbar.utils.get_stack()`` and
+  ``debug_toolbar.utils.tidy_stacktrace()`` functions are deprecated in favor
+  of the new ``debug_toolbar.utils.get_stack_trace()`` function.  They will
+  removed in the next major version of the Debug Toolbar.
 
 3.4.0 (2022-05-03)
 ------------------

tests/test_utils.py

@@ -1,6 +1,11 @@
 import unittest
 
-from debug_toolbar.utils import get_name_from_obj, render_stacktrace
+from debug_toolbar.utils import (
+    get_name_from_obj,
+    get_stack,
+    render_stacktrace,
+    tidy_stacktrace,
+)
 
 
 class GetNameFromObjTestCase(unittest.TestCase):
@@ -47,3 +52,13 @@ def test_importlib_path_issue_1612(self):
             '<span class="djdt-file">&lt;frozen importlib._bootstrap&gt;</span> in',
             result,
         )
+
+
+class StackTraceTestCase(unittest.TestCase):
+    def test_deprecated_functions(self):
+        with self.assertWarns(DeprecationWarning):
+            stack = get_stack()
+        self.assertEqual(stack[0][1], __file__)
+        with self.assertWarns(DeprecationWarning):
+            stack_trace = tidy_stacktrace(reversed(stack))
+        self.assertEqual(stack_trace[-1][0], __file__)