From 92ec806e36734b00c350eb989b1f5a3a95359912 Mon Sep 17 00:00:00 2001 From: Fabio Zadrozny Date: Tue, 7 Nov 2023 08:52:25 -0300 Subject: [PATCH] wip 669 --- .../pysrc/_pydevd_bundle/pydevd_constants.py | 1 + .../pysrc/_pydevd_bundle/pydevd_cython.c | 4 +- .../pydevd_frame_eval_main.py | 8 +- .../pydevd_sys_monitoring.py | 329 ++++++++++++++++++ plugins/org.python.pydev.core/pysrc/pydevd.py | 39 ++- .../pysrc/tests_python/test_sys_monitoring.py | 41 +++ 6 files changed, 415 insertions(+), 7 deletions(-) create mode 100644 plugins/org.python.pydev.core/pysrc/_pydevd_sys_monitoring/pydevd_sys_monitoring.py create mode 100644 plugins/org.python.pydev.core/pysrc/tests_python/test_sys_monitoring.py diff --git a/plugins/org.python.pydev.core/pysrc/_pydevd_bundle/pydevd_constants.py b/plugins/org.python.pydev.core/pysrc/_pydevd_bundle/pydevd_constants.py index 01d1951f51..310f27bbcb 100644 --- a/plugins/org.python.pydev.core/pysrc/_pydevd_bundle/pydevd_constants.py +++ b/plugins/org.python.pydev.core/pysrc/_pydevd_bundle/pydevd_constants.py @@ -169,6 +169,7 @@ def _current_frames(): IS_PY39_OR_GREATER = sys.version_info >= (3, 9) IS_PY310_OR_GREATER = sys.version_info >= (3, 10) IS_PY311_OR_GREATER = sys.version_info >= (3, 11) +IS_PY312_OR_GREATER = sys.version_info >= (3, 12) def version_str(v): diff --git a/plugins/org.python.pydev.core/pysrc/_pydevd_bundle/pydevd_cython.c b/plugins/org.python.pydev.core/pysrc/_pydevd_bundle/pydevd_cython.c index 767d6b1171..6ade939787 100644 --- a/plugins/org.python.pydev.core/pysrc/_pydevd_bundle/pydevd_cython.c +++ b/plugins/org.python.pydev.core/pysrc/_pydevd_bundle/pydevd_cython.c @@ -1458,8 +1458,8 @@ static const char *__pyx_filename; /* #### Code section: filename_table ### */ static const char *__pyx_f[] = { - "_pydevd_bundle\\\\pydevd_cython.pyx", - "_pydevd_bundle\\\\pydevd_cython.pxd", + "_pydevd_bundle/pydevd_cython.pyx", + "_pydevd_bundle/pydevd_cython.pxd", "", "type.pxd", }; diff --git a/plugins/org.python.pydev.core/pysrc/_pydevd_frame_eval/pydevd_frame_eval_main.py b/plugins/org.python.pydev.core/pysrc/_pydevd_frame_eval/pydevd_frame_eval_main.py index bfebea28ff..a4e1ce67dc 100644 --- a/plugins/org.python.pydev.core/pysrc/_pydevd_frame_eval/pydevd_frame_eval_main.py +++ b/plugins/org.python.pydev.core/pysrc/_pydevd_frame_eval/pydevd_frame_eval_main.py @@ -4,7 +4,7 @@ from _pydevd_bundle.pydevd_trace_dispatch import USING_CYTHON from _pydevd_bundle.pydevd_constants import USE_CYTHON_FLAG, ENV_FALSE_LOWER_VALUES, \ ENV_TRUE_LOWER_VALUES, IS_PY36_OR_GREATER, IS_PY38_OR_GREATER, SUPPORT_GEVENT, IS_PYTHON_STACKLESS, \ - PYDEVD_USE_FRAME_EVAL, PYDEVD_IPYTHON_COMPATIBLE_DEBUGGING + PYDEVD_USE_FRAME_EVAL, PYDEVD_IPYTHON_COMPATIBLE_DEBUGGING, IS_PY311_OR_GREATER frame_eval_func = None stop_frame_eval = None @@ -32,7 +32,8 @@ # Same problem with Stackless. # https://github.com/stackless-dev/stackless/issues/240 -elif PYDEVD_USE_FRAME_EVAL in ENV_TRUE_LOWER_VALUES: +elif PYDEVD_USE_FRAME_EVAL in ENV_TRUE_LOWER_VALUES and not IS_PY311_OR_GREATER: + # Python 3.11 onwards doesn't have frame eval mode implemented # Fail if unable to use from _pydevd_frame_eval.pydevd_frame_eval_cython_wrapper import frame_eval_func, stop_frame_eval, dummy_trace_dispatch, clear_thread_local_info USING_FRAME_EVAL = True @@ -40,7 +41,8 @@ else: USING_FRAME_EVAL = False # Try to use if possible - if IS_PY36_OR_GREATER: + if IS_PY36_OR_GREATER and not IS_PY311_OR_GREATER: + # Python 3.11 onwards doesn't have frame eval mode implemented try: from _pydevd_frame_eval.pydevd_frame_eval_cython_wrapper import frame_eval_func, stop_frame_eval, dummy_trace_dispatch, clear_thread_local_info USING_FRAME_EVAL = True diff --git a/plugins/org.python.pydev.core/pysrc/_pydevd_sys_monitoring/pydevd_sys_monitoring.py b/plugins/org.python.pydev.core/pysrc/_pydevd_sys_monitoring/pydevd_sys_monitoring.py new file mode 100644 index 0000000000..7d0c1e0ed6 --- /dev/null +++ b/plugins/org.python.pydev.core/pysrc/_pydevd_sys_monitoring/pydevd_sys_monitoring.py @@ -0,0 +1,329 @@ +from _pydevd_bundle.pydevd_constants import IS_PY312_OR_GREATER, \ + GlobalDebuggerHolder +import threading +from _pydevd_bundle.pydevd_additional_thread_info import _set_additional_thread_info_lock, PyDBAdditionalThreadInfo +from pydevd_file_utils import NORM_PATHS_AND_BASE_CONTAINER, \ + get_abs_path_real_path_and_base_from_file +import dis +from collections import namedtuple +import sys +from types import CodeType +from typing import Dict, Optional, Set + +DEBUGGER_ID = sys.monitoring.DEBUGGER_ID +monitor = sys.monitoring + +USING_FRAME_MONITORING = False +if IS_PY312_OR_GREATER: + import sys + USING_FRAME_MONITORING = hasattr(sys, 'monitoring') + +_thread_local_info = threading.local() +_get_ident = threading.get_ident +_thread_active = threading._active + + +class ThreadInfo: + + additional_info: PyDBAdditionalThreadInfo + is_pydevd_thread: bool + fully_initialized: bool + can_create_dummy_thread: bool + + def initialize_can_create_dummy_thread(self, code): + self.additional_info = None + self.is_pydevd_thread = False + self.fully_initialized = False + self.thread_trace_func = None + + basename = code.co_filename + i = basename.rfind('/') + j = basename.rfind('\\') + if j > i: + i = j + if i >= 0: + basename = basename[i + 1:] + # remove ext + i = basename.rfind('.') + if i >= 0: + basename = basename[:i] + + co_name = code.co_name + + # In these cases we cannot create a dummy thread (an actual + # thread will be created later or tracing will already be set). + if basename == 'threading' and co_name in ('__bootstrap', '_bootstrap', '__bootstrap_inner', '_bootstrap_inner'): + self.can_create_dummy_thread = False + elif basename == 'pydev_monkey' and co_name == '__call__': + self.can_create_dummy_thread = False + elif basename == 'pydevd' and co_name in ('run', 'main', '_exec'): + self.can_create_dummy_thread = False + elif basename == 'pydevd_tracing': + self.can_create_dummy_thread = False + else: + self.can_create_dummy_thread = True + + def initialize_if_possible(self): + # Don't call threading.currentThread because if we're too early in the process + # we may create a dummy thread. + thread_ident = _get_ident() + t = _thread_active.get(thread_ident) + if t is None: + assert self.can_create_dummy_thread + # Initialize the dummy thread and set the tracing (both are needed to + # actually stop on breakpoints). + t = threading.current_thread() + + if getattr(t, 'is_pydev_daemon_thread', False): + self.is_pydevd_thread = True + self.fully_initialized = True + else: + try: + additional_info = t.additional_info + if additional_info is None: + raise AttributeError() + except: + with _set_additional_thread_info_lock: + # If it's not there, set it within a lock to avoid any racing + # conditions. + additional_info = getattr(t, 'additional_info', None) + if additional_info is None: + additional_info = PyDBAdditionalThreadInfo() + t.additional_info = additional_info + self.additional_info = additional_info + self.fully_initialized = True + + +class FuncCodeInfo: + + co_filename: str + co_name: str + canonical_normalized_filename: str + always_skip_code: bool + breakpoint_found: bool + breakpoints_hit_at_lines: Set[int] + function_breakpoint: bool + + # When breakpoints_mtime != PyDb.mtime the validity of breakpoints have + # to be re-evaluated (if invalid a new FuncCodeInfo must be created and + # tracing can't be disabled for the related frames). + breakpoints_mtime: int + + def __init__(self): + self.co_filename = '' + self.canonical_normalized_filename = '' + self.always_skip_code = False + + self.breakpoint_found = False + self.breakpoints_mtime = -1 + + self.breakpoints_hit_at_lines = set() + self.function_breakpoint = False + + +def get_thread_info(code) -> Optional[ThreadInfo]: + ''' + Provides thread-related info. + + May return None if the thread is still not active. + ''' + try: + # Note: changing to a `dict[thread.ident] = thread_info` had almost no + # effect in the performance. + return _thread_local_info.thread_info + except: + thread_info = ThreadInfo() + thread_info.initialize_can_create_dummy_thread(code) + if not thread_info.can_create_dummy_thread: + return None + thread_info.initialize_if_possible() + + _thread_local_info.thread_info = thread_info + return _thread_local_info.thread_info + + +_CodeLineInfo = namedtuple('_CodeLineInfo', 'line_to_offset, first_line, last_line') + + +# Note: this method has a version in cython too +def _get_code_line_info(code_obj): + line_to_offset = {} + first_line = None + last_line = None + + for offset, line in dis.findlinestarts(code_obj): + line_to_offset[line] = offset + + if line_to_offset: + first_line = min(line_to_offset) + last_line = max(line_to_offset) + return _CodeLineInfo(line_to_offset, first_line, last_line) + + +code_to_func_code_info: Dict[CodeType, 'FuncCodeInfo'] = {} + + +def get_func_code_info(thread_info: ThreadInfo, code_obj, code_to_func_code_info=code_to_func_code_info) -> FuncCodeInfo: + ''' + Provides code-object related info. + + Note that it contains informations on the breakpoints for a given function. + If breakpoints change a new FuncCodeInfo instance will be created. + ''' + main_debugger = GlobalDebuggerHolder.global_dbg + + func_code_info_obj = code_to_func_code_info.get(code_obj) + if func_code_info_obj is not None: + if func_code_info_obj.breakpoints_mtime == main_debugger.mtime: + # if DEBUG: + # print('get_func_code_info: matched mtime', f_code.co_name, f_code.co_filename) + return func_code_info_obj + + co_filename: str = code_obj.co_filename + co_name: str = code_obj.co_name + cache_file_type: dict + cache_file_type_key: tuple + + func_code_info = FuncCodeInfo() + func_code_info.breakpoints_mtime = main_debugger.mtime + + func_code_info.co_filename = co_filename + func_code_info.co_name = co_name + + # Compute whether to always skip this. + try: + abs_path_real_path_and_base = NORM_PATHS_AND_BASE_CONTAINER[co_filename] + except: + abs_path_real_path_and_base = get_abs_path_real_path_and_base_from_file(co_filename) + + func_code_info.canonical_normalized_filename = abs_path_real_path_and_base[1] + + cache_file_type = main_debugger.get_cache_file_type() + # Note: this cache key must be the same from PyDB.get_file_type() -- see it for comments + # on the cache. + cache_file_type_key = (code_obj.co_firstlineno, abs_path_real_path_and_base[0], code_obj) + try: + file_type = cache_file_type[cache_file_type_key] # Make it faster + except: + file_type = main_debugger.get_file_type_from_code(code_obj, abs_path_real_path_and_base) # we don't want to debug anything related to pydevd + + if file_type is not None: + func_code_info.always_skip_code = True + + if not func_code_info.always_skip_code: + if main_debugger is not None: + + breakpoints: dict = main_debugger.breakpoints.get(func_code_info.canonical_normalized_filename) + function_breakpoint: object = main_debugger.function_breakpoint_name_to_breakpoint.get(func_code_info.co_name) + # print('\n---') + # print(main_debugger.breakpoints) + # print(func_code_info.canonical_normalized_filename) + # print(main_debugger.breakpoints.get(func_code_info.canonical_normalized_filename)) + if function_breakpoint: + # Go directly into tracing mode + func_code_info.breakpoint_found = True + func_code_info.function_breakpoint = True + + if breakpoints: + # if DEBUG: + # print('found breakpoints', code_obj_py.co_name, breakpoints) + + breakpoints_hit_at_lines = set() + code_line_info = _get_code_line_info(code_obj) + line_to_offset = code_line_info.line_to_offset + + for breakpoint_line in breakpoints: + if breakpoint_line in line_to_offset: + breakpoints_hit_at_lines.add(breakpoint_line) + + func_code_info.breakpoint_found = bool(breakpoints_hit_at_lines) + func_code_info.breakpoints_hit_at_lines = breakpoints_hit_at_lines + + code_to_func_code_info[code_obj] = func_code_info_obj + + return func_code_info + + +def _start_method(code, instruction_offset): + try: + thread_info = _thread_local_info.thread_info + except: + thread_info = get_thread_info(code) + + additional_info = thread_info.additional_info + if thread_info.is_pydevd_thread: + func_code_info: FuncCodeInfo = get_func_code_info(thread_info, code) + if func_code_info.always_skip_code: + return monitor.DISABLE + return # We can't disable it here as another thread could use it too. + + STATE_SUSPEND: int = 2 + CMD_STEP_INTO: int = 107 + CMD_STEP_OVER: int = 108 + CMD_STEP_OVER_MY_CODE: int = 159 + CMD_STEP_INTO_MY_CODE: int = 144 + CMD_STEP_INTO_COROUTINE: int = 206 + CMD_SMART_STEP_INTO: int = 128 + can_skip: bool = True + + # We know the frame depth. + frame = sys._getframe(1) + + main_debugger: object = GlobalDebuggerHolder.global_dbg + if main_debugger is None: + return monitor.DISABLE + + if additional_info.pydev_step_cmd in (CMD_STEP_INTO, CMD_STEP_INTO_MY_CODE, CMD_STEP_INTO_COROUTINE, CMD_SMART_STEP_INTO): + # Stepping (must have line tracing enabled). + enable_line_tracing(code) + + if additional_info.pydev_step_cmd in (CMD_STEP_OVER, CMD_STEP_OVER_MY_CODE) and main_debugger.show_return_values and frame.f_back is additional_info.pydev_step_stop: + # Show return values on step over. + enable_return_tracing(code) + + if main_debugger.break_on_caught_exceptions or main_debugger.break_on_user_uncaught_exceptions or main_debugger.has_plugin_exception_breaks: + enable_exception_tracing(code) + + if main_debugger.signature_factory: + pass + + func_code_info: FuncCodeInfo = get_func_code_info(thread_info, frame, code) + # if DEBUG: + # print('get_bytecode_while_frame_eval always skip', func_code_info.always_skip_code) + if not func_code_info.always_skip_code: + if main_debugger.has_plugin_line_breaks or main_debugger.has_plugin_exception_breaks: + can_skip = main_debugger.plugin.can_skip(main_debugger, frame) + + if not can_skip: + if main_debugger.has_plugin_line_breaks: + enable_line_tracing(code) + + if main_debugger.has_plugin_exception_breaks: + enable_exception_tracing(code) + + if func_code_info.breakpoint_found: + enable_line_tracing(code) + + +def start_monitoring(): + DEBUGGER_ID = sys.monitoring.DEBUGGER_ID + if not sys.monitoring.get_tool(DEBUGGER_ID): + sys.monitoring.use_tool_id(DEBUGGER_ID, 'pydevd') + sys.monitoring.set_events(DEBUGGER_ID, sys.monitoring.events.PY_START | sys.monitoring.events.PY_RESUME) + + sys.monitoring.register_callback(DEBUGGER_ID, sys.monitoring.events.PY_START, _start_method) + sys.monitoring.register_callback(DEBUGGER_ID, sys.monitoring.events.PY_RESUME, _start_method) + + # sys.monitoring.register_callback(DEBUGGER_ID, sys.monitoring.events.LINE, self._line_callback) + # + # sys.monitoring.register_callback(DEBUGGER_ID, sys.monitoring.events.PY_RETURN, self._return_callback) + # + # sys.monitoring.register_callback(DEBUGGER_ID, sys.monitoring.events.RAISE, self._raise_callback) + + # Activate exception raise callback if exception breakpoints are registered. + current_events = sys.monitoring.get_events(DEBUGGER_ID) + sys.monitoring.set_events(DEBUGGER_ID, current_events | sys.monitoring.events.RAISE) + + +def stop_monitoring(): + sys.monitoring.set_events(sys.monitoring.DEBUGGER_ID, 0) diff --git a/plugins/org.python.pydev.core/pysrc/pydevd.py b/plugins/org.python.pydev.core/pysrc/pydevd.py index 6ea4521c90..336dd51e07 100644 --- a/plugins/org.python.pydev.core/pysrc/pydevd.py +++ b/plugins/org.python.pydev.core/pysrc/pydevd.py @@ -73,7 +73,7 @@ from _pydevd_bundle.pydevd_source_mapping import SourceMapping from _pydevd_bundle.pydevd_concurrency_analyser.pydevd_concurrency_logger import ThreadingLogger, AsyncioLogger, send_concurrency_message, cur_time from _pydevd_bundle.pydevd_concurrency_analyser.pydevd_thread_wrappers import wrap_threads -from pydevd_file_utils import get_abs_path_real_path_and_base_from_frame, NORM_PATHS_AND_BASE_CONTAINER +from pydevd_file_utils import get_abs_path_real_path_and_base_from_frame, get_abs_path_real_path_and_base_from_file, NORM_PATHS_AND_BASE_CONTAINER from pydevd_file_utils import get_fullname, get_package_dir from os.path import abspath as os_path_abspath import pydevd_tracing @@ -983,6 +983,42 @@ def dont_trace_external_files(self, abs_path): # be changed for another function in PyDevdAPI.set_dont_trace_start_end_patterns. return False + def get_file_type_from_code(self, code, abs_real_path_and_basename=None, _cache_file_type=_CACHE_FILE_TYPE): + if abs_real_path_and_basename is None: + try: + # Make fast path faster! + abs_real_path_and_basename = NORM_PATHS_AND_BASE_CONTAINER[code.co_filename] + except: + abs_real_path_and_basename = get_abs_path_real_path_and_base_from_file(code.co_filename) + + # Note 1: we have to take into account that we may have files as '', and that in + # this case the cache key can't rely only on the filename. With the current cache, there's + # still a potential miss if 2 functions which have exactly the same content are compiled + # with '', but in practice as we only separate the one from python -c from the rest + # this shouldn't be a problem in practice. + + # Note 2: firstlineno added to make misses faster in the first comparison. + + # Note 3: this cache key is repeated in pydevd_frame_evaluator.pyx:get_func_code_info (for + # speedups). + cache_key = (code.co_firstlineno, abs_real_path_and_basename[0], code) + try: + return _cache_file_type[cache_key] + except: + if abs_real_path_and_basename[0] == '': + # Note that we return as a LIB_FILE and not PYDEV_FILE because we still want + # to show it in the stack. + _cache_file_type[cache_key] = LIB_FILE + return LIB_FILE + + file_type = self._internal_get_file_type(abs_real_path_and_basename) + if file_type is None: + if self.dont_trace_external_files(abs_real_path_and_basename[0]): + file_type = PYDEV_FILE + + _cache_file_type[cache_key] = file_type + return file_type + def get_file_type(self, frame, abs_real_path_and_basename=None, _cache_file_type=_CACHE_FILE_TYPE): ''' :param abs_real_path_and_basename: @@ -1022,7 +1058,6 @@ def get_file_type(self, frame, abs_real_path_and_basename=None, _cache_file_type return _cache_file_type[cache_key] except: if abs_real_path_and_basename[0] == '': - # Consider it an untraceable file unless there's no back frame (ignoring # internal files and runpy.py). f = frame.f_back diff --git a/plugins/org.python.pydev.core/pysrc/tests_python/test_sys_monitoring.py b/plugins/org.python.pydev.core/pysrc/tests_python/test_sys_monitoring.py new file mode 100644 index 0000000000..13db7d3bc2 --- /dev/null +++ b/plugins/org.python.pydev.core/pysrc/tests_python/test_sys_monitoring.py @@ -0,0 +1,41 @@ +import sys + +DEBUGGER_ID = sys.monitoring.DEBUGGER_ID +monitor = sys.monitoring + +if __name__ == '__main__': + code_to_break_at_line = {} + do_change_line = [0] + + def _start_method(code, offset): + monitor.set_local_events(DEBUGGER_ID, code, monitor.events.LINE) + code_to_break_at_line[code] = {code.co_firstlineno + 3} + return monitor.DISABLE + + def _on_line(code, line): + lines_to_break = code_to_break_at_line.get(code) + if lines_to_break and line in lines_to_break: + do_change_line[0] += 1 + if do_change_line[0] == 2: + frame = sys._getframe().f_back + print(frame.f_lineno) + frame.f_lineno = line - 2 + + monitor.use_tool_id(DEBUGGER_ID, 'pydevd') + monitor.set_events(DEBUGGER_ID, monitor.events.PY_START | monitor.events.PY_RESUME) + + monitor.register_callback(DEBUGGER_ID, monitor.events.PY_START , _start_method) + monitor.register_callback(DEBUGGER_ID, monitor.events.PY_RESUME, _start_method) + monitor.register_callback(DEBUGGER_ID, monitor.events.LINE, _on_line) + + def method1(): # code.co_firstlineno + a = 1 # code.co_firstlineno + 1 + print('before a=2') # code.co_firstlineno + 2 + a = 2 # code.co_firstlineno + 3 + print('before a=3') # code.co_firstlineno + 4 + # a = 3 # code.co_firstlineno + 5 + + for _i in range(3): + method1() + + sys.monitoring.set_events(sys.monitoring.DEBUGGER_ID, 0)