From 6070d5582082634f04fa4a334dd2d8ff9889812c Mon Sep 17 00:00:00 2001 From: Fabio Zadrozny Date: Sun, 21 Jan 2024 07:08:25 -0300 Subject: [PATCH] wip --- _pydevd_bundle/pydevd_bytecode_utils.py | 8 +- _pydevd_frame_eval/vendored/README.txt | 2 +- .../COPYING | 2 +- .../INSTALLER | 0 .../bytecode-0.13.0.dev0.dist-info/METADATA | 77 + .../bytecode-0.13.0.dev0.dist-info/RECORD | 42 + .../REQUESTED | 0 .../WHEEL | 2 +- .../direct_url.json | 1 + .../top_level.txt | 0 .../METADATA | 102 -- .../RECORD | 23 - .../direct_url.json | 1 - .../vendored/bytecode/__init__.py | 175 +- .../vendored/bytecode/bytecode.py | 241 +-- _pydevd_frame_eval/vendored/bytecode/cfg.py | 940 ++-------- .../vendored/bytecode/concrete.py | 1153 +++---------- _pydevd_frame_eval/vendored/bytecode/flags.py | 64 +- _pydevd_frame_eval/vendored/bytecode/instr.py | 977 +++-------- .../vendored/bytecode/peephole_opt.py | 491 ++++++ _pydevd_frame_eval/vendored/bytecode/py.typed | 0 .../vendored/bytecode/tests/__init__.py | 154 ++ .../vendored/bytecode/tests/test_bytecode.py | 488 ++++++ .../vendored/bytecode/tests/test_cfg.py | 836 +++++++++ .../vendored/bytecode/tests/test_code.py | 93 + .../vendored/bytecode/tests/test_concrete.py | 1513 +++++++++++++++++ .../vendored/bytecode/tests/test_flags.py | 159 ++ .../vendored/bytecode/tests/test_instr.py | 362 ++++ .../vendored/bytecode/tests/test_misc.py | 270 +++ .../bytecode/tests/test_peephole_opt.py | 985 +++++++++++ .../bytecode/tests/util_annotation.py | 17 + .../vendored/bytecode/version.py | 19 - 32 files changed, 6246 insertions(+), 2951 deletions(-) rename _pydevd_frame_eval/vendored/{bytecode-0.15.2.dev4+gc87faa2.dist-info => bytecode-0.13.0.dev0.dist-info}/COPYING (95%) rename _pydevd_frame_eval/vendored/{bytecode-0.15.2.dev4+gc87faa2.dist-info => bytecode-0.13.0.dev0.dist-info}/INSTALLER (100%) create mode 100644 _pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/METADATA create mode 100644 _pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/RECORD rename _pydevd_frame_eval/vendored/{bytecode-0.15.2.dev4+gc87faa2.dist-info => bytecode-0.13.0.dev0.dist-info}/REQUESTED (100%) rename _pydevd_frame_eval/vendored/{bytecode-0.15.2.dev4+gc87faa2.dist-info => bytecode-0.13.0.dev0.dist-info}/WHEEL (65%) create mode 100644 _pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/direct_url.json rename _pydevd_frame_eval/vendored/{bytecode-0.15.2.dev4+gc87faa2.dist-info => bytecode-0.13.0.dev0.dist-info}/top_level.txt (100%) delete mode 100644 _pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/METADATA delete mode 100644 _pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/RECORD delete mode 100644 _pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/direct_url.json create mode 100644 _pydevd_frame_eval/vendored/bytecode/peephole_opt.py delete mode 100644 _pydevd_frame_eval/vendored/bytecode/py.typed create mode 100644 _pydevd_frame_eval/vendored/bytecode/tests/__init__.py create mode 100644 _pydevd_frame_eval/vendored/bytecode/tests/test_bytecode.py create mode 100644 _pydevd_frame_eval/vendored/bytecode/tests/test_cfg.py create mode 100644 _pydevd_frame_eval/vendored/bytecode/tests/test_code.py create mode 100644 _pydevd_frame_eval/vendored/bytecode/tests/test_concrete.py create mode 100644 _pydevd_frame_eval/vendored/bytecode/tests/test_flags.py create mode 100644 _pydevd_frame_eval/vendored/bytecode/tests/test_instr.py create mode 100644 _pydevd_frame_eval/vendored/bytecode/tests/test_misc.py create mode 100644 _pydevd_frame_eval/vendored/bytecode/tests/test_peephole_opt.py create mode 100644 _pydevd_frame_eval/vendored/bytecode/tests/util_annotation.py delete mode 100644 _pydevd_frame_eval/vendored/bytecode/version.py diff --git a/_pydevd_bundle/pydevd_bytecode_utils.py b/_pydevd_bundle/pydevd_bytecode_utils.py index fe2ce1ea8..0d2ef79f1 100644 --- a/_pydevd_bundle/pydevd_bytecode_utils.py +++ b/_pydevd_bundle/pydevd_bytecode_utils.py @@ -6,8 +6,7 @@ from _pydev_bundle import pydev_log from types import CodeType -from _pydevd_frame_eval.vendored.bytecode.instr import _Variable, TryBegin, \ - TryEnd, Label +from _pydevd_frame_eval.vendored.bytecode.instr import _Variable, Label from _pydevd_frame_eval.vendored import bytecode from _pydevd_frame_eval.vendored.bytecode import cfg as bytecode_cfg import dis @@ -739,8 +738,7 @@ def _get_smart_step_into_targets(code): :return list(Target) ''' b = bytecode.Bytecode.from_code(code) - # cfg = bytecode_cfg.ControlFlowGraph.from_bytecode(b) - cfg = [b] + cfg = bytecode_cfg.ControlFlowGraph.from_bytecode(b) ret = [] @@ -749,7 +747,7 @@ def _get_smart_step_into_targets(code): print('\nStart block----') stack = _StackInterpreter(block) for instr in block: - if isinstance(instr, (TryBegin, TryEnd, Label)): + if isinstance(instr, (Label,)): # No name for these continue try: diff --git a/_pydevd_frame_eval/vendored/README.txt b/_pydevd_frame_eval/vendored/README.txt index 19d63d456..d15aa2091 100644 --- a/_pydevd_frame_eval/vendored/README.txt +++ b/_pydevd_frame_eval/vendored/README.txt @@ -8,7 +8,7 @@ pip install bytecode --target . or from master (if needed for some early bugfix): -python -m pip install git+https://github.com/MatthieuDartiailh/bytecode.git --target . +python -m pip install https://github.com/MatthieuDartiailh/bytecode/archive/main.zip --target . Then run 'pydevd_fix_code.py' to fix the imports on the vendored file, run its tests (to see if things are still ok) and commit. diff --git a/_pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/COPYING b/_pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/COPYING similarity index 95% rename from _pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/COPYING rename to _pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/COPYING index ba5a523fc..81d7e37c3 100644 --- a/_pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/COPYING +++ b/_pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/COPYING @@ -1,5 +1,5 @@ The MIT License (MIT) -Copyright Contributors to the bytecode project. +Copyright (c) 2016 Red Hat. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/_pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/INSTALLER b/_pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/INSTALLER similarity index 100% rename from _pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/INSTALLER rename to _pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/INSTALLER diff --git a/_pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/METADATA b/_pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/METADATA new file mode 100644 index 000000000..e1d5e0120 --- /dev/null +++ b/_pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/METADATA @@ -0,0 +1,77 @@ +Metadata-Version: 2.1 +Name: bytecode +Version: 0.13.0.dev0 +Summary: Python module to generate and modify bytecode +Home-page: https://github.com/MatthieuDartiailh/bytecode +Author: Victor Stinner +Author-email: victor.stinner@gmail.com +Maintainer: Matthieu C. Dartiailh +Maintainer-email: m.dartiailh@gmail.com +License: MIT license +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Natural Language :: English +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Requires-Python: >=3.6 + +******** +bytecode +******** + +.. image:: https://img.shields.io/pypi/v/bytecode.svg + :alt: Latest release on the Python Cheeseshop (PyPI) + :target: https://pypi.python.org/pypi/bytecode + +.. image:: https://github.com/MatthieuDartiailh/bytecode/workflows/Continuous%20Integration/badge.svg + :target: https://github.com/MatthieuDartiailh/bytecode/actions + :alt: Continuous integration + +.. image:: https://github.com/MatthieuDartiailh/bytecode/workflows/Documentation%20building/badge.svg + :target: https://github.com/MatthieuDartiailh/bytecode/actions + :alt: Documentation building + +.. image:: https://img.shields.io/codecov/c/github/MatthieuDartiailh/bytecode/master.svg + :alt: Code coverage of bytecode on codecov.io + :target: https://codecov.io/github/MatthieuDartiailh/bytecode + +.. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :alt: Code formatted using Black + :target: https://github.com/psf/black + +``bytecode`` is a Python module to generate and modify bytecode. + +* `bytecode project homepage at GitHub + `_ (code, bugs) +* `bytecode documentation + `_ +* `Download latest bytecode release at the Python Cheeseshop (PyPI) + `_ + +Install bytecode: ``python3 -m pip install bytecode``. It requires Python 3.6 +or newer. The latest release that supports Python 3.5 is 0.12.0. For Python 2.7 +support, have a look at `dead-bytecode +`_ instead. + +Example executing ``print('Hello World!')``: + +.. code:: python + + from bytecode import Instr, Bytecode + + bytecode = Bytecode([Instr("LOAD_NAME", 'print'), + Instr("LOAD_CONST", 'Hello World!'), + Instr("CALL_FUNCTION", 1), + Instr("POP_TOP"), + Instr("LOAD_CONST", None), + Instr("RETURN_VALUE")]) + code = bytecode.to_code() + exec(code) + diff --git a/_pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/RECORD b/_pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/RECORD new file mode 100644 index 000000000..3890ece18 --- /dev/null +++ b/_pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/RECORD @@ -0,0 +1,42 @@ +bytecode-0.13.0.dev0.dist-info/COPYING,sha256=baWkm-Te2LLURwK7TL0zOkMSVjVCU_ezvObHBo298Tk,1074 +bytecode-0.13.0.dev0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +bytecode-0.13.0.dev0.dist-info/METADATA,sha256=9XadDK6YTQ-FPowYI5DS4ieA7hRGnRP_fM5Z9ioPkEQ,2929 +bytecode-0.13.0.dev0.dist-info/RECORD,, +bytecode-0.13.0.dev0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +bytecode-0.13.0.dev0.dist-info/WHEEL,sha256=OqRkF0eY5GHssMorFjlbTIq072vpHpF60fIQA6lS9xA,92 +bytecode-0.13.0.dev0.dist-info/direct_url.json,sha256=s58Rb4KXRlMKxk-mzpvr_tJRQ-Hx8-DHsU6NdohCnAg,93 +bytecode-0.13.0.dev0.dist-info/top_level.txt,sha256=9BhdB7HqYZ-PvHNoWX6ilwLYWQqcgEOLwdb3aXm5Gys,9 +bytecode/__init__.py,sha256=d-yk4Xh4SwOWq9NgoD2rmBLG6RhUFNljeqs-NjMNSYM,3885 +bytecode/__pycache__/__init__.cpython-38.pyc,, +bytecode/__pycache__/bytecode.cpython-38.pyc,, +bytecode/__pycache__/cfg.cpython-38.pyc,, +bytecode/__pycache__/concrete.cpython-38.pyc,, +bytecode/__pycache__/flags.cpython-38.pyc,, +bytecode/__pycache__/instr.cpython-38.pyc,, +bytecode/__pycache__/peephole_opt.cpython-38.pyc,, +bytecode/bytecode.py,sha256=IMCcatHMtQ7M31nwj4r3drcvQuGVJAOP0d7C0O8P_SE,6894 +bytecode/cfg.py,sha256=RmJGJqwCxR-XYaPH9YGY4wNDycdtLvIBJb1OGSmxcN0,15274 +bytecode/concrete.py,sha256=0eb6Yh_NDLmzJNcMs2TFom0EqFVSM1cO3inMH90YE-s,21683 +bytecode/flags.py,sha256=hAvM_B2yQKRw44leHP0oCae0aaJraAbDDTpqIf4I1CM,5987 +bytecode/instr.py,sha256=HYc65LjNSOB3GCWkNkCSkee1rRzUyr89rgdjbKBaTpE,11616 +bytecode/peephole_opt.py,sha256=W-cFVPOZN-JKfDV3aImsYenDSZkSNBDTVQqeMrGPU18,15712 +bytecode/tests/__init__.py,sha256=BAdOXXNRdMVX4D8TuRYPlG9PHU7Cb0bzvyfA9s435kM,4968 +bytecode/tests/__pycache__/__init__.cpython-38.pyc,, +bytecode/tests/__pycache__/test_bytecode.cpython-38.pyc,, +bytecode/tests/__pycache__/test_cfg.cpython-38.pyc,, +bytecode/tests/__pycache__/test_code.cpython-38.pyc,, +bytecode/tests/__pycache__/test_concrete.cpython-38.pyc,, +bytecode/tests/__pycache__/test_flags.cpython-38.pyc,, +bytecode/tests/__pycache__/test_instr.cpython-38.pyc,, +bytecode/tests/__pycache__/test_misc.cpython-38.pyc,, +bytecode/tests/__pycache__/test_peephole_opt.cpython-38.pyc,, +bytecode/tests/__pycache__/util_annotation.cpython-38.pyc,, +bytecode/tests/test_bytecode.py,sha256=buvtlDC0NwoQ3zuZ7OENIIDngSqtiO9WkAa2-UvxGkI,15584 +bytecode/tests/test_cfg.py,sha256=c0xT8OfV-mDHu-DIDWr6LVlZQyK4GfgLSmT5AsodbMk,28194 +bytecode/tests/test_code.py,sha256=XCOH29rOXSoQz130s-AIC62r23e9qNjk8Y2xDB2LmSc,2100 +bytecode/tests/test_concrete.py,sha256=qT2qvabkF0yC7inniNx53cMSDN-2Qi0IE3pwBZSzF8g,49253 +bytecode/tests/test_flags.py,sha256=DY9U3c6tJdxJFm0jEm_To1Cc0I99EidQv_0guud-4oE,5684 +bytecode/tests/test_instr.py,sha256=rYeF8u-L0aW8bLPBxTUSy_T7KP6SaXyJKv9OhC8k6aA,11295 +bytecode/tests/test_misc.py,sha256=wyK1wpVPHRfaXgo-EqUI-F1nyB9-UACerHsHbExAo1U,6758 +bytecode/tests/test_peephole_opt.py,sha256=niUfhgEbiFR7IAmdQ_N9Qgh7D3wdRQ_zS0V8mKC4EzI,32640 +bytecode/tests/util_annotation.py,sha256=wKq6yPWrzkNlholl5Y10b3VjuCkoiYVgvcIjk_8jzf8,485 diff --git a/_pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/REQUESTED b/_pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/REQUESTED similarity index 100% rename from _pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/REQUESTED rename to _pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/REQUESTED diff --git a/_pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/WHEEL b/_pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/WHEEL similarity index 65% rename from _pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/WHEEL rename to _pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/WHEEL index 98c0d20b7..385faab05 100644 --- a/_pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/WHEEL +++ b/_pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/WHEEL @@ -1,5 +1,5 @@ Wheel-Version: 1.0 -Generator: bdist_wheel (0.42.0) +Generator: bdist_wheel (0.36.2) Root-Is-Purelib: true Tag: py3-none-any diff --git a/_pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/direct_url.json b/_pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/direct_url.json new file mode 100644 index 000000000..3c32b5716 --- /dev/null +++ b/_pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/direct_url.json @@ -0,0 +1 @@ +{"archive_info": {}, "url": "https://github.com/MatthieuDartiailh/bytecode/archive/main.zip"} \ No newline at end of file diff --git a/_pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/top_level.txt b/_pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/top_level.txt similarity index 100% rename from _pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/top_level.txt rename to _pydevd_frame_eval/vendored/bytecode-0.13.0.dev0.dist-info/top_level.txt diff --git a/_pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/METADATA b/_pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/METADATA deleted file mode 100644 index ae8fbe56c..000000000 --- a/_pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/METADATA +++ /dev/null @@ -1,102 +0,0 @@ -Metadata-Version: 2.1 -Name: bytecode -Version: 0.15.2.dev4+gc87faa2 -Summary: Python module to generate and modify bytecode -Author-email: Victor Stinner -Maintainer-email: "Matthieu C. Dartiailh" -License: The MIT License (MIT) - Copyright Contributors to the bytecode project. - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -Project-URL: homepage, https://github.com/MatthieuDartiailh/bytecode -Project-URL: documentation, https://bytecode.readthedocs.io/en/latest/ -Project-URL: repository, https://github.com/MatthieuDartiailh/bytecode -Project-URL: changelog, https://github.com/MatthieuDartiailh/bytecode/blob/main/doc/changelog.rst -Classifier: Development Status :: 4 - Beta -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: Natural Language :: English -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Requires-Python: >=3.8 -Description-Content-Type: text/x-rst -License-File: COPYING -Requires-Dist: typing-extensions ; python_version < "3.10" - -******** -bytecode -******** - -.. image:: https://img.shields.io/pypi/v/bytecode.svg - :alt: Latest release on the Python Cheeseshop (PyPI) - :target: https://pypi.python.org/pypi/bytecode - -.. image:: https://github.com/MatthieuDartiailh/bytecode/workflows/Continuous%20Integration/badge.svg - :target: https://github.com/MatthieuDartiailh/bytecode/actions - :alt: Continuous integration - -.. image:: https://github.com/MatthieuDartiailh/bytecode/workflows/Documentation%20building/badge.svg - :target: https://github.com/MatthieuDartiailh/bytecode/actions - :alt: Documentation building - -.. image:: https://img.shields.io/codecov/c/github/MatthieuDartiailh/bytecode/master.svg - :alt: Code coverage of bytecode on codecov.io - :target: https://codecov.io/github/MatthieuDartiailh/bytecode - -.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json - :target: https://github.com/astral-sh/ruff - :alt: Ruff - -``bytecode`` is a Python module to generate and modify bytecode. - -* `bytecode project homepage at GitHub - `_ (code, bugs) -* `bytecode documentation - `_ -* `Download latest bytecode release at the Python Cheeseshop (PyPI) - `_ - -Install bytecode: ``python3 -m pip install bytecode``. It requires Python 3.8 -or newer. The latest release that supports Python 3.7 and 3.6 is 0.13.0. -The latest release that supports Python 3.5 is 0.12.0. For Python 2.7 support, -have a look at `dead-bytecode `_ -instead. - -Example executing ``print('Hello World!')``: - -.. code:: python - - from bytecode import Instr, Bytecode - - bytecode = Bytecode([Instr("LOAD_NAME", 'print'), - Instr("LOAD_CONST", 'Hello World!'), - Instr("CALL_FUNCTION", 1), - Instr("POP_TOP"), - Instr("LOAD_CONST", None), - Instr("RETURN_VALUE")]) - code = bytecode.to_code() - exec(code) diff --git a/_pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/RECORD b/_pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/RECORD deleted file mode 100644 index ac7d4b362..000000000 --- a/_pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/RECORD +++ /dev/null @@ -1,23 +0,0 @@ -bytecode-0.15.2.dev4+gc87faa2.dist-info/COPYING,sha256=15CDvwHVcioF_s6S_mWdkWdw96tvB21WZKc8jvc8N5M,1094 -bytecode-0.15.2.dev4+gc87faa2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -bytecode-0.15.2.dev4+gc87faa2.dist-info/METADATA,sha256=msfGAb_7qRcxbHQwTg-b2IAAXopiE6KWzm5F2zCSPMg,4780 -bytecode-0.15.2.dev4+gc87faa2.dist-info/RECORD,, -bytecode-0.15.2.dev4+gc87faa2.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -bytecode-0.15.2.dev4+gc87faa2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92 -bytecode-0.15.2.dev4+gc87faa2.dist-info/direct_url.json,sha256=If2BeyiZnL-o6_qt3rkJuCphTAj0k2pM1S7TvbCLAAw,145 -bytecode-0.15.2.dev4+gc87faa2.dist-info/top_level.txt,sha256=9BhdB7HqYZ-PvHNoWX6ilwLYWQqcgEOLwdb3aXm5Gys,9 -bytecode/__init__.py,sha256=QyRvM53HT111OMumYJRD1FC3al7MlZL1bXf2MJ5XBMo,6848 -bytecode/__pycache__/__init__.cpython-311.pyc,, -bytecode/__pycache__/bytecode.cpython-311.pyc,, -bytecode/__pycache__/cfg.cpython-311.pyc,, -bytecode/__pycache__/concrete.cpython-311.pyc,, -bytecode/__pycache__/flags.cpython-311.pyc,, -bytecode/__pycache__/instr.cpython-311.pyc,, -bytecode/__pycache__/version.cpython-311.pyc,, -bytecode/bytecode.py,sha256=6oveflTRGnrzTQEP9Z0tp6ySwmXQ_DXIibdAGOZt5lY,11126 -bytecode/cfg.py,sha256=F8GhLWqDJlQc8cbL6X8HMa1QSVLGmZmBuYC8G4P201A,41794 -bytecode/concrete.py,sha256=kVTVMtA6yUsxLFa6LpCiXOUlgo3OxCIS2ykANO-MWjw,52189 -bytecode/flags.py,sha256=2ONreTJOAFDPXlDZle1t9FnkxI1E7SQGj3DDtrnYoMg,6093 -bytecode/instr.py,sha256=T0y13qaYG_9N6KXoafmFASKHITqdkvjOQnEor6QkF-I,26785 -bytecode/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -bytecode/version.py,sha256=nfCHWDf48S87YSKe2l_-l3kZTq6AFgQOPx7-Q20xhnE,566 diff --git a/_pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/direct_url.json b/_pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/direct_url.json deleted file mode 100644 index 9ee6c4f88..000000000 --- a/_pydevd_frame_eval/vendored/bytecode-0.15.2.dev4+gc87faa2.dist-info/direct_url.json +++ /dev/null @@ -1 +0,0 @@ -{"url": "https://github.com/MatthieuDartiailh/bytecode.git", "vcs_info": {"commit_id": "c87faa29ccc0cbc1198d581500848f1ab9096bd5", "vcs": "git"}} \ No newline at end of file diff --git a/_pydevd_frame_eval/vendored/bytecode/__init__.py b/_pydevd_frame_eval/vendored/bytecode/__init__.py index 1d6c17751..f970fbc00 100644 --- a/_pydevd_frame_eval/vendored/bytecode/__init__.py +++ b/_pydevd_frame_eval/vendored/bytecode/__init__.py @@ -1,3 +1,5 @@ +__version__ = "0.13.0.dev" + __all__ = [ "Label", "Instr", @@ -8,57 +10,34 @@ "ControlFlowGraph", "CompilerFlags", "Compare", - "BinaryOp", - "__version__", ] -from io import StringIO -from typing import List, Union - -# import needed to use it in bytecode.py +from _pydevd_frame_eval.vendored.bytecode.flags import CompilerFlags +from _pydevd_frame_eval.vendored.bytecode.instr import ( + UNSET, + Label, + SetLineno, + Instr, + CellVar, + FreeVar, # noqa + Compare, +) from _pydevd_frame_eval.vendored.bytecode.bytecode import ( BaseBytecode, - Bytecode, _BaseBytecodeList, _InstrList, -) - -# import needed to use it in bytecode.py -from _pydevd_frame_eval.vendored.bytecode.cfg import BasicBlock, ControlFlowGraph - -# import needed to use it in bytecode.py + Bytecode, +) # noqa from _pydevd_frame_eval.vendored.bytecode.concrete import ( - ConcreteBytecode, ConcreteInstr, + ConcreteBytecode, # noqa + # import needed to use it in bytecode.py _ConvertBytecodeToConcrete, ) -from _pydevd_frame_eval.vendored.bytecode.flags import CompilerFlags - -# import needed to use it in bytecode.py -from _pydevd_frame_eval.vendored.bytecode.instr import ( - UNSET, - BinaryOp, - CellVar, - Compare, - FreeVar, - Instr, - Intrinsic1Op, - Intrinsic2Op, - Label, - SetLineno, - TryBegin, - TryEnd, -) -from _pydevd_frame_eval.vendored.bytecode.version import __version__ - - -def format_bytecode( - bytecode: Union[Bytecode, ConcreteBytecode, ControlFlowGraph], - *, - lineno: bool = False, -) -> str: - try_begins: List[TryBegin] = [] +from _pydevd_frame_eval.vendored.bytecode.cfg import BasicBlock, ControlFlowGraph # noqa +import sys +def dump_bytecode(bytecode, *, lineno=False, stream=sys.stdout): def format_line(index, line): nonlocal cur_lineno, prev_lineno if lineno: @@ -90,34 +69,6 @@ def format_instr(instr, labels=None): text = "%s %s" % (text, arg) return text - def format_try_begin(instr: TryBegin, labels: dict) -> str: - if isinstance(instr.target, Label): - try: - arg = "<%s>" % labels[instr.target] - except KeyError: - arg = "" - else: - try: - arg = "<%s>" % labels[id(instr.target)] - except KeyError: - arg = "" - line = "TryBegin %s -> %s [%s]" % ( - len(try_begins), - arg, - instr.stack_depth, - ) + (" last_i" if instr.push_lasti else "") - - # Track the seen try begin - try_begins.append(instr) - - return line - - def format_try_end(instr: TryEnd) -> str: - i = try_begins.index(instr.entry) if instr.entry in try_begins else "" - return "TryEnd (%s)" % i - - buffer = StringIO() - indent = " " * 4 cur_lineno = bytecode.first_lineno @@ -125,35 +76,22 @@ def format_try_end(instr: TryEnd) -> str: if isinstance(bytecode, ConcreteBytecode): offset = 0 - for c_instr in bytecode: + for instr in bytecode: fields = [] - if c_instr.lineno is not None: - cur_lineno = c_instr.lineno + if instr.lineno is not None: + cur_lineno = instr.lineno if lineno: - fields.append(format_instr(c_instr)) + fields.append(format_instr(instr)) line = "".join(fields) line = format_line(offset, line) else: - fields.append("% 3s %s" % (offset, format_instr(c_instr))) + fields.append("% 3s %s" % (offset, format_instr(instr))) line = "".join(fields) - buffer.write(line + "\n") - - if isinstance(c_instr, ConcreteInstr): - offset += c_instr.size - - if bytecode.exception_table: - buffer.write("\n") - buffer.write("Exception table:\n") - for entry in bytecode.exception_table: - buffer.write( - f"{entry.start_offset} to {entry.stop_offset} -> " - f"{entry.target} [{entry.stack_depth}]" - + (" lasti" if entry.push_lasti else "") - + "\n" - ) + print(line, file=stream) + offset += instr.size elif isinstance(bytecode, Bytecode): - labels: dict[Label, str] = {} + labels = {} for index, instr in enumerate(bytecode): if isinstance(instr, Label): labels[instr] = "label_instr%s" % index @@ -163,59 +101,30 @@ def format_try_end(instr: TryEnd) -> str: label = labels[instr] line = "%s:" % label if index != 0: - buffer.write("\n") - elif isinstance(instr, TryBegin): - line = indent + format_line(index, format_try_begin(instr, labels)) - indent += " " - elif isinstance(instr, TryEnd): - indent = indent[:-2] - line = indent + format_line(index, format_try_end(instr)) + print(file=stream) else: if instr.lineno is not None: cur_lineno = instr.lineno line = format_instr(instr, labels) line = indent + format_line(index, line) - buffer.write(line + "\n") - buffer.write("\n") - + print(line, file=stream) + print(file=stream) elif isinstance(bytecode, ControlFlowGraph): - cfg_labels = {} + labels = {} for block_index, block in enumerate(bytecode, 1): - cfg_labels[id(block)] = "block%s" % block_index + labels[id(block)] = "block%s" % block_index - for block in bytecode: - buffer.write("%s:\n" % cfg_labels[id(block)]) - seen_instr = False + for block_index, block in enumerate(bytecode, 1): + print("%s:" % labels[id(block)], file=stream) + prev_lineno = None for index, instr in enumerate(block): - if isinstance(instr, TryBegin): - line = indent + format_line( - index, format_try_begin(instr, cfg_labels) - ) - indent += " " - elif isinstance(instr, TryEnd): - if seen_instr: - indent = indent[:-2] - line = indent + format_line(index, format_try_end(instr)) - else: - if isinstance(instr, Instr): - seen_instr = True - if instr.lineno is not None: - cur_lineno = instr.lineno - line = format_instr(instr, cfg_labels) - line = indent + format_line(index, line) - buffer.write(line + "\n") + if instr.lineno is not None: + cur_lineno = instr.lineno + line = format_instr(instr, labels) + line = indent + format_line(index, line) + print(line, file=stream) if block.next_block is not None: - buffer.write(indent + "-> %s\n" % cfg_labels[id(block.next_block)]) - buffer.write("\n") + print(indent + "-> %s" % labels[id(block.next_block)], file=stream) + print(file=stream) else: raise TypeError("unknown bytecode class") - - return buffer.getvalue()[:-1] - - -def dump_bytecode( - bytecode: Union[Bytecode, ConcreteBytecode, ControlFlowGraph], - *, - lineno: bool = False, -) -> None: - print(format_bytecode(bytecode, lineno=lineno)) diff --git a/_pydevd_frame_eval/vendored/bytecode/bytecode.py b/_pydevd_frame_eval/vendored/bytecode/bytecode.py index 6de7aa59d..2ead43f08 100644 --- a/_pydevd_frame_eval/vendored/bytecode/bytecode.py +++ b/_pydevd_frame_eval/vendored/bytecode/bytecode.py @@ -1,75 +1,49 @@ # alias to keep the 'bytecode' variable free import sys -import types -from abc import abstractmethod -from typing import ( - Any, - Dict, - Generic, - Iterator, - List, - Optional, - Sequence, - SupportsIndex, - TypeVar, - Union, - overload, -) - from _pydevd_frame_eval.vendored import bytecode as _bytecode -from _pydevd_frame_eval.vendored.bytecode.flags import CompilerFlags, infer_flags -from _pydevd_frame_eval.vendored.bytecode.instr import ( - _UNSET, - UNSET, - BaseInstr, - Instr, - Label, - SetLineno, - TryBegin, - TryEnd, -) +from _pydevd_frame_eval.vendored.bytecode.instr import UNSET, Label, SetLineno, Instr +from _pydevd_frame_eval.vendored.bytecode.flags import infer_flags class BaseBytecode: - def __init__(self) -> None: + + def __init__(self): self.argcount = 0 - self.posonlyargcount = 0 + if sys.version_info > (3, 8): + self.posonlyargcount = 0 self.kwonlyargcount = 0 self.first_lineno = 1 self.name = "" - self.qualname = self.name self.filename = "" - self.docstring: Union[str, None, _UNSET] = UNSET - # We cannot recreate cellvars/freevars from instructions because of super() - # special-case, which involves an implicit __class__ cell/free variable - # We could try to detect it. - # CPython itself breaks if one aliases super so we could maybe make it work - # but it will require careful design and will be done later in the future. - self.cellvars: List[str] = [] - self.freevars: List[str] = [] - self._flags: CompilerFlags = CompilerFlags(0) - - def _copy_attr_from(self, bytecode: "BaseBytecode") -> None: + self.docstring = UNSET + self.cellvars = [] + # we cannot recreate freevars from instructions because of super() + # special-case + self.freevars = [] + self._flags = _bytecode.CompilerFlags(0) + + def _copy_attr_from(self, bytecode): self.argcount = bytecode.argcount - self.posonlyargcount = bytecode.posonlyargcount + if sys.version_info > (3, 8): + self.posonlyargcount = bytecode.posonlyargcount self.kwonlyargcount = bytecode.kwonlyargcount self.flags = bytecode.flags self.first_lineno = bytecode.first_lineno self.name = bytecode.name - self.qualname = bytecode.qualname self.filename = bytecode.filename self.docstring = bytecode.docstring self.cellvars = list(bytecode.cellvars) self.freevars = list(bytecode.freevars) - def __eq__(self, other: Any) -> bool: - if type(self) is not type(other): + def __eq__(self, other): + if type(self) != type(other): return False if self.argcount != other.argcount: return False - if self.posonlyargcount != other.posonlyargcount: - return False + if sys.version_info > (3, 8): + if self.posonlyargcount != other.posonlyargcount: + return False if self.kwonlyargcount != other.kwonlyargcount: return False if self.flags != other.flags: @@ -80,8 +54,6 @@ def __eq__(self, other: Any) -> bool: return False if self.name != other.name: return False - if self.qualname != other.qualname: - return False if self.docstring != other.docstring: return False if self.cellvars != other.cellvars: @@ -94,39 +66,22 @@ def __eq__(self, other: Any) -> bool: return True @property - def flags(self) -> CompilerFlags: + def flags(self): return self._flags @flags.setter - def flags(self, value: CompilerFlags) -> None: - if not isinstance(value, CompilerFlags): - value = CompilerFlags(value) + def flags(self, value): + if not isinstance(value, _bytecode.CompilerFlags): + value = _bytecode.CompilerFlags(value) self._flags = value - def update_flags(self, *, is_async: Optional[bool] = None) -> None: - # infer_flags reasonably only accept concrete subclasses - self.flags = infer_flags(self, is_async) # type: ignore - - @abstractmethod - def compute_stacksize(self, *, check_pre_and_post: bool = True) -> int: - raise NotImplementedError + def update_flags(self, *, is_async=None): + self.flags = infer_flags(self, is_async) -T = TypeVar("T", bound="_BaseBytecodeList") -U = TypeVar("U") - - -class _BaseBytecodeList(BaseBytecode, list, Generic[U]): +class _BaseBytecodeList(BaseBytecode, list): """List subclass providing type stable slicing and copying.""" - @overload - def __getitem__(self, index: SupportsIndex) -> U: - ... - - @overload - def __getitem__(self: T, index: slice) -> T: - ... - def __getitem__(self, index): value = super().__getitem__(index) if isinstance(index, slice): @@ -135,13 +90,12 @@ def __getitem__(self, index): return value - def copy(self: T) -> T: - # This is a list subclass and works - new = type(self)(super().copy()) # type: ignore + def copy(self): + new = type(self)(super().copy()) new._copy_attr_from(self) return new - def legalize(self) -> None: + def legalize(self): """Check that all the element of the list are valid and remove SetLineno.""" lineno_pos = [] set_lineno = None @@ -152,20 +106,20 @@ def legalize(self) -> None: set_lineno = instr.lineno lineno_pos.append(pos) continue - # Filter out other pseudo instructions - if not isinstance(instr, BaseInstr): + # Filter out Labels + if not isinstance(instr, Instr): continue if set_lineno is not None: instr.lineno = set_lineno - elif instr.lineno is UNSET: + elif instr.lineno is None: instr.lineno = current_lineno - elif instr.lineno is not None: + else: current_lineno = instr.lineno for i in reversed(lineno_pos): del self[i] - def __iter__(self) -> Iterator[U]: + def __iter__(self): instructions = super().__iter__() for instr in instructions: self._check_instr(instr) @@ -175,38 +129,22 @@ def _check_instr(self, instr): raise NotImplementedError() -V = TypeVar("V") - +class _InstrList(list): -class _InstrList(List[V]): - # Providing a stricter typing for this helper whose use is limited to the __eq__ - # implementation is more effort than it is worth. - def _flat(self) -> List: - instructions: List = [] + def _flat(self): + instructions = [] labels = {} jumps = [] - try_begins: Dict[TryBegin, int] = {} - try_jumps = [] offset = 0 - instr: Any for index, instr in enumerate(self): if isinstance(instr, Label): instructions.append("label_instr%s" % index) labels[instr] = offset - elif isinstance(instr, TryBegin): - try_begins.setdefault(instr, len(try_begins)) - assert isinstance(instr.target, Label) - try_jumps.append((instr.target, len(instructions))) - instructions.append(instr) - elif isinstance(instr, TryEnd): - instructions.append(("TryEnd", try_begins[instr.entry])) else: if isinstance(instr, Instr) and isinstance(instr.arg, Label): target_label = instr.arg - instr = _bytecode.ConcreteInstr( - instr.name, 0, location=instr.location - ) + instr = _bytecode.ConcreteInstr(instr.name, 0, lineno=instr.lineno) jumps.append((target_label, instr)) instructions.append(instr) offset += 1 @@ -214,117 +152,64 @@ def _flat(self) -> List: for target_label, instr in jumps: instr.arg = labels[target_label] - for target_label, index in try_jumps: - instr = instructions[index] - assert isinstance(instr, TryBegin) - instructions[index] = ( - "TryBegin", - try_begins[instr], - labels[target_label], - instr.push_lasti, - ) - return instructions - def __eq__(self, other: Any) -> bool: + def __eq__(self, other): if not isinstance(other, _InstrList): other = _InstrList(other) return self._flat() == other._flat() -class Bytecode( - _InstrList[Union[Instr, Label, TryBegin, TryEnd, SetLineno]], - _BaseBytecodeList[Union[Instr, Label, TryBegin, TryEnd, SetLineno]], -): - def __init__( - self, - instructions: Sequence[Union[Instr, Label, TryBegin, TryEnd, SetLineno]] = (), - ) -> None: +class Bytecode(_InstrList, _BaseBytecodeList): + + def __init__(self, instructions=()): BaseBytecode.__init__(self) - self.argnames: List[str] = [] + self.argnames = [] for instr in instructions: self._check_instr(instr) self.extend(instructions) - def __iter__(self) -> Iterator[Union[Instr, Label, TryBegin, TryEnd, SetLineno]]: + def __iter__(self): instructions = super().__iter__() - seen_try_begin = False for instr in instructions: self._check_instr(instr) - if isinstance(instr, TryBegin): - if seen_try_begin: - raise RuntimeError("TryBegin pseudo instructions cannot be nested.") - seen_try_begin = True - elif isinstance(instr, TryEnd): - seen_try_begin = False yield instr - def _check_instr(self, instr: Any) -> None: - if not isinstance(instr, (Label, SetLineno, Instr, TryBegin, TryEnd)): + def _check_instr(self, instr): + if not isinstance(instr, (Label, SetLineno, Instr)): raise ValueError( "Bytecode must only contain Label, " "SetLineno, and Instr objects, " "but %s was found" % type(instr).__name__ ) - def _copy_attr_from(self, bytecode: BaseBytecode) -> None: + def _copy_attr_from(self, bytecode): super()._copy_attr_from(bytecode) if isinstance(bytecode, Bytecode): self.argnames = bytecode.argnames @staticmethod - def from_code( - code: types.CodeType, - prune_caches: bool = True, - conserve_exception_block_stackdepth: bool = False, - ) -> "Bytecode": + def from_code(code): + if sys.version_info[:2] >= (3, 11): + raise RuntimeError('This is not updated for Python 3.11 onwards, use only up to Python 3.10!!') concrete = _bytecode.ConcreteBytecode.from_code(code) - return concrete.to_bytecode( - prune_caches=prune_caches, - conserve_exception_block_stackdepth=conserve_exception_block_stackdepth, - ) + return concrete.to_bytecode() - def compute_stacksize(self, *, check_pre_and_post: bool = True) -> int: + def compute_stacksize(self, *, check_pre_and_post=True): cfg = _bytecode.ControlFlowGraph.from_bytecode(self) return cfg.compute_stacksize(check_pre_and_post=check_pre_and_post) def to_code( - self, - compute_jumps_passes: Optional[int] = None, - stacksize: Optional[int] = None, - *, - check_pre_and_post: bool = True, - compute_exception_stack_depths: bool = True, - ) -> types.CodeType: + self, compute_jumps_passes=None, stacksize=None, *, check_pre_and_post=True + ): # Prevent reconverting the concrete bytecode to bytecode and cfg to do the # calculation if we need to do it. - if stacksize is None or ( - sys.version_info >= (3, 11) and compute_exception_stack_depths - ): - cfg = _bytecode.ControlFlowGraph.from_bytecode(self) - stacksize = cfg.compute_stacksize( - check_pre_and_post=check_pre_and_post, - compute_exception_stack_depths=compute_exception_stack_depths, - ) - self = cfg.to_bytecode() - compute_exception_stack_depths = False # avoid redoing everything - bc = self.to_concrete_bytecode( - compute_jumps_passes=compute_jumps_passes, - compute_exception_stack_depths=compute_exception_stack_depths, - ) - return bc.to_code( - stacksize=stacksize, - compute_exception_stack_depths=compute_exception_stack_depths, - ) - - def to_concrete_bytecode( - self, - compute_jumps_passes: Optional[int] = None, - compute_exception_stack_depths: bool = True, - ) -> "_bytecode.ConcreteBytecode": + if stacksize is None: + stacksize = self.compute_stacksize(check_pre_and_post=check_pre_and_post) + bc = self.to_concrete_bytecode(compute_jumps_passes=compute_jumps_passes) + return bc.to_code(stacksize=stacksize) + + def to_concrete_bytecode(self, compute_jumps_passes=None): converter = _bytecode._ConvertBytecodeToConcrete(self) - return converter.to_concrete_bytecode( - compute_jumps_passes=compute_jumps_passes, - compute_exception_stack_depths=compute_exception_stack_depths, - ) + return converter.to_concrete_bytecode(compute_jumps_passes=compute_jumps_passes) diff --git a/_pydevd_frame_eval/vendored/bytecode/cfg.py b/_pydevd_frame_eval/vendored/bytecode/cfg.py index 132085015..5e2f32290 100644 --- a/_pydevd_frame_eval/vendored/bytecode/cfg.py +++ b/_pydevd_frame_eval/vendored/bytecode/cfg.py @@ -1,61 +1,33 @@ import sys -import types -from collections import defaultdict -from dataclasses import dataclass -from typing import ( - Any, - Dict, - Generator, - Iterable, - Iterator, - List, - Optional, - Set, - SupportsIndex, - Tuple, - TypeVar, - Union, - overload, -) # alias to keep the 'bytecode' variable free from _pydevd_frame_eval.vendored import bytecode as _bytecode from _pydevd_frame_eval.vendored.bytecode.concrete import ConcreteInstr from _pydevd_frame_eval.vendored.bytecode.flags import CompilerFlags -from _pydevd_frame_eval.vendored.bytecode.instr import UNSET, Instr, Label, SetLineno, TryBegin, TryEnd +from _pydevd_frame_eval.vendored.bytecode.instr import Label, SetLineno, Instr -T = TypeVar("T", bound="BasicBlock") -U = TypeVar("U", bound="ControlFlowGraph") - -class BasicBlock(_bytecode._InstrList[Union[Instr, SetLineno, TryBegin, TryEnd]]): - def __init__( - self, - instructions: Optional[ - Iterable[Union[Instr, SetLineno, TryBegin, TryEnd]] - ] = None, - ) -> None: +class BasicBlock(_bytecode._InstrList): + def __init__(self, instructions=None): # a BasicBlock object, or None - self.next_block: Optional["BasicBlock"] = None + self.next_block = None if instructions: super().__init__(instructions) - def __iter__(self) -> Iterator[Union[Instr, SetLineno, TryBegin, TryEnd]]: + def __iter__(self): index = 0 while index < len(self): instr = self[index] index += 1 - if not isinstance(instr, (SetLineno, Instr, TryBegin, TryEnd)): + if not isinstance(instr, (SetLineno, Instr)): raise ValueError( "BasicBlock must only contain SetLineno and Instr objects, " "but %s was found" % instr.__class__.__name__ ) if isinstance(instr, Instr) and instr.has_jump(): - if index < len(self) and any( - isinstance(self[i], Instr) for i in range(index, len(self)) - ): + if index < len(self): raise ValueError( "Only the last instruction of a basic " "block can be a jump" ) @@ -66,25 +38,8 @@ def __iter__(self) -> Iterator[Union[Instr, SetLineno, TryBegin, TryEnd]]: type(instr.arg).__name__, ) - if isinstance(instr, TryBegin): - if not isinstance(instr.target, BasicBlock): - raise ValueError( - "TryBegin target must a BasicBlock, got %s", - type(instr.target).__name__, - ) - yield instr - @overload - def __getitem__( - self, index: SupportsIndex - ) -> Union[Instr, SetLineno, TryBegin, TryEnd]: - ... - - @overload - def __getitem__(self: T, index: slice) -> T: - ... - def __getitem__(self, index): value = super().__getitem__(index) if isinstance(index, slice): @@ -93,19 +48,12 @@ def __getitem__(self, index): return value - def get_last_non_artificial_instruction(self) -> Optional[Instr]: - for instr in reversed(self): - if isinstance(instr, Instr): - return instr - - return None - - def copy(self: T) -> T: + def copy(self): new = type(self)(super().copy()) new.next_block = self.next_block return new - def legalize(self, first_lineno: int) -> int: + def legalize(self, first_lineno): """Check that all the element of the list are valid and remove SetLineno.""" lineno_pos = [] set_lineno = None @@ -116,14 +64,11 @@ def legalize(self, first_lineno: int) -> int: set_lineno = current_lineno = instr.lineno lineno_pos.append(pos) continue - if isinstance(instr, (TryBegin, TryEnd)): - continue - if set_lineno is not None: instr.lineno = set_lineno - elif instr.lineno is UNSET: + elif instr.lineno is None: instr.lineno = current_lineno - elif instr.lineno is not None: + else: current_lineno = instr.lineno for i in reversed(lineno_pos): @@ -131,373 +76,137 @@ def legalize(self, first_lineno: int) -> int: return current_lineno - def get_jump(self) -> Optional["BasicBlock"]: + def get_jump(self): if not self: return None - last_instr = self.get_last_non_artificial_instruction() - if last_instr is None or not last_instr.has_jump(): + last_instr = self[-1] + if not (isinstance(last_instr, Instr) and last_instr.has_jump()): return None target_block = last_instr.arg assert isinstance(target_block, BasicBlock) return target_block - def get_trailing_try_end(self, index: int): - while index + 1 < len(self): - if isinstance(b := self[index + 1], TryEnd): - return b - index += 1 - - return None - - -def _update_size(pre_delta, post_delta, size, maxsize, minsize): - size += pre_delta - if size < 0: - msg = "Failed to compute stacksize, got negative size" - raise RuntimeError(msg) - size += post_delta - maxsize = max(maxsize, size) - minsize = min(minsize, size) - return size, maxsize, minsize - - -# We can never have nested TryBegin, so we can simply update the min stack size -# when we encounter one and use the number we have when we encounter the TryEnd - - -@dataclass -class _StackSizeComputationStorage: - """Common storage shared by the computers involved in computing CFG stack usage.""" - - #: Should we check that all stack operation are "safe" i.e. occurs while there - #: is a sufficient number of items on the stack. - check_pre_and_post: bool - - #: Id the blocks for which an analysis is under progress to avoid getting stuck - #: in recursions. - seen_blocks: Set[int] - - #: Sizes and exception handling status with which the analysis of the block - #: has been performed. Used to avoid running multiple times equivalent analysis. - blocks_startsizes: Dict[int, Set[Tuple[int, Optional[bool]]]] - - #: Track the encountered TryBegin pseudo-instruction to update their target - #: depth at the end of the calculation. - try_begins: List[TryBegin] - - #: Stacksize that should be used for exception blocks. This is the smallest size - #: with which this block was reached which is the only size that can be safely - #: restored. - exception_block_startsize: Dict[int, int] - - #: Largest stack size used in an exception block. We record the size corresponding - #: to the smallest start size for the block since the interpreter enforces that - #: we start with this size. - exception_block_maxsize: Dict[int, int] - - -class _StackSizeComputer: - """Helper computing the stack usage for a single block.""" - - #: Common storage shared by all helpers involved in the stack size computation - common: _StackSizeComputationStorage - - #: Block this helper is running the computation for. - block: BasicBlock - - #: Current stack usage. - size: int - - #: Maximal stack usage. - maxsize: int - - #: Minimal stack usage. This value is only relevant in between a TryBegin/TryEnd - #: pair and determine the startsize for the exception handling block associated - #: with the try begin. - minsize: int - - #: Flag indicating if the block analyzed is an exception handler (i.e. a target - #: of a TryBegin). - exception_handler: Optional[bool] - - #: TryBegin that was encountered before jumping to this block and for which - #: no try end was met yet. - pending_try_begin: Optional[TryBegin] - - def __init__( - self, - common: _StackSizeComputationStorage, - block: BasicBlock, - size: int, - maxsize: int, - minsize: int, - exception_handler: Optional[bool], - pending_try_begin: Optional[TryBegin], - ) -> None: - self.common = common - self.block = block - self.size = size - self.maxsize = maxsize - self.minsize = minsize - self.exception_handler = exception_handler - self.pending_try_begin = pending_try_begin - self._current_try_begin = pending_try_begin - - def run(self) -> Generator[Union["_StackSizeComputer", int], int, None]: - """Iterate over the block instructions to compute stack usage.""" - # Blocks are not hashable but in this particular context we know we won't be - # modifying blocks in place so we can safely use their id as hash rather than - # making them generally hashable which would be weird since they are list - # subclasses - block_id = id(self.block) - - # If the block is currently being visited (seen = True) or - # it was visited previously with parameters that makes the computation - # irrelevant return the maxsize. - fingerprint = (self.size, self.exception_handler) - if id(self.block) in self.common.seen_blocks or ( - not self._is_stacksize_computation_relevant(block_id, fingerprint) - ): - yield self.maxsize - - # Prevent recursive visit of block if two blocks are nested (jump from one - # to the other). - self.common.seen_blocks.add(block_id) - - # Track which size has been used to run an analysis to avoid re-running multiple - # times the same calculation. - self.common.blocks_startsizes[block_id].add(fingerprint) - - # If this block is an exception handler reached through the exception table - # we will push some extra objects on the stack before processing start. - if self.exception_handler is not None: - self._update_size(0, 1 + self.exception_handler) - # True is used to indicated that push_lasti is True, leading to pushing - # an extra object on the stack. - - for i, instr in enumerate(self.block): - # Ignore SetLineno - if isinstance(instr, (SetLineno)): - continue - - # When we encounter a TryBegin, we: - # - store it as the current TryBegin (since TryBegin cannot be nested) - # - record its existence to remember to update its stack size when - # the computation ends - # - update the minsize to the current size value since we need to - # know the minimal stack usage between the TryBegin/TryEnd pair to - # set the startsize of the exception handling block - # - # This approach does not require any special handling for with statements. - if isinstance(instr, TryBegin): - assert self._current_try_begin is None - self.common.try_begins.append(instr) - self._current_try_begin = instr - self.minsize = self.size - continue - - elif isinstance(instr, TryEnd): - # When we encounter a TryEnd we can start the computation for the - # exception block using the minimum stack size encountered since - # the TryBegin matching this TryEnd. - - # TryBegin cannot be nested so a TryEnd should always match the - # current try begin. However inside the CFG some blocks may - # start with a TryEnd relevant only when reaching this block - # through a particular jump. So we are lenient here. - if instr.entry is not self._current_try_begin: - continue - - # Compute the stack usage of the exception handler - assert isinstance(instr.entry.target, BasicBlock) - yield from self._compute_exception_handler_stack_usage( - instr.entry.target, - instr.entry.push_lasti, - ) - self._current_try_begin = None - continue - - # For instructions with a jump first compute the stacksize required when the - # jump is taken. - if instr.has_jump(): - effect = ( - instr.pre_and_post_stack_effect(jump=True) - if self.common.check_pre_and_post - else (instr.stack_effect(jump=True), 0) - ) - taken_size, maxsize, minsize = _update_size( - *effect, self.size, self.maxsize, self.minsize - ) - - # Yield the parameters required to compute the stacksize required - # by the block to which the jump points to and resume when we now - # the maxsize. - assert isinstance(instr.arg, BasicBlock) - maxsize = yield _StackSizeComputer( - self.common, - instr.arg, - taken_size, - maxsize, - minsize, - None, - # Do not propagate the TryBegin if a final instruction is followed - # by a TryEnd. - None - if instr.is_final() and self.block.get_trailing_try_end(i) - else self._current_try_begin, - ) - - # Update the maximum used size by the usage implied by the following - # the jump - self.maxsize = max(self.maxsize, maxsize) - - # For unconditional jumps abort early since the other instruction will - # never be seen. - if instr.is_uncond_jump(): - # Check for TryEnd after the final instruction which is possible - # TryEnd being only pseudo instructions - if te := self.block.get_trailing_try_end(i): - # TryBegin cannot be nested - assert te.entry is self._current_try_begin - - assert isinstance(te.entry.target, BasicBlock) - yield from self._compute_exception_handler_stack_usage( - te.entry.target, - te.entry.push_lasti, - ) - - self.common.seen_blocks.remove(id(self.block)) - yield self.maxsize - - # jump=False: non-taken path of jumps, or any non-jump +def _compute_stack_size(block, size, maxsize, *, check_pre_and_post=True): + """Generator used to reduce the use of function stacks. + + This allows to avoid nested recursion and allow to treat more cases. + + HOW-TO: + Following the methods of Trampoline + (see https://en.wikipedia.org/wiki/Trampoline_(computing)), + + We yield either: + + - the arguments that would be used in the recursive calls, i.e, + 'yield block, size, maxsize' instead of making a recursive call + '_compute_stack_size(block, size, maxsize)', if we encounter an + instruction jumping to another block or if the block is linked to + another one (ie `next_block` is set) + - the required stack from the stack if we went through all the instructions + or encountered an unconditional jump. + + In the first case, the calling function is then responsible for creating a + new generator with those arguments, iterating over it till exhaustion to + determine the stacksize required by the block and resuming this function + with the determined stacksize. + + """ + # If the block is currently being visited (seen = True) or if it was visited + # previously by using a larger starting size than the one in use, return the + # maxsize. + if block.seen or block.startsize >= size: + yield maxsize + + def update_size(pre_delta, post_delta, size, maxsize): + size += pre_delta + if size < 0: + msg = "Failed to compute stacksize, got negative size" + raise RuntimeError(msg) + size += post_delta + maxsize = max(maxsize, size) + return size, maxsize + + # Prevent recursive visit of block if two blocks are nested (jump from one + # to the other). + block.seen = True + block.startsize = size + + for instr in block: + + # Ignore SetLineno + if isinstance(instr, SetLineno): + continue + + # For instructions with a jump first compute the stacksize required when the + # jump is taken. + if instr.has_jump(): effect = ( - instr.pre_and_post_stack_effect(jump=False) - if self.common.check_pre_and_post - else (instr.stack_effect(jump=False), 0) + instr.pre_and_post_stack_effect(jump=True) + if check_pre_and_post + else (instr.stack_effect(jump=True), 0) ) - self._update_size(*effect) - - # Instruction is final (return, raise, ...) so any following instruction - # in the block is dead code. - if instr.is_final(): - # Check for TryEnd after the final instruction which is possible - # TryEnd being only pseudo instructions. - if te := self.block.get_trailing_try_end(i): - assert isinstance(te.entry.target, BasicBlock) - yield from self._compute_exception_handler_stack_usage( - te.entry.target, - te.entry.push_lasti, - ) - - self.common.seen_blocks.remove(id(self.block)) - - yield self.maxsize - - if self.block.next_block: - self.maxsize = yield _StackSizeComputer( - self.common, - self.block.next_block, - self.size, - self.maxsize, - self.minsize, - None, - self._current_try_begin, - ) - - self.common.seen_blocks.remove(id(self.block)) - - yield self.maxsize - - # --- Private API + taken_size, maxsize = update_size(*effect, size, maxsize) + # Yield the parameters required to compute the stacksize required + # by the block to which the jumnp points to and resume when we now + # the maxsize. + maxsize = yield instr.arg, taken_size, maxsize + + # For unconditional jumps abort early since the other instruction will + # never be seen. + if instr.is_uncond_jump(): + block.seen = False + yield maxsize + + # jump=False: non-taken path of jumps, or any non-jump + effect = ( + instr.pre_and_post_stack_effect(jump=False) + if check_pre_and_post + else (instr.stack_effect(jump=False), 0) + ) + size, maxsize = update_size(*effect, size, maxsize) - _current_try_begin: Optional[TryBegin] + if block.next_block: + maxsize = yield block.next_block, size, maxsize - def _update_size(self, pre_delta: int, post_delta: int) -> None: - size, maxsize, minsize = _update_size( - pre_delta, post_delta, self.size, self.maxsize, self.minsize - ) - self.size = size - self.minsize = minsize - self.maxsize = maxsize - - def _compute_exception_handler_stack_usage( - self, block: BasicBlock, push_lasti: bool - ) -> Generator[Union["_StackSizeComputer", int], int, None]: - b_id = id(block) - if self.minsize < self.common.exception_block_startsize[b_id]: - block_size = yield _StackSizeComputer( - self.common, - block, - self.minsize, - self.maxsize, - self.minsize, - push_lasti, - None, - ) - # The entry cannot be smaller than abs(stc.minimal_entry_size) as otherwise - # we an underflow would have occured. - self.common.exception_block_startsize[b_id] = self.minsize - self.common.exception_block_maxsize[b_id] = block_size - - def _is_stacksize_computation_relevant( - self, block_id: int, fingerprint: Tuple[int, Optional[bool]] - ) -> bool: - if sys.version_info >= (3, 11): - # The computation is relevant if the block was not visited previously - # with the same starting size and exception handler status than the - # one in use - return fingerprint not in self.common.blocks_startsizes[block_id] - else: - # The computation is relevant if the block was only visited with smaller - # starting sizes than the one in use - if sizes := self.common.blocks_startsizes[block_id]: - return fingerprint[0] > max(f[0] for f in sizes) - else: - return True + block.seen = False + yield maxsize class ControlFlowGraph(_bytecode.BaseBytecode): - def __init__(self) -> None: + def __init__(self): super().__init__() - self._blocks: List[BasicBlock] = [] - self._block_index: Dict[int, int] = {} - self.argnames: List[str] = [] + self._blocks = [] + self._block_index = {} + self.argnames = [] self.add_block() - def legalize(self) -> None: + def legalize(self): """Legalize all blocks.""" current_lineno = self.first_lineno for block in self._blocks: current_lineno = block.legalize(current_lineno) - def get_block_index(self, block: BasicBlock) -> int: + def get_block_index(self, block): try: return self._block_index[id(block)] except KeyError: - raise ValueError(f"the block {block} is not part of this bytecode") # noqa + raise ValueError("the block is not part of this bytecode") - def _add_block(self, block: BasicBlock) -> None: + def _add_block(self, block): block_index = len(self._blocks) self._blocks.append(block) self._block_index[id(block)] = block_index - def add_block( - self, instructions: Optional[Iterable[Union[Instr, SetLineno]]] = None - ) -> BasicBlock: + def add_block(self, instructions=None): block = BasicBlock(instructions) self._add_block(block) return block - def compute_stacksize( - self, - *, - check_pre_and_post: bool = True, - compute_exception_stack_depths: bool = True, - ) -> int: + def compute_stacksize(self, *, check_pre_and_post=True): """Compute the stack size by iterating through the blocks The implementation make use of a generator function to avoid issue with @@ -508,15 +217,10 @@ def compute_stacksize( if not self: return 0 - # Create the common storage for the calculation - common = _StackSizeComputationStorage( - check_pre_and_post, - seen_blocks=set(), - blocks_startsizes={id(b): set() for b in self}, - exception_block_startsize=dict.fromkeys([id(b) for b in self], 32768), - exception_block_maxsize=dict.fromkeys([id(b) for b in self], -32768), - try_begins=[], - ) + # Ensure that previous calculation do not impact this one. + for block in self: + block.seen = False + block.startsize = -32768 # INT_MIN # Starting with Python 3.10, generator and coroutines start with one object # on the stack (None, anything is an error). @@ -529,12 +233,12 @@ def compute_stacksize( initial_stack_size = 1 # Create a generator/coroutine responsible of dealing with the first block - coro = _StackSizeComputer( - common, self[0], initial_stack_size, 0, 0, None, None - ).run() + coro = _compute_stack_size( + self[0], initial_stack_size, 0, check_pre_and_post=check_pre_and_post + ) # Create a list of generator that have not yet been exhausted - coroutines: List[Generator[Union[_StackSizeComputer, int], int, None]] = [] + coroutines = [] push_coroutine = coroutines.append pop_coroutine = coroutines.pop @@ -542,12 +246,10 @@ def compute_stacksize( try: while True: - # Mypy does not seem to honor the fact that one must send None - # to a brand new generator irrespective of its send type. - args = coro.send(None) # type: ignore + args = coro.send(None) # Consume the stored generators as long as they return a simple - # integer that is to be used to resume the last stored generator. + # interger that is to be used to resume the last stored generator. while isinstance(args, int): coro = pop_coroutine() args = coro.send(args) @@ -555,134 +257,75 @@ def compute_stacksize( # Otherwise we enter a new block and we store the generator under # use and create a new one to process the new block push_coroutine(coro) - coro = args.run() + coro = _compute_stack_size(*args, check_pre_and_post=check_pre_and_post) except IndexError: # The exception occurs when all the generators have been exhausted - # in which case the last yielded value is the stacksize. - assert args is not None and isinstance(args, int) - - # Exception handling block size is reported separately since we need - # to report only the stack usage for the smallest start size for the - # block - args = max(args, *common.exception_block_maxsize.values()) - - # Check if there is dead code that may contain TryBegin/TryEnd pairs. - # For any such pair we set a huge size (the exception table format does not - # mandate a maximum value). We do so so that if the pair is fused with - # another it does not alter the computed size. - for block in self: - if not common.blocks_startsizes[id(block)]: - for i in block: - if isinstance(i, TryBegin) and i.stack_depth is UNSET: - i.stack_depth = 32768 - - # If requested update the TryBegin stack size - if compute_exception_stack_depths: - for tb in common.try_begins: - size = common.exception_block_startsize[id(tb.target)] - assert size >= 0 - tb.stack_depth = size - + # in which case teh last yielded value is the stacksize. + assert args is not None return args - def __repr__(self) -> str: + def __repr__(self): return "" % len(self._blocks) - # Helper to obtain a flat list of instr, which does not refer to block at - # anymore. Used for comparison of different CFG. - def _get_instructions( - self, - ) -> List: - instructions: List = [] - try_begins: Dict[TryBegin, int] = {} + def get_instructions(self): + instructions = [] + jumps = [] for block in self: - for index, instr in enumerate(block): - if isinstance(instr, TryBegin): - assert isinstance(instr.target, BasicBlock) - try_begins.setdefault(instr, len(try_begins)) - instructions.append( - ( - "TryBegin", - try_begins[instr], - self.get_block_index(instr.target), - instr.push_lasti, - ) - ) - elif isinstance(instr, TryEnd): - instructions.append(("TryEnd", try_begins[instr.entry])) - elif isinstance(instr, Instr) and ( - instr.has_jump() or instr.is_final() - ): - if instr.has_jump(): - target_block = instr.arg - assert isinstance(target_block, BasicBlock) - # We use a concrete instr here to be able to use an integer as - # argument rather than a Label. This is fine for comparison - # purposes which is our sole goal here. - c_instr = ConcreteInstr( - instr.name, - self.get_block_index(target_block), - location=instr.location, - ) - instructions.append(c_instr) - else: - instructions.append(instr) - - if te := block.get_trailing_try_end(index): - instructions.append(("TryEnd", try_begins[te.entry])) - break - else: - instructions.append(instr) + target_block = block.get_jump() + if target_block is not None: + instr = block[-1] + instr = ConcreteInstr(instr.name, 0, lineno=instr.lineno) + jumps.append((target_block, instr)) + + instructions.extend(block[:-1]) + instructions.append(instr) + else: + instructions.extend(block) + + for target_block, instr in jumps: + instr.arg = self.get_block_index(target_block) return instructions - def __eq__(self, other: Any) -> bool: - if type(self) is not type(other): + def __eq__(self, other): + if type(self) != type(other): return False if self.argnames != other.argnames: return False - instrs1 = self._get_instructions() - instrs2 = other._get_instructions() + instrs1 = self.get_instructions() + instrs2 = other.get_instructions() if instrs1 != instrs2: return False # FIXME: compare block.next_block return super().__eq__(other) - def __len__(self) -> int: + def __len__(self): return len(self._blocks) - def __iter__(self) -> Iterator[BasicBlock]: + def __iter__(self): return iter(self._blocks) - @overload - def __getitem__(self, index: Union[int, BasicBlock]) -> BasicBlock: - ... - - @overload - def __getitem__(self: U, index: slice) -> U: - ... - def __getitem__(self, index): if isinstance(index, BasicBlock): index = self.get_block_index(index) return self._blocks[index] - def __delitem__(self, index: Union[int, BasicBlock]) -> None: + def __delitem__(self, index): if isinstance(index, BasicBlock): index = self.get_block_index(index) block = self._blocks[index] del self._blocks[index] del self._block_index[id(block)] - for i in range(index, len(self)): - block = self._blocks[i] + for index in range(index, len(self)): + block = self._blocks[index] self._block_index[id(block)] -= 1 - def split_block(self, block: BasicBlock, index: int) -> BasicBlock: + def split_block(self, block, index): if not isinstance(block, BasicBlock): raise TypeError("expected block") block_index = self.get_block_index(block) @@ -715,225 +358,66 @@ def split_block(self, block: BasicBlock, index: int) -> BasicBlock: return block2 - def get_dead_blocks(self) -> List[BasicBlock]: - if not self: - return [] - - seen_block_ids = set() - stack = [self[0]] - while stack: - block = stack.pop() - if id(block) in seen_block_ids: - continue - seen_block_ids.add(id(block)) - for i in block: - if isinstance(i, Instr) and isinstance(i.arg, BasicBlock): - stack.append(i.arg) - elif isinstance(i, TryBegin): - assert isinstance(i.target, BasicBlock) - stack.append(i.target) - - return [b for b in self if id(b) not in seen_block_ids] - @staticmethod - def from_bytecode(bytecode: _bytecode.Bytecode) -> "ControlFlowGraph": + def from_bytecode(bytecode): # label => instruction index label_to_block_index = {} jumps = [] - try_end_locations = {} + block_starts = {} for index, instr in enumerate(bytecode): if isinstance(instr, Label): label_to_block_index[instr] = index - elif isinstance(instr, Instr) and isinstance(instr.arg, Label): - jumps.append((index, instr.arg)) - elif isinstance(instr, TryBegin): - assert isinstance(instr.target, Label) - jumps.append((index, instr.target)) - elif isinstance(instr, TryEnd): - try_end_locations[instr.entry] = index - - # Figure out on which index block targeted by a label start - block_starts = {} + else: + if isinstance(instr, Instr) and isinstance(instr.arg, Label): + jumps.append((index, instr.arg)) + for target_index, target_label in jumps: target_index = label_to_block_index[target_label] block_starts[target_index] = target_label - bytecode_blocks = ControlFlowGraph() + bytecode_blocks = _bytecode.ControlFlowGraph() bytecode_blocks._copy_attr_from(bytecode) bytecode_blocks.argnames = list(bytecode.argnames) # copy instructions, convert labels to block labels block = bytecode_blocks[0] labels = {} - jumping_instrs: List[Instr] = [] - # Map input TryBegin to CFG TryBegins (split across blocks may yield multiple - # TryBegin from a single in the bytecode). - try_begins: Dict[TryBegin, list[TryBegin]] = {} - # Storage for TryEnds that need to be inserted at the beginning of a block. - # We use a list because the same block can be reached through several paths - # with different active TryBegins - add_try_end: Dict[Label, List[TryEnd]] = defaultdict(list) - - # Track the currently active try begin - active_try_begin: Optional[TryBegin] = None - try_begin_inserted_in_block = False - last_instr: Optional[Instr] = None + jumps = [] for index, instr in enumerate(bytecode): - # Reference to the current block if we create a new one in the following. - old_block: BasicBlock | None = None - - # First we determine if we need to create a new block: - # - by checking the current instruction index if index in block_starts: old_label = block_starts[index] - # Create a new block if the last created one is not empty - # (of real instructions) - if index != 0 and (li := block.get_last_non_artificial_instruction()): - old_block = block + if index != 0: new_block = bytecode_blocks.add_block() - # If the last non artificial instruction is not final connect - # this block to the next. - if not li.is_final(): + if not block[-1].is_final(): block.next_block = new_block block = new_block if old_label is not None: labels[old_label] = block - - # - by inspecting the last instr - elif block.get_last_non_artificial_instruction() and last_instr is not None: - # The last instruction is final but we did not create a block - # -> sounds like a block of dead code but we preserve it - if last_instr.is_final(): - old_block = block + elif block and isinstance(block[-1], Instr): + if block[-1].is_final(): block = bytecode_blocks.add_block() - - # We are dealing with a conditional jump - elif last_instr.has_jump(): - assert isinstance(last_instr.arg, Label) - old_block = block + elif block[-1].has_jump(): new_block = bytecode_blocks.add_block() block.next_block = new_block block = new_block - # If we created a new block, we check: - # - if the current instruction is a TryEnd and if the last instruction - # is final in which case we insert the TryEnd in the old block. - # - if we have a currently active TryBegin for which we may need to - # create a TryEnd in the previous block and a new TryBegin in the - # new one because the blocks are not connected. - if old_block is not None: - temp = try_begin_inserted_in_block - try_begin_inserted_in_block = False - - if old_block is not None and last_instr is not None: - # The last instruction is final, if the current instruction is a - # TryEnd insert it in the same block and move to the next instruction - if last_instr.is_final() and isinstance(instr, TryEnd): - assert active_try_begin - nte = instr.copy() - nte.entry = try_begins[active_try_begin][-1] - old_block.append(nte) - active_try_begin = None - continue - - # If we have an active TryBegin and last_instr is: - elif active_try_begin is not None: - # - a jump whose target is beyond the TryEnd of the active - # TryBegin: we remember TryEnd should be prepended to the - # target block. - if ( - last_instr.has_jump() - and active_try_begin in try_end_locations - and ( - # last_instr is a jump so arg is a Label - label_to_block_index[last_instr.arg] # type: ignore - >= try_end_locations[active_try_begin] - ) - ): - assert isinstance(last_instr.arg, Label) - add_try_end[last_instr.arg].append( - TryEnd(try_begins[active_try_begin][-1]) - ) - - # - final and the try begin originate from the current block: - # we insert a TryEnd in the old block and a new TryBegin in - # the new one since the blocks are disconnected. - if last_instr.is_final() and temp: - old_block.append(TryEnd(try_begins[active_try_begin][-1])) - new_tb = TryBegin( - active_try_begin.target, active_try_begin.push_lasti - ) - block.append(new_tb) - # Add this new TryBegin to the map to properly update - # the target. - try_begins[active_try_begin].append(new_tb) - try_begin_inserted_in_block = True - - last_instr = None - if isinstance(instr, Label): continue # don't copy SetLineno objects - if isinstance(instr, (Instr, TryBegin, TryEnd)): - new = instr.copy() - if isinstance(instr, TryBegin): - assert active_try_begin is None - active_try_begin = instr - try_begin_inserted_in_block = True - assert isinstance(new, TryBegin) - try_begins[instr] = [new] - elif isinstance(instr, TryEnd): - assert isinstance(new, TryEnd) - new.entry = try_begins[instr.entry][-1] - active_try_begin = None - try_begin_inserted_in_block = False - else: - last_instr = instr - if isinstance(instr.arg, Label): - assert isinstance(new, Instr) - jumping_instrs.append(new) - - instr = new - + if isinstance(instr, Instr): + instr = instr.copy() + if isinstance(instr.arg, Label): + jumps.append(instr) block.append(instr) - # Insert the necessary TryEnds at the beginning of block that were marked - # (if we did not already insert an equivalent TryEnd earlier). - for lab, tes in add_try_end.items(): - block = labels[lab] - existing_te_entries = set() - index = 0 - # We use a while loop since the block cannot yet be iterated on since - # jumps still use labels instead of blocks - while index < len(block): - i = block[index] - index += 1 - if isinstance(i, TryEnd): - existing_te_entries.add(i.entry) - else: - break - for te in tes: - if te.entry not in existing_te_entries: - labels[lab].insert(0, te) - existing_te_entries.add(te.entry) - - # Replace labels by block in jumping instructions - for instr in jumping_instrs: + for instr in jumps: label = instr.arg - assert isinstance(label, Label) instr.arg = labels[label] - # Replace labels by block in TryBegin - for b_tb, c_tbs in try_begins.items(): - label = b_tb.target - assert isinstance(label, Label) - for c_tb in c_tbs: - c_tb.target = labels[label] - return bytecode_blocks - def to_bytecode(self) -> _bytecode.Bytecode: + def to_bytecode(self): """Convert to Bytecode.""" used_blocks = set() @@ -942,21 +426,9 @@ def to_bytecode(self) -> _bytecode.Bytecode: if target_block is not None: used_blocks.add(id(target_block)) - for tb in (i for i in block if isinstance(i, TryBegin)): - used_blocks.add(id(tb.target)) - labels = {} jumps = [] - try_begins = {} - seen_try_end: Set[TryBegin] = set() - instructions: List[Union[Instr, Label, TryBegin, TryEnd, SetLineno]] = [] - - # Track the last seen TryBegin and TryEnd to be able to fuse adjacent - # TryEnd/TryBegin pair which share the same target. - # In each case, we store the value found in the CFG and the value - # inserted in the bytecode. - last_try_begin: tuple[TryBegin, TryBegin] | None = None - last_try_end: tuple[TryEnd, TryEnd] | None = None + instructions = [] for block in self: if id(block) in used_blocks: @@ -966,73 +438,16 @@ def to_bytecode(self) -> _bytecode.Bytecode: for instr in block: # don't copy SetLineno objects - if isinstance(instr, (Instr, TryBegin, TryEnd)): - new = instr.copy() - if isinstance(instr, TryBegin): - # If due to jumps and split TryBegin, we encounter a TryBegin - # while we still have a TryBegin ensure they can be fused. - if last_try_begin is not None: - cfg_tb, byt_tb = last_try_begin - assert instr.target is cfg_tb.target - assert instr.push_lasti == cfg_tb.push_lasti - byt_tb.stack_depth = min( - byt_tb.stack_depth, instr.stack_depth - ) - - # If the TryBegin share the target and push_lasti of the - # entry of an adjacent TryEnd, omit the new TryBegin that - # was inserted to allow analysis of the CFG and remove - # the already inserted TryEnd. - if last_try_end is not None: - cfg_te, byt_te = last_try_end - entry = cfg_te.entry - if ( - entry.target is instr.target - and entry.push_lasti == instr.push_lasti - ): - # If we did not yet compute the required stack depth - # keep the value as UNSET - if entry.stack_depth is UNSET: - assert instr.stack_depth is UNSET - byt_te.entry.stack_depth = UNSET - else: - byt_te.entry.stack_depth = min( - entry.stack_depth, instr.stack_depth - ) - try_begins[instr] = byt_te.entry - instructions.remove(byt_te) - continue - assert isinstance(new, TryBegin) - try_begins[instr] = new - last_try_begin = (instr, new) - last_try_end = None - elif isinstance(instr, TryEnd): - # Only keep the first seen TryEnd matching a TryBegin - assert isinstance(new, TryEnd) - if instr.entry in seen_try_end: - continue - seen_try_end.add(instr.entry) - new.entry = try_begins[instr.entry] - last_try_begin = None - last_try_end = (instr, new) - elif isinstance(instr.arg, BasicBlock): - assert isinstance(new, Instr) - jumps.append(new) - last_try_end = None - else: - last_try_end = None - - instr = new - + if isinstance(instr, Instr): + instr = instr.copy() + if isinstance(instr.arg, BasicBlock): + jumps.append(instr) instructions.append(instr) # Map to new labels for instr in jumps: instr.arg = labels[id(instr.arg)] - for tb in set(try_begins.values()): - tb.target = labels[id(tb.target)] - bytecode = _bytecode.Bytecode() bytecode._copy_attr_from(self) bytecode.argnames = list(self.argnames) @@ -1040,22 +455,9 @@ def to_bytecode(self) -> _bytecode.Bytecode: return bytecode - def to_code( - self, - stacksize: Optional[int] = None, - *, - check_pre_and_post: bool = True, - compute_exception_stack_depths: bool = True, - ) -> types.CodeType: + def to_code(self, stacksize=None): """Convert to code.""" if stacksize is None: - stacksize = self.compute_stacksize( - check_pre_and_post=check_pre_and_post, - compute_exception_stack_depths=compute_exception_stack_depths, - ) + stacksize = self.compute_stacksize() bc = self.to_bytecode() - return bc.to_code( - stacksize=stacksize, - check_pre_and_post=False, - compute_exception_stack_depths=False, - ) + return bc.to_code(stacksize=stacksize) diff --git a/_pydevd_frame_eval/vendored/bytecode/concrete.py b/_pydevd_frame_eval/vendored/bytecode/concrete.py index 609150b10..bd756cba7 100644 --- a/_pydevd_frame_eval/vendored/bytecode/concrete.py +++ b/_pydevd_frame_eval/vendored/bytecode/concrete.py @@ -4,59 +4,29 @@ import struct import sys import types -from typing import ( - Any, - Dict, - Iterable, - Iterator, - List, - MutableSequence, - Optional, - Sequence, - Set, - Tuple, - Type, - TypeVar, - Union, -) # alias to keep the 'bytecode' variable free from _pydevd_frame_eval.vendored import bytecode as _bytecode -from _pydevd_frame_eval.vendored.bytecode.flags import CompilerFlags from _pydevd_frame_eval.vendored.bytecode.instr import ( - _UNSET, - BITFLAG2_INSTRUCTIONS, - BITFLAG_INSTRUCTIONS, - INTRINSIC, - INTRINSIC_1OP, - INTRINSIC_2OP, - PLACEHOLDER_LABEL, UNSET, - BaseInstr, - CellVar, - Compare, - FreeVar, Instr, - InstrArg, - InstrLocation, - Intrinsic1Op, - Intrinsic2Op, Label, SetLineno, - TryBegin, - TryEnd, - _check_arg_int, + FreeVar, + CellVar, + Compare, const_key, - opcode_has_argument, + _check_arg_int, ) + # - jumps use instruction # - lineno use bytes (dis.findlinestarts(code)) # - dis displays bytes OFFSET_AS_INSTRUCTION = sys.version_info >= (3, 10) -def _set_docstring(code: _bytecode.BaseBytecode, consts: Sequence) -> None: +def _set_docstring(code, consts): if not consts: return first_const = consts[0] @@ -64,60 +34,36 @@ def _set_docstring(code: _bytecode.BaseBytecode, consts: Sequence) -> None: code.docstring = first_const -T = TypeVar("T", bound="ConcreteInstr") - - -class ConcreteInstr(BaseInstr[int]): +class ConcreteInstr(Instr): """Concrete instruction. arg must be an integer in the range 0..2147483647. It has a read-only size attribute. - """ - # For ConcreteInstr the argument is always an integer - _arg: int - __slots__ = ("_size", "_extended_args", "offset") - def __init__( - self, - name: str, - arg: int=UNSET, - *, - lineno: Union[int, None, _UNSET]=UNSET, - location: Optional[InstrLocation]=None, - extended_args: Optional[int]=None, - offset=None, - ): + def __init__(self, name, arg=UNSET, *, lineno=None, extended_args=None, offset=None): # Allow to remember a potentially meaningless EXTENDED_ARG emitted by # Python to properly compute the size and avoid messing up the jump # targets self._extended_args = extended_args + self._set(name, arg, lineno) self.offset = offset - super().__init__(name, arg, lineno=lineno, location=location) - def _check_arg(self, name: str, opcode: int, arg: int) -> None: - if opcode_has_argument(opcode): + def _check_arg(self, name, opcode, arg): + if opcode >= _opcode.HAVE_ARGUMENT: if arg is UNSET: raise ValueError("operation %s requires an argument" % name) - _check_arg_int(arg, name) - # opcode == 0 corresponds to CACHE instruction in 3.11+ and was unused before - elif opcode == 0: - arg = arg if arg is not UNSET else 0 - _check_arg_int(arg, name) + _check_arg_int(name, arg) else: if arg is not UNSET: raise ValueError("operation %s has no argument" % name) - def _set( - self, - name: str, - arg: int, - ) -> None: - super()._set(name, arg) + def _set(self, name, arg, lineno): + super()._set(name, arg, lineno) size = 2 if arg is not UNSET: while arg > 0xFF: @@ -128,30 +74,21 @@ def _set( self._size = size @property - def size(self) -> int: + def size(self): return self._size - def _cmp_key(self) -> Tuple[Optional[InstrLocation], str, int]: - return (self._location, self._name, self._arg) - - def get_jump_target(self, instr_offset: int) -> Optional[int]: - # When a jump arg is zero the jump always points to the first non-CACHE - # opcode following the jump. The passed in offset is the offset at - # which the jump opcode starts. So to compute the target, we add to it - # the instruction size (accounting for extended args) and the - # number of caches expected to follow the jump instruction. - s = ( - (self._size // 2) if OFFSET_AS_INSTRUCTION else self._size - ) + self.use_cache_opcodes() - if self.is_forward_rel_jump(): + def _cmp_key(self, labels=None): + return (self._lineno, self._name, self._arg) + + def get_jump_target(self, instr_offset): + if self._opcode in _opcode.hasjrel: + s = (self._size // 2) if OFFSET_AS_INSTRUCTION else self._size return instr_offset + s + self._arg - if self.is_backward_rel_jump(): - return instr_offset + s - self._arg - if self.is_abs_jump(): + if self._opcode in _opcode.hasjabs: return self._arg return None - def assemble(self) -> bytes: + def assemble(self): if self._arg is UNSET: return bytes((self._opcode, 0)) @@ -168,10 +105,10 @@ def assemble(self) -> bytes: return bytes(b) @classmethod - def disassemble(cls: Type[T], lineno: Optional[int], code: bytes, offset: int) -> T: + def disassemble(cls, lineno, code, offset): index = 2 * offset if OFFSET_AS_INSTRUCTION else offset op = code[index] - if opcode_has_argument(op): + if op >= _opcode.HAVE_ARGUMENT: arg = code[index + 1] else: arg = UNSET @@ -179,112 +116,26 @@ def disassemble(cls: Type[T], lineno: Optional[int], code: bytes, offset: int) - # fabioz: added offset to ConcreteBytecode # Need to keep an eye on https://github.com/MatthieuDartiailh/bytecode/issues/48 in # case the library decides to add this in some other way. - return cls(name, arg, lineno=lineno, offset=offset) - - def use_cache_opcodes(self) -> int: - return ( - # Not supposed to be used but we need it - dis._inline_cache_entries[self._opcode] # type: ignore - if sys.version_info >= (3, 11) - else 0 - ) - + return cls(name, arg, lineno=lineno, offset=index) -class ExceptionTableEntry: - """Entry for a given line in the exception table. - All offset are expressed in instructions not in bytes. - - """ - - # : Offset in instruction between the beginning of the bytecode and the beginning - # : of this entry. - start_offset: int - - # : Offset in instruction between the beginning of the bytecode and the end - # : of this entry. This offset is inclusive meaning that the instruction it points - # : to is included in the try/except handling. - stop_offset: int - - # : Offset in instruction to the first instruction of the exception handling block. - target: int - - # : Minimal stack depth in the block delineated by start and stop - # : offset of the exception table entry. Used to restore the stack (by - # : popping items) when entering the exception handling block. - stack_depth: int - - # : Should the offset, at which an exception was raised, be pushed on the stack - # : before the exception itself (which is pushed as a single value)). - push_lasti: bool - - __slots__ = ("start_offset", "stop_offset", "target", "stack_depth", "push_lasti") - - def __init__( - self, - start_offset: int, - stop_offset: int, - target: int, - stack_depth: int, - push_lasti: bool, - ) -> None: - self.start_offset = start_offset - self.stop_offset = stop_offset - self.target = target - self.stack_depth = stack_depth - self.push_lasti = push_lasti - - def __repr__(self) -> str: - return ( - "ExceptionTableEntry(" - f"start_offset={self.start_offset}, " - f"stop_offset={self.stop_offset}, " - f"target={self.target}, " - f"stack_depth={self.stack_depth}, " - f"push_lasti={self.push_lasti}" - ) - - -class ConcreteBytecode(_bytecode._BaseBytecodeList[Union[ConcreteInstr, SetLineno]]): - # : List of "constant" objects for the bytecode - consts: List - - # : List of names used by local variables. - names: List[str] - - # : List of names used by input variables. - varnames: List[str] - - # : Table describing portion of the bytecode in which exceptions are caught and - # : where there are handled. - # : Used only in Python 3.11+ - exception_table: List[ExceptionTableEntry] - - def __init__( - self, - instructions=(), - *, - consts: tuple=(), - names: Tuple[str, ...]=(), - varnames: Iterable[str]=(), - exception_table: Optional[List[ExceptionTableEntry]]=None, - ): +class ConcreteBytecode(_bytecode._BaseBytecodeList): + def __init__(self, instructions=(), *, consts=(), names=(), varnames=()): super().__init__() self.consts = list(consts) self.names = list(names) self.varnames = list(varnames) - self.exception_table = exception_table or [] for instr in instructions: self._check_instr(instr) self.extend(instructions) - def __iter__(self) -> Iterator[Union[ConcreteInstr, SetLineno]]: + def __iter__(self): instructions = super().__iter__() for instr in instructions: self._check_instr(instr) yield instr - def _check_instr(self, instr: Any) -> None: + def _check_instr(self, instr): if not isinstance(instr, (ConcreteInstr, SetLineno)): raise ValueError( "ConcreteBytecode must only contain " @@ -299,11 +150,11 @@ def _copy_attr_from(self, bytecode): self.names = bytecode.names self.varnames = bytecode.varnames - def __repr__(self) -> str: + def __repr__(self): return "" % len(self) - def __eq__(self, other: Any) -> bool: - if type(self) is not type(other): + def __eq__(self, other): + if type(self) != type(other): return False const_keys1 = list(map(const_key, self.consts)) @@ -319,48 +170,26 @@ def __eq__(self, other: Any) -> bool: return super().__eq__(other) @staticmethod - def from_code( - code: types.CodeType, *, extended_arg: bool=False - ) -> "ConcreteBytecode": - instructions: MutableSequence[Union[SetLineno, ConcreteInstr]] - # For Python 3.11+ we use dis to extract the detailed location information at - # reduced maintenance cost. - if sys.version_info >= (3, 11): - instructions = [ - # dis.get_instructions automatically handle extended arg which - # we do not want, so we fold back arguments to be between 0 and 255 - ConcreteInstr( - i.opname, - i.arg % 256 if i.arg is not None else UNSET, - location=InstrLocation.from_positions(i.positions) - if i.positions - else None, - offset=i.offset - ) - for i in dis.get_instructions(code, show_caches=True) - ] - else: - if sys.version_info >= (3, 10): - line_starts = {offset: lineno for offset, _, lineno in code.co_lines()} - else: - line_starts = dict(dis.findlinestarts(code)) + def from_code(code, *, extended_arg=False): + line_starts = dict(dis.findlinestarts(code)) - # find block starts - instructions = [] - offset = 0 - lineno: Optional[int] = code.co_firstlineno - while offset < (len(code.co_code) // (2 if OFFSET_AS_INSTRUCTION else 1)): - lineno_off = (2 * offset) if OFFSET_AS_INSTRUCTION else offset - if lineno_off in line_starts: - lineno = line_starts[lineno_off] + # find block starts + instructions = [] + offset = 0 + lineno = code.co_firstlineno + while offset < (len(code.co_code) // (2 if OFFSET_AS_INSTRUCTION else 1)): + lineno_off = (2 * offset) if OFFSET_AS_INSTRUCTION else offset + if lineno_off in line_starts: + lineno = line_starts[lineno_off] - instr = ConcreteInstr.disassemble(lineno, code.co_code, offset) + instr = ConcreteInstr.disassemble(lineno, code.co_code, offset) - instructions.append(instr) - offset += (instr.size // 2) if OFFSET_AS_INSTRUCTION else instr.size + instructions.append(instr) + offset += (instr.size // 2) if OFFSET_AS_INSTRUCTION else instr.size bytecode = ConcreteBytecode() + # replace jump targets with blocks # HINT : in some cases Python generate useless EXTENDED_ARG opcode # with a value of zero. Such opcodes do not increases the size of the # following opcode the way a normal EXTENDED_ARG does. As a @@ -372,9 +201,10 @@ def from_code( bytecode.name = code.co_name bytecode.filename = code.co_filename - bytecode.flags = CompilerFlags(code.co_flags) + bytecode.flags = code.co_flags bytecode.argcount = code.co_argcount - bytecode.posonlyargcount = code.co_posonlyargcount + if sys.version_info >= (3, 8): + bytecode.posonlyargcount = code.co_posonlyargcount bytecode.kwonlyargcount = code.co_kwonlyargcount bytecode.first_lineno = code.co_firstlineno bytecode.names = list(code.co_names) @@ -383,38 +213,23 @@ def from_code( bytecode.freevars = list(code.co_freevars) bytecode.cellvars = list(code.co_cellvars) _set_docstring(bytecode, code.co_consts) - if sys.version_info >= (3, 11): - bytecode.exception_table = bytecode._parse_exception_table( - code.co_exceptiontable - ) - bytecode.qualname = code.co_qualname - else: - bytecode.qualname = bytecode.qualname bytecode[:] = instructions return bytecode @staticmethod - def _normalize_lineno( - instructions: Sequence[Union[ConcreteInstr, SetLineno]], first_lineno: int - ) -> Iterator[Tuple[int, ConcreteInstr]]: + def _normalize_lineno(instructions, first_lineno): lineno = first_lineno - # For each instruction compute an "inherited" lineno used: - # - on 3.8 and 3.9 for which a lineno is mandatory - # - to infer a lineno on 3.10+ if no lineno was provided for instr in instructions: - i_lineno = instr.lineno # if instr.lineno is not set, it's inherited from the previous # instruction, or from self.first_lineno - if i_lineno is not None and i_lineno is not UNSET: - lineno = i_lineno + if instr.lineno is not None: + lineno = instr.lineno if isinstance(instr, ConcreteInstr): yield (lineno, instr) - def _assemble_code( - self, - ) -> Tuple[bytes, List[Tuple[int, int, int, Optional[InstrLocation]]]]: + def _assemble_code(self): offset = 0 code_str = [] linenos = [] @@ -422,29 +237,26 @@ def _assemble_code( code_str.append(instr.assemble()) i_size = instr.size linenos.append( - ( - (offset * 2) if OFFSET_AS_INSTRUCTION else offset, - i_size, - lineno, - instr.location, - ) + ((offset * 2) if OFFSET_AS_INSTRUCTION else offset, i_size, lineno) ) offset += (i_size // 2) if OFFSET_AS_INSTRUCTION else i_size + code_str = b"".join(code_str) + return (code_str, linenos) - return (b"".join(code_str), linenos) - - # Used on 3.8 and 3.9 @staticmethod - def _assemble_lnotab( - first_lineno: int, linenos: List[Tuple[int, int, int, Optional[InstrLocation]]] - ) -> bytes: + def _assemble_lnotab(first_lineno, linenos): lnotab = [] old_offset = 0 old_lineno = first_lineno - for offset, _, lineno, _ in linenos: + for offset, _, lineno in linenos: dlineno = lineno - old_lineno if dlineno == 0: continue + # FIXME: be kind, force monotonic line numbers? add an option? + if dlineno < 0 and sys.version_info < (3, 6): + raise ValueError( + "negative line number delta is not supported " "on Python < 3.6" + ) old_lineno = lineno doff = offset - old_offset @@ -472,28 +284,16 @@ def _assemble_lnotab( return b"".join(lnotab) @staticmethod - def _pack_linetable( - linetable: List[bytes], doff: int, dlineno: Optional[int] - ) -> None: - if dlineno is not None: - # Ensure linenos are between -126 and +126, by using 127 lines jumps with - # a 0 byte offset - while dlineno < -127: - linetable.append(struct.pack("Bb", 0, -127)) - dlineno -= -127 + def _pack_linetable(doff, dlineno, linetable): - while dlineno > 127: - linetable.append(struct.pack("Bb", 0, 127)) - dlineno -= 127 + while dlineno < -127: + linetable.append(struct.pack("Bb", 0, -127)) + dlineno -= -127 - assert -127 <= dlineno <= 127 - else: - dlineno = -128 + while dlineno > 127: + linetable.append(struct.pack("Bb", 0, 127)) + dlineno -= 127 - # Ensure offsets are less than 255. - # If an offset is larger, we first mark the line change with an offset of 254 - # then use as many 254 offset with no line change to reduce the offset to - # less than 254. if doff > 254: linetable.append(struct.pack("Bb", 254, dlineno)) doff -= 254 @@ -507,238 +307,40 @@ def _pack_linetable( linetable.append(struct.pack("Bb", doff, dlineno)) assert 0 <= doff <= 254 + assert -127 <= dlineno <= 127 - # Used on 3.10 - def _assemble_linestable( - self, - first_lineno: int, - linenos: Iterable[Tuple[int, int, int, Optional[InstrLocation]]], - ) -> bytes: + + def _assemble_linestable(self, first_lineno, linenos): if not linenos: return b"" - linetable: List[bytes] = [] + linetable = [] old_offset = 0 - + iter_in = iter(linenos) - - offset, i_size, old_lineno, old_location = next(iter_in) - if old_location is not None: - old_dlineno = ( - old_location.lineno - first_lineno - if old_location.lineno is not None - else None - ) - else: - old_dlineno = old_lineno - first_lineno - - # i_size is used after we exit the loop - for offset, i_size, lineno, location in iter_in: # noqa - if location is not None: - dlineno = ( - location.lineno - old_lineno - if location.lineno is not None - else None - ) - else: - dlineno = lineno - old_lineno - - if dlineno == 0 or (old_dlineno is None and dlineno is None): + + offset, i_size, old_lineno = next(iter_in) + old_dlineno = old_lineno - first_lineno + for offset, i_size, lineno in iter_in: + dlineno = lineno - old_lineno + if dlineno == 0: continue old_lineno = lineno doff = offset - old_offset old_offset = offset - self._pack_linetable(linetable, doff, old_dlineno) + self._pack_linetable(doff, old_dlineno, linetable) old_dlineno = dlineno # Pack the line of the last instruction. doff = offset + i_size - old_offset - self._pack_linetable(linetable, doff, old_dlineno) + self._pack_linetable(doff, old_dlineno, linetable) return b"".join(linetable) - # The formats are describes in CPython/Objects/locations.md - @staticmethod - def _encode_location_varint(varint: int) -> bytearray: - encoded = bytearray() - # We encode on 6 bits - while True: - encoded.append(varint & 0x3F) - varint >>= 6 - if varint: - encoded[-1] |= 0x40 # bit 6 is set except on the last entry - else: - break - return encoded - - def _encode_location_svarint(self, svarint: int) -> bytearray: - if svarint < 0: - return self._encode_location_varint(((-svarint) << 1) | 1) - else: - return self._encode_location_varint(svarint << 1) - - # Python 3.11+ location format encoding - @staticmethod - def _pack_location_header(code: int, size: int) -> int: - return (1 << 7) + (code << 3) + (size - 1 if size <= 8 else 7) - - def _pack_location( - self, size: int, lineno: int, location: Optional[InstrLocation] - ) -> bytearray: - packed = bytearray() - - l_lineno: Optional[int] - # The location was not set so we infer a line. - if location is None: - l_lineno, end_lineno, col_offset, end_col_offset = ( - lineno, - None, - None, - None, - ) - else: - l_lineno, end_lineno, col_offset, end_col_offset = ( - location.lineno, - location.end_lineno, - location.col_offset, - location.end_col_offset, - ) - - # We have no location information so the code is 15 - if l_lineno is None: - packed.append(self._pack_location_header(15, size)) - - # No column info, code 13 - elif col_offset is None: - if end_lineno is not None and end_lineno != l_lineno: - raise ValueError( - "An instruction cannot have no column offset and span " - f"multiple lines (lineno: {l_lineno}, end lineno: {end_lineno}" - ) - packed.extend( - ( - self._pack_location_header(13, size), - *self._encode_location_svarint(l_lineno - lineno), - ) - ) - - # We enforce the end_lineno to be defined - else: - assert end_lineno is not None - assert end_col_offset is not None - - # Short forms - if ( - end_lineno == l_lineno - and l_lineno - lineno == 0 - and col_offset < 72 - and (end_col_offset - col_offset) <= 15 - ): - packed.extend( - ( - self._pack_location_header(col_offset // 8, size), - ((col_offset % 8) << 4) + (end_col_offset - col_offset), - ) - ) - - # One line form - elif ( - end_lineno == l_lineno - and l_lineno - lineno in (1, 2) - and col_offset < 256 - and end_col_offset < 256 - ): - packed.extend( - ( - self._pack_location_header(10 + l_lineno - lineno, size), - col_offset, - end_col_offset, - ) - ) - - # Long form - else: - packed.extend( - ( - self._pack_location_header(14, size), - *self._encode_location_svarint(l_lineno - lineno), - *self._encode_location_varint(end_lineno - l_lineno), - # When decoding in codeobject.c::advance_with_locations - # we remove 1 from the offset ... - *self._encode_location_varint(col_offset + 1), - *self._encode_location_varint(end_col_offset + 1), - ) - ) - - return packed - - def _push_locations( - self, - locations: List[bytearray], - size: int, - lineno: int, - location: InstrLocation, - ) -> int: - # We need the size in instruction not in bytes - size //= 2 - - # Repeatedly add element since we cannot cover more than 8 code - # elements. We recompute each time since in practice we will - # rarely loop. - while True: - locations.append(self._pack_location(size, lineno, location)) - # Update the lineno since if we need more than one entry the - # reference for the delta of the lineno change - lineno = location.lineno if location.lineno is not None else lineno - size -= 8 - if size < 1: - break - - return lineno - - def _assemble_locations( - self, - first_lineno: int, - linenos: Iterable[Tuple[int, int, int, Optional[InstrLocation]]], - ) -> bytes: - if not linenos: - return b"" - - locations: List[bytearray] = [] - - iter_in = iter(linenos) - - _, size, lineno, old_location = next(iter_in) - # Infer the line if location is None - old_location = old_location or InstrLocation(lineno, None, None, None) - lineno = first_lineno - - # We track the last set lineno to be able to compute deltas - for _, i_size, new_lineno, location in iter_in: - # Infer the line if location is None - location = location or InstrLocation(new_lineno, None, None, None) - - # Group together instruction with equivalent locations - if old_location.lineno and old_location == location: - size += i_size - continue - - lineno = self._push_locations(locations, size, lineno, old_location) - - size = i_size - old_location = location - - # Pack the line of the last instruction. - self._push_locations(locations, size, lineno, old_location) - - return b"".join(locations) - @staticmethod - def _remove_extended_args( - instructions: MutableSequence[Union[SetLineno, ConcreteInstr]] - ) -> None: + def _remove_extended_args(instructions): # replace jump targets with blocks # HINT : in some cases Python generate useless EXTENDED_ARG opcode # with a value of zero. Such opcodes do not increases the size of the @@ -767,13 +369,13 @@ def _remove_extended_args( continue if extended_arg is not None: - arg = UNSET if instr.name == "NOP" else (extended_arg << 8) + instr.arg + arg = (extended_arg << 8) + instr.arg extended_arg = None instr = ConcreteInstr( instr.name, arg, - location=instr.location, + lineno=instr.lineno, extended_args=nb_extended_args, offset=instr.offset, ) @@ -785,106 +387,25 @@ def _remove_extended_args( if extended_arg is not None: raise ValueError("EXTENDED_ARG at the end of the code") - # Taken and adapted from exception_handling_notes.txt in cpython/Objects - @staticmethod - def _parse_varint(except_table_iterator: Iterator[int]) -> int: - b = next(except_table_iterator) - val = b & 63 - while b & 64: - val <<= 6 - b = next(except_table_iterator) - val |= b & 63 - return val - - def _parse_exception_table( - self, exception_table: bytes - ) -> List[ExceptionTableEntry]: - assert sys.version_info >= (3, 11) - table = [] - iterator = iter(exception_table) - try: - while True: - start = self._parse_varint(iterator) - length = self._parse_varint(iterator) - end = start + length - 1 # Present as inclusive - target = self._parse_varint(iterator) - dl = self._parse_varint(iterator) - depth = dl >> 1 - lasti = bool(dl & 1) - table.append(ExceptionTableEntry(start, end, target, depth, lasti)) - except StopIteration: - return table - - @staticmethod - def _encode_varint(value: int, set_begin_marker: bool=False) -> Iterator[int]: - # Encode value as a varint on 7 bits (MSB should come first) and set - # the begin marker if requested. - temp: List[int] = [] - assert value >= 0 - while value: - temp.append(value & 63 | (64 if temp else 0)) - value >>= 6 - temp = temp or [0] - if set_begin_marker: - temp[-1] |= 128 - return reversed(temp) - - def _assemble_exception_table(self) -> bytes: - table = bytearray() - for entry in self.exception_table or []: - size = entry.stop_offset - entry.start_offset + 1 - depth = (entry.stack_depth << 1) + entry.push_lasti - table.extend(self._encode_varint(entry.start_offset, True)) - table.extend(self._encode_varint(size)) - table.extend(self._encode_varint(entry.target)) - table.extend(self._encode_varint(depth)) - - return bytes(table) - - def compute_stacksize(self, *, check_pre_and_post: bool=True) -> int: + def compute_stacksize(self, *, check_pre_and_post=True): bytecode = self.to_bytecode() cfg = _bytecode.ControlFlowGraph.from_bytecode(bytecode) return cfg.compute_stacksize(check_pre_and_post=check_pre_and_post) - def to_code( - self, - stacksize: Optional[int]=None, - *, - check_pre_and_post: bool=True, - compute_exception_stack_depths: bool=True, - ) -> types.CodeType: - # Prevent reconverting the concrete bytecode to bytecode and cfg to do the - # calculation if we need to do it. - if stacksize is None or ( - sys.version_info >= (3, 11) and compute_exception_stack_depths - ): - cfg = _bytecode.ControlFlowGraph.from_bytecode(self.to_bytecode()) - stacksize = cfg.compute_stacksize( - check_pre_and_post=check_pre_and_post, - compute_exception_stack_depths=compute_exception_stack_depths, - ) - self = cfg.to_bytecode().to_concrete_bytecode( - compute_exception_stack_depths=False - ) - - # Assemble the code string after round tripping to CFG if necessary. + def to_code(self, stacksize=None, *, check_pre_and_post=True): code_str, linenos = self._assemble_code() - lnotab = ( - self._assemble_locations(self.first_lineno, linenos) - if sys.version_info >= (3, 11) - else ( - self._assemble_linestable(self.first_lineno, linenos) - if sys.version_info >= (3, 10) - else self._assemble_lnotab(self.first_lineno, linenos) - ) + self._assemble_linestable(self.first_lineno, linenos) + if sys.version_info >= (3, 10) + else self._assemble_lnotab(self.first_lineno, linenos) ) nlocals = len(self.varnames) + if stacksize is None: + stacksize = self.compute_stacksize(check_pre_and_post=check_pre_and_post) - if sys.version_info >= (3, 11): + if sys.version_info < (3, 8): return types.CodeType( self.argcount, - self.posonlyargcount, self.kwonlyargcount, nlocals, stacksize, @@ -895,10 +416,8 @@ def to_code( tuple(self.varnames), self.filename, self.name, - self.qualname, self.first_lineno, lnotab, - self._assemble_exception_table(), tuple(self.freevars), tuple(self.cellvars), ) @@ -922,172 +441,80 @@ def to_code( tuple(self.cellvars), ) - def to_bytecode( - self, - prune_caches: bool=True, - conserve_exception_block_stackdepth: bool=False, - ) -> _bytecode.Bytecode: - # On 3.11 we generate pseudo-instruction from the exception table + def to_bytecode(self): # Copy instruction and remove extended args if any (in-place) c_instructions = self[:] self._remove_extended_args(c_instructions) - # Find jump targets - jump_targets: Set[int] = set() + # find jump targets + jump_targets = set() offset = 0 - for c_instr in c_instructions: - if isinstance(c_instr, SetLineno): + for instr in c_instructions: + if isinstance(instr, SetLineno): continue - target = c_instr.get_jump_target(offset) + target = instr.get_jump_target(offset) if target is not None: jump_targets.add(target) - offset += (c_instr.size // 2) if OFFSET_AS_INSTRUCTION else c_instr.size - - # On 3.11+ we need to also look at the exception table for jump targets - for ex_entry in self.exception_table: - jump_targets.add(ex_entry.target) - - # Create look up dict to find entries based on either exception handling - # block exit or entry offsets. Several blocks can end on the same instruction - # so we store a list of entry per offset. - ex_start: Dict[int, ExceptionTableEntry] = {} - ex_end: Dict[int, List[ExceptionTableEntry]] = {} - for entry in self.exception_table: - # Ensure we do not have more than one entry with identical starting - # offsets - assert entry.start_offset not in ex_start - ex_start[entry.start_offset] = entry - ex_end.setdefault(entry.stop_offset, []).append(entry) - - # Create labels and instructions - jumps: List[Tuple[int, int]] = [] - instructions: List[Union[Instr, Label, TryBegin, TryEnd, SetLineno]] = [] + offset += (instr.size // 2) if OFFSET_AS_INSTRUCTION else instr.size + + # create labels + jumps = [] + instructions = [] labels = {} - tb_instrs: Dict[ExceptionTableEntry, TryBegin] = {} offset = 0 - # In Python 3.11+ cell and varnames can be shared and are indexed in a single - # array. - # As a consequence, the instruction argument can be either: - # - < len(varnames): the name is shared an we can directly use - # the index to access the name in cellvars - # - > len(varnames): the name is not shared and is offset by the - # number unshared varname. - # Free vars are never shared and correspond to index larger than the - # largest cell var. - # See PyCode_NewWithPosOnlyArgs - if sys.version_info >= (3, 11): - cells_lookup = self.varnames + [ - n for n in self.cellvars if n not in self.varnames - ] - ncells = len(cells_lookup) - else: - ncells = len(self.cellvars) - cells_lookup = self.cellvars + ncells = len(self.cellvars) - for lineno, c_instr in self._normalize_lineno( - c_instructions, self.first_lineno - ): + for lineno, instr in self._normalize_lineno(c_instructions, self.first_lineno): if offset in jump_targets: label = Label() labels[offset] = label instructions.append(label) - # Handle TryBegin pseudo instructions - if offset in ex_start: - entry = ex_start[offset] - tb_instr = TryBegin( - Label(), - entry.push_lasti, - entry.stack_depth if conserve_exception_block_stackdepth else UNSET, - ) - # Per entry store the pseudo instruction associated - tb_instrs[entry] = tb_instr - instructions.append(tb_instr) - - jump_target = c_instr.get_jump_target(offset) - size = c_instr.size - # If an instruction uses extended args, those appear before the instruction - # causing the instruction to appear at offset that accounts for extended - # args. So we first update the offset to account for extended args, then - # record the instruction offset and then add the instruction itself to the - # offset. - offset += (size // 2 - 1) if OFFSET_AS_INSTRUCTION else (size - 2) - current_instr_offset = offset - offset += 1 if OFFSET_AS_INSTRUCTION else 2 - - # on Python 3.11+ remove CACHE opcodes if we are requested to do so. - # We are careful to first advance the offset and check that the CACHE - # is not a jump target. It should never be the case but we double check. - if prune_caches and c_instr.name == "CACHE": - assert jump_target is None - - # We may need to insert a TryEnd after a CACHE so we need to run the - # through the last block. - else: - arg: InstrArg - c_arg = c_instr.arg - # FIXME: better error reporting - if c_instr.opcode in _opcode.hasconst: - arg = self.consts[c_arg] - elif c_instr.opcode in _opcode.haslocal: - arg = self.varnames[c_arg] - elif c_instr.opcode in _opcode.hasname: - if c_instr.name in BITFLAG_INSTRUCTIONS: - arg = (bool(c_arg & 1), self.names[c_arg >> 1]) - elif c_instr.name in BITFLAG2_INSTRUCTIONS: - arg = (bool(c_arg & 1), bool(c_arg & 2), self.names[c_arg >> 2]) - else: - arg = self.names[c_arg] - elif c_instr.opcode in _opcode.hasfree: - if c_arg < ncells: - name = cells_lookup[c_arg] - arg = CellVar(name) - else: - name = self.freevars[c_arg - ncells] - arg = FreeVar(name) - elif c_instr.opcode in _opcode.hascompare: - arg = Compare( - (c_arg >> 4) if sys.version_info >= (3, 12) else c_arg - ) - elif c_instr.opcode in INTRINSIC_1OP: - arg = Intrinsic1Op(c_arg) - elif c_instr.opcode in INTRINSIC_2OP: - arg = Intrinsic2Op(c_arg) - else: - arg = c_arg - - location = c_instr.location or InstrLocation(lineno, None, None, None) + jump_target = instr.get_jump_target(offset) + size = instr.size - if jump_target is not None: - arg = PLACEHOLDER_LABEL - instr_index = len(instructions) - jumps.append((instr_index, jump_target)) + arg = instr.arg + # FIXME: better error reporting + if instr.opcode in _opcode.hasconst: + arg = self.consts[arg] + elif instr.opcode in _opcode.haslocal: + arg = self.varnames[arg] + elif instr.opcode in _opcode.hasname: + arg = self.names[arg] + elif instr.opcode in _opcode.hasfree: + if arg < ncells: + name = self.cellvars[arg] + arg = CellVar(name) + else: + name = self.freevars[arg - ncells] + arg = FreeVar(name) + elif instr.opcode in _opcode.hascompare: + arg = Compare(arg) - instructions.append(Instr(c_instr.name, arg, location=location, offset=c_instr.offset)) + if jump_target is None: + instr = Instr(instr.name, arg, lineno=lineno, offset=instr.offset) + else: + instr_index = len(instructions) + instructions.append(instr) + offset += (size // 2) if OFFSET_AS_INSTRUCTION else size - # We now insert the TryEnd entries - if current_instr_offset in ex_end: - entries = ex_end[current_instr_offset] - for entry in reversed(entries): - instructions.append(TryEnd(tb_instrs[entry])) + if jump_target is not None: + jumps.append((instr_index, jump_target)) - # Replace jump targets with labels + # replace jump targets with labels for index, jump_target in jumps: instr = instructions[index] - assert isinstance(instr, Instr) and instr.arg is PLACEHOLDER_LABEL # FIXME: better error reporting on missing label - instr.arg = labels[jump_target] - - # Set the label for TryBegin - for entry, tb in tb_instrs.items(): - tb.target = labels[entry.target] + label = labels[jump_target] + instructions[index] = Instr(instr.name, label, lineno=instr.lineno, offset=instr.offset) bytecode = _bytecode.Bytecode() bytecode._copy_attr_from(self) nargs = bytecode.argcount + bytecode.kwonlyargcount - nargs += bytecode.posonlyargcount + if sys.version_info > (3, 8): + nargs += bytecode.posonlyargcount if bytecode.flags & inspect.CO_VARARGS: nargs += 1 if bytecode.flags & inspect.CO_VARKEYWORDS: @@ -1100,31 +527,27 @@ def to_bytecode( class _ConvertBytecodeToConcrete: - # XXX document attributes - # : Default number of passes of compute_jumps() before giving up. Refer to - # : assemble_jump_offsets() in compile.c for background. + # Default number of passes of compute_jumps() before giving up. Refer to + # assemble_jump_offsets() in compile.c for background. _compute_jumps_passes = 10 - def __init__(self, code: _bytecode.Bytecode) -> None: + def __init__(self, code): assert isinstance(code, _bytecode.Bytecode) self.bytecode = code # temporary variables - self.instructions: List[ConcreteInstr] = [] - self.jumps: List[Tuple[int, Label, ConcreteInstr]] = [] - self.labels: Dict[Label, int] = {} - self.exception_handling_blocks: Dict[TryBegin, ExceptionTableEntry] = {} - self.required_caches = 0 - self.seen_manual_cache = False + self.instructions = [] + self.jumps = [] + self.labels = {} # used to build ConcreteBytecode() object - self.consts_indices: Dict[Union[bytes, Tuple[type, int]], int] = {} - self.consts_list: List[Any] = [] - self.names: List[str] = [] - self.varnames: List[str] = [] + self.consts_indices = {} + self.consts_list = [] + self.names = [] + self.varnames = [] - def add_const(self, value: Any) -> int: + def add_const(self, value): key = const_key(value) if key in self.consts_indices: return self.consts_indices[key] @@ -1134,7 +557,7 @@ def add_const(self, value: Any) -> int: return index @staticmethod - def add(names: List[str], name: str) -> int: + def add(names, name): try: index = names.index(name) except ValueError: @@ -1142,42 +565,11 @@ def add(names: List[str], name: str) -> int: names.append(name) return index - def concrete_instructions(self) -> None: + def concrete_instructions(self): + ncells = len(self.bytecode.cellvars) lineno = self.bytecode.first_lineno - # Track instruction (index) using cell vars and free vars to be able to update - # the index used once all the names are known. - cell_instrs: List[int] = [] - free_instrs: List[int] = [] for instr in self.bytecode: - # Enforce proper use of CACHE opcode on Python 3.11+ by checking we get the - # number we expect or directly generate the needed ones. - if isinstance(instr, Instr) and instr.name == "CACHE": - if not self.required_caches: - raise RuntimeError("Found a CACHE opcode when none was expected.") - self.seen_manual_cache = True - self.required_caches -= 1 - - elif self.required_caches: - if not self.seen_manual_cache: - # We preserve the location of the instruction requiring the - # presence of cache instructions - self.instructions.extend( - [ - ConcreteInstr( - "CACHE", 0, location=self.instructions[-1].location - ) - for i in range(self.required_caches) - ] - ) - self.required_caches = 0 - self.seen_manual_cache = False - else: - raise RuntimeError( - "Found some manual opcode but less than expected. " - f"Missing {self.required_caches} CACHE opcodes." - ) - if isinstance(instr, Label): self.labels[instr] = len(self.instructions) continue @@ -1186,177 +578,62 @@ def concrete_instructions(self) -> None: lineno = instr.lineno continue - if isinstance(instr, TryBegin): - # We expect the stack depth to have be provided or computed earlier - assert instr.stack_depth is not UNSET - # NOTE here we store the index of the instruction at which the - # exception table entry starts. This is not the final value we want, - # we want the offset in the bytecode but that requires to compute - # the jumps first to resolve any possible extended arg needed in a - # jump. - self.exception_handling_blocks[instr] = ExceptionTableEntry( - len(self.instructions), 0, 0, instr.stack_depth, instr.push_lasti - ) - continue - - # Do not handle TryEnd before we insert possible CACHE opcode - if isinstance(instr, TryEnd): - entry = self.exception_handling_blocks[instr.entry] - # The TryEnd is located after the last opcode in the exception entry - # so we move the offset by one. We choose one so that the end does - # encompass a possible EXTENDED_ARG - entry.stop_offset = len(self.instructions) - 1 - continue - - assert isinstance(instr, Instr) - - if instr.lineno is not UNSET and instr.lineno is not None: - lineno = instr.lineno - elif instr.lineno is UNSET: - instr.lineno = lineno - - arg = instr.arg - is_jump = False - if isinstance(arg, Label): - label = arg - # fake value, real value is set in compute_jumps() - arg = 0 - is_jump = True - elif instr.opcode in _opcode.hasconst: - arg = self.add_const(arg) - elif instr.opcode in _opcode.haslocal: - assert isinstance(arg, str) - arg = self.add(self.varnames, arg) - elif instr.opcode in _opcode.hasname: - if instr.name in BITFLAG_INSTRUCTIONS: - assert ( - isinstance(arg, tuple) - and len(arg) == 2 - and isinstance(arg[0], bool) - and isinstance(arg[1], str) - ), arg - index = self.add(self.names, arg[1]) - arg = int(arg[0]) + (index << 1) - elif instr.name in BITFLAG2_INSTRUCTIONS: - assert ( - isinstance(arg, tuple) - and len(arg) == 3 - and isinstance(arg[0], bool) - and isinstance(arg[1], bool) - and isinstance(arg[2], str) - ), arg - index = self.add(self.names, arg[2]) - arg = int(arg[0]) + 2 * int(arg[1]) + (index << 2) - else: - assert isinstance(arg, str), f"Got {arg}, expected a str" + if isinstance(instr, ConcreteInstr): + instr = instr.copy() + else: + assert isinstance(instr, Instr) + + if instr.lineno is not None: + lineno = instr.lineno + + arg = instr.arg + is_jump = isinstance(arg, Label) + if is_jump: + label = arg + # fake value, real value is set in compute_jumps() + arg = 0 + elif instr.opcode in _opcode.hasconst: + arg = self.add_const(arg) + elif instr.opcode in _opcode.haslocal: + arg = self.add(self.varnames, arg) + elif instr.opcode in _opcode.hasname: arg = self.add(self.names, arg) - elif instr.opcode in _opcode.hasfree: - if isinstance(arg, CellVar): - cell_instrs.append(len(self.instructions)) - arg = self.bytecode.cellvars.index(arg.name) - else: - assert isinstance(arg, FreeVar) - free_instrs.append(len(self.instructions)) - arg = self.bytecode.freevars.index(arg.name) - elif instr.opcode in _opcode.hascompare: - if isinstance(arg, Compare): - # In Python 3.12 the 4 lowest bits are used for caching - # See compare_masks in compile.c - if sys.version_info >= (3, 12): - arg = arg._get_mask() + (arg.value << 4) + elif instr.opcode in _opcode.hasfree: + if isinstance(arg, CellVar): + arg = self.bytecode.cellvars.index(arg.name) else: + assert isinstance(arg, FreeVar) + arg = ncells + self.bytecode.freevars.index(arg.name) + elif instr.opcode in _opcode.hascompare: + if isinstance(arg, Compare): arg = arg.value - elif instr.opcode in INTRINSIC: - if isinstance(arg, (Intrinsic1Op, Intrinsic2Op)): - arg = arg.value - - # The above should have performed all the necessary conversion - assert isinstance(arg, int) - c_instr = ConcreteInstr(instr.name, arg, location=instr.location) - if is_jump: - self.jumps.append((len(self.instructions), label, c_instr)) - - # If the instruction expect some cache - if sys.version_info >= (3, 11): - self.required_caches = c_instr.use_cache_opcodes() - self.seen_manual_cache = False - - self.instructions.append(c_instr) - - # On Python 3.11 varnames and cells can share some names. Wind the shared - # names and update the arg argument of instructions using cell vars. - # We also track by how much to offset free vars which are stored in a - # contiguous array after the cell vars - if sys.version_info >= (3, 11): - # Map naive cell index to shared index - shared_name_indexes: Dict[int, int] = {} - n_shared = 0 - n_unshared = 0 - for i, name in enumerate(self.bytecode.cellvars): - if name in self.varnames: - shared_name_indexes[i] = self.varnames.index(name) - n_shared += 1 - else: - shared_name_indexes[i] = len(self.varnames) + n_unshared - n_unshared += 1 - for index in cell_instrs: - c_instr = self.instructions[index] - c_instr.arg = shared_name_indexes[c_instr.arg] + instr = ConcreteInstr(instr.name, arg, lineno=lineno) + if is_jump: + self.jumps.append((len(self.instructions), label, instr)) - free_offset = len(self.varnames) + len(self.bytecode.cellvars) - n_shared - else: - free_offset = len(self.bytecode.cellvars) - - for index in free_instrs: - c_instr = self.instructions[index] - c_instr.arg += free_offset - - def compute_jumps(self) -> bool: - # For labels we need the offset before the instruction at a given index but for - # exception table entries we need the offset of the instruction which can differ - # in the presence of extended args... - label_offsets = [] - instruction_offsets = [] + self.instructions.append(instr) + + def compute_jumps(self): + offsets = [] offset = 0 - for instr in self.instructions: - label_offsets.append(offset) - # If an instruction uses extended args, those appear before the instruction - # causing the instruction to appear at offset that accounts for extended - # args. - offset += ( - (instr.size // 2 - 1) if OFFSET_AS_INSTRUCTION else (instr.size - 2) - ) - instruction_offsets.append(offset) - offset += 1 if OFFSET_AS_INSTRUCTION else 2 + for index, instr in enumerate(self.instructions): + offsets.append(offset) + offset += instr.size // 2 if OFFSET_AS_INSTRUCTION else instr.size # needed if a label is at the end - label_offsets.append(offset) + offsets.append(offset) - # FIXME may need some extra check to validate jump forward vs jump backward # fix argument of jump instructions: resolve labels modified = False for index, label, instr in self.jumps: target_index = self.labels[label] - target_offset = label_offsets[target_index] + target_offset = offsets[target_index] - # FIXME use opcode - # Under 3.12+, FOR_ITER, SEND jump is increased by 1 implicitely - # to skip over END_FOR, END_SEND see Python/instrumentation.c - if sys.version_info >= (3, 12) and instr.name in ("FOR_ITER", "SEND"): - target_offset -= 1 - - if instr.is_forward_rel_jump(): - instr_offset = label_offsets[index] + if instr.opcode in _opcode.hasjrel: + instr_offset = offsets[index] target_offset -= instr_offset + ( instr.size // 2 if OFFSET_AS_INSTRUCTION else instr.size ) - elif instr.is_backward_rel_jump(): - instr_offset = label_offsets[index] - target_offset = ( - instr_offset - +(instr.size // 2 if OFFSET_AS_INSTRUCTION else instr.size) - -target_offset - ) old_size = instr.size # FIXME: better error report if target_offset is negative @@ -1364,37 +641,9 @@ def compute_jumps(self) -> bool: if instr.size != old_size: modified = True - # If a jump required an extended arg hence invalidating the calculation - # we return early before filling the exception table entries - if modified: - return modified - - # Resolve labels for exception handling entries - for tb, entry in self.exception_handling_blocks.items(): - # Set the offset for the start and end offset from the instruction - # index stored when assembling the concrete instructions. - entry.start_offset = instruction_offsets[entry.start_offset] - entry.stop_offset = instruction_offsets[entry.stop_offset] - - # Set the offset to the target instruction - lb = tb.target - assert isinstance(lb, Label) - target_index = self.labels[lb] - target_offset = label_offsets[target_index] - entry.target = target_offset - - return False - - def to_concrete_bytecode( - self, - compute_jumps_passes: Optional[int]=None, - compute_exception_stack_depths: bool=True, - ) -> ConcreteBytecode: - if sys.version_info >= (3, 11) and compute_exception_stack_depths: - cfg = _bytecode.ControlFlowGraph.from_bytecode(self.bytecode) - cfg.compute_stacksize(compute_exception_stack_depths=True) - self.bytecode = cfg.to_bytecode() + return modified + def to_concrete_bytecode(self, compute_jumps_passes=None): if compute_jumps_passes is None: compute_jumps_passes = self._compute_jumps_passes @@ -1405,22 +654,20 @@ def to_concrete_bytecode( self.varnames.extend(self.bytecode.argnames) self.concrete_instructions() - for _ in range(0, compute_jumps_passes): + for pas in range(0, compute_jumps_passes): modified = self.compute_jumps() if not modified: break else: raise RuntimeError( - "compute_jumps() failed to converge after" - " %d passes" % (compute_jumps_passes) + "compute_jumps() failed to converge after" " %d passes" % (pas + 1) ) concrete = ConcreteBytecode( self.instructions, - consts=tuple(self.consts_list), - names=tuple(self.names), + consts=self.consts_list.copy(), + names=self.names, varnames=self.varnames, - exception_table=list(self.exception_handling_blocks.values()), ) concrete._copy_attr_from(self.bytecode) return concrete diff --git a/_pydevd_frame_eval/vendored/bytecode/flags.py b/_pydevd_frame_eval/vendored/bytecode/flags.py index d5ba3b580..b0c5239cd 100644 --- a/_pydevd_frame_eval/vendored/bytecode/flags.py +++ b/_pydevd_frame_eval/vendored/bytecode/flags.py @@ -1,9 +1,6 @@ -import opcode +# alias to keep the 'bytecode' variable free import sys from enum import IntFlag -from typing import Optional, Union - -# alias to keep the 'bytecode' variable free from _pydevd_frame_eval.vendored import bytecode as _bytecode @@ -15,39 +12,35 @@ class CompilerFlags(IntFlag): """ - OPTIMIZED = 0x00001 - NEWLOCALS = 0x00002 - VARARGS = 0x00004 - VARKEYWORDS = 0x00008 - NESTED = 0x00010 - GENERATOR = 0x00020 - NOFREE = 0x00040 + OPTIMIZED = 0x00001 # noqa + NEWLOCALS = 0x00002 # noqa + VARARGS = 0x00004 # noqa + VARKEYWORDS = 0x00008 # noqa + NESTED = 0x00010 # noqa + GENERATOR = 0x00020 # noqa + NOFREE = 0x00040 # noqa # New in Python 3.5 # Used for coroutines defined using async def ie native coroutine - COROUTINE = 0x00080 + COROUTINE = 0x00080 # noqa # Used for coroutines defined as a generator and then decorated using # types.coroutine - ITERABLE_COROUTINE = 0x00100 + ITERABLE_COROUTINE = 0x00100 # noqa # New in Python 3.6 # Generator defined in an async def function - ASYNC_GENERATOR = 0x00200 + ASYNC_GENERATOR = 0x00200 # noqa # __future__ flags # future flags changed in Python 3.9 if sys.version_info < (3, 9): - FUTURE_GENERATOR_STOP = 0x80000 - FUTURE_ANNOTATIONS = 0x100000 + FUTURE_GENERATOR_STOP = 0x80000 # noqa + if sys.version_info > (3, 6): + FUTURE_ANNOTATIONS = 0x100000 else: - FUTURE_GENERATOR_STOP = 0x800000 + FUTURE_GENERATOR_STOP = 0x800000 # noqa FUTURE_ANNOTATIONS = 0x1000000 -def infer_flags( - bytecode: Union[ - "_bytecode.Bytecode", "_bytecode.ConcreteBytecode", "_bytecode.ControlFlowGraph" - ], - is_async: Optional[bool] = None, -): +def infer_flags(bytecode, is_async=None): """Infer the proper flags for a bytecode based on the instructions. Because the bytecode does not have enough context to guess if a function @@ -76,22 +69,14 @@ def infer_flags( raise ValueError(msg % bytecode) instructions = ( - bytecode._get_instructions() + bytecode.get_instructions() if isinstance(bytecode, _bytecode.ControlFlowGraph) else bytecode ) instr_names = { i.name for i in instructions - if not isinstance( - i, - ( - _bytecode.SetLineno, - _bytecode.Label, - _bytecode.TryBegin, - _bytecode.TryEnd, - ), - ) + if not isinstance(i, (_bytecode.SetLineno, _bytecode.Label)) } # Identify optimized code @@ -99,7 +84,16 @@ def infer_flags( flags |= CompilerFlags.OPTIMIZED # Check for free variables - if not (instr_names & {opcode.opname[i] for i in opcode.hasfree}): + if not ( + instr_names + & { + "LOAD_CLOSURE", + "LOAD_DEREF", + "STORE_DEREF", + "DELETE_DEREF", + "LOAD_CLASSDEREF", + } + ): flags |= CompilerFlags.NOFREE # Copy flags for which we cannot infer the right value @@ -120,12 +114,12 @@ def infer_flags( "BEFORE_ASYNC_WITH", "SETUP_ASYNC_WITH", "END_ASYNC_FOR", - "ASYNC_GEN_WRAP", # New in 3.11 } # If performing inference or forcing an async behavior, first inspect # the flags since this is the only way to identify iterable coroutines if is_async in (None, True): + if bytecode.flags & CompilerFlags.COROUTINE: if sure_generator: flags |= CompilerFlags.ASYNC_GENERATOR diff --git a/_pydevd_frame_eval/vendored/bytecode/instr.py b/_pydevd_frame_eval/vendored/bytecode/instr.py index 49362ebc7..9247a5495 100644 --- a/_pydevd_frame_eval/vendored/bytecode/instr.py +++ b/_pydevd_frame_eval/vendored/bytecode/instr.py @@ -1,46 +1,12 @@ -import dis import enum +import dis import opcode as _opcode import sys -from abc import abstractmethod -from dataclasses import dataclass from marshal import dumps as _dumps -from typing import Any, Callable, Dict, Generic, Optional, Tuple, TypeVar, Union - -try: - from typing import TypeGuard -except ImportError: - from typing_extensions import TypeGuard # type: ignore from _pydevd_frame_eval.vendored import bytecode as _bytecode -# --- Instruction argument tools and - -MIN_INSTRUMENTED_OPCODE = getattr(_opcode, "MIN_INSTRUMENTED_OPCODE", 256) - -# Instructions relying on a bit to modify its behavior. -# The lowest bit is used to encode custom behavior. -BITFLAG_INSTRUCTIONS = ( - ("LOAD_GLOBAL", "LOAD_ATTR") - if sys.version_info >= (3, 12) - else ("LOAD_GLOBAL",) - if sys.version_info >= (3, 11) - else () -) - -BITFLAG2_INSTRUCTIONS = ("LOAD_SUPER_ATTR",) if sys.version_info >= (3, 12) else () - -# Intrinsic related opcodes -INTRINSIC_1OP = ( - (_opcode.opmap["CALL_INTRINSIC_1"],) if sys.version_info >= (3, 12) else () -) -INTRINSIC_2OP = ( - (_opcode.opmap["CALL_INTRINSIC_2"],) if sys.version_info >= (3, 12) else () -) -INTRINSIC = INTRINSIC_1OP + INTRINSIC_2OP - -# Used for COMPARE_OP opcode argument @enum.unique class Compare(enum.IntEnum): LT = 0 @@ -49,152 +15,17 @@ class Compare(enum.IntEnum): NE = 3 GT = 4 GE = 5 - if sys.version_info < (3, 9): - IN = 6 - NOT_IN = 7 - IS = 8 - IS_NOT = 9 - EXC_MATCH = 10 - - if sys.version_info >= (3, 12): - - def _get_mask(self): - if self == Compare.EQ: - return 8 - elif self == Compare.NE: - return 1 + 2 + 4 - elif self == Compare.LT: - return 2 - elif self == Compare.LE: - return 2 + 8 - elif self == Compare.GT: - return 4 - elif self == Compare.GE: - return 4 + 8 - - -# Used for BINARY_OP under Python 3.11+ -@enum.unique -class BinaryOp(enum.IntEnum): - ADD = 0 - AND = 1 - FLOOR_DIVIDE = 2 - LSHIFT = 3 - MATRIX_MULTIPLY = 4 - MULTIPLY = 5 - REMAINDER = 6 - OR = 7 - POWER = 8 - RSHIFT = 9 - SUBTRACT = 10 - TRUE_DIVIDE = 11 - XOR = 12 - INPLACE_ADD = 13 - INPLACE_AND = 14 - INPLACE_FLOOR_DIVIDE = 15 - INPLACE_LSHIFT = 16 - INPLACE_MATRIX_MULTIPLY = 17 - INPLACE_MULTIPLY = 18 - INPLACE_REMAINDER = 19 - INPLACE_OR = 20 - INPLACE_POWER = 21 - INPLACE_RSHIFT = 22 - INPLACE_SUBTRACT = 23 - INPLACE_TRUE_DIVIDE = 24 - INPLACE_XOR = 25 + IN = 6 + NOT_IN = 7 + IS = 8 + IS_NOT = 9 + EXC_MATCH = 10 -@enum.unique -class Intrinsic1Op(enum.IntEnum): - INTRINSIC_1_INVALID = 0 - INTRINSIC_PRINT = 1 - INTRINSIC_IMPORT_STAR = 2 - INTRINSIC_STOPITERATION_ERROR = 3 - INTRINSIC_ASYNC_GEN_WRAP = 4 - INTRINSIC_UNARY_POSITIVE = 5 - INTRINSIC_LIST_TO_TUPLE = 6 - INTRINSIC_TYPEVAR = 7 - INTRINSIC_PARAMSPEC = 8 - INTRINSIC_TYPEVARTUPLE = 9 - INTRINSIC_SUBSCRIPT_GENERIC = 10 - INTRINSIC_TYPEALIAS = 11 +UNSET = object() -@enum.unique -class Intrinsic2Op(enum.IntEnum): - INTRINSIC_2_INVALID = 0 - INTRINSIC_PREP_RERAISE_STAR = 1 - INTRINSIC_TYPEVAR_WITH_BOUND = 2 - INTRINSIC_TYPEVAR_WITH_CONSTRAINTS = 3 - INTRINSIC_SET_FUNCTION_TYPE_PARAMS = 4 - - -# This make type checking happy but means it won't catch attempt to manipulate an unset -# statically. We would need guard on object attribute narrowed down through methods -class _UNSET(int): - instance = None - - def __new__(cls): - if cls.instance is None: - cls.instance = super().__new__(cls) - return cls.instance - - def __eq__(self, other) -> bool: - return self is other - - -for op in [ - "__abs__", - "__add__", - "__and__", - "__bool__", - "__ceil__", - "__divmod__", - "__float__", - "__floor__", - "__floordiv__", - "__ge__", - "__gt__", - "__hash__", - "__index__", - "__int__", - "__invert__", - "__le__", - "__lshift__", - "__lt__", - "__mod__", - "__mul__", - "__ne__", - "__neg__", - "__or__", - "__pos__", - "__pow__", - "__radd__", - "__rand__", - "__rdivmod__", - "__rfloordiv__", - "__rlshift__", - "__rmod__", - "__rmul__", - "__ror__", - "__round__", - "__rpow__", - "__rrshift__", - "__rshift__", - "__rsub__", - "__rtruediv__", - "__rxor__", - "__sub__", - "__truediv__", - "__trunc__", - "__xor__", -]: - setattr(_UNSET, op, lambda *args: NotImplemented) - -UNSET = _UNSET() - - -def const_key(obj: Any) -> Union[bytes, Tuple[type, int]]: +def const_key(obj): try: return _dumps(obj) except ValueError: @@ -203,30 +34,82 @@ def const_key(obj: Any) -> Union[bytes, Tuple[type, int]]: return (type(obj), id(obj)) -class Label: - __slots__ = () +def _pushes_back(opname): + if opname in ["CALL_FINALLY"]: + # CALL_FINALLY pushes the address of the "finally" block instead of a + # value, hence we don't treat it as pushing back op + return False + return ( + opname.startswith("UNARY_") + or opname.startswith("GET_") + # BUILD_XXX_UNPACK have been removed in 3.9 + or opname.startswith("BINARY_") + or opname.startswith("INPLACE_") + or opname.startswith("BUILD_") + or opname.startswith("CALL_") + ) or opname in ( + "LIST_TO_TUPLE", + "LIST_EXTEND", + "SET_UPDATE", + "DICT_UPDATE", + "DICT_MERGE", + "IS_OP", + "CONTAINS_OP", + "FORMAT_VALUE", + "MAKE_FUNCTION", + "IMPORT_NAME", + # technically, these three do not push back, but leave the container + # object on TOS + "SET_ADD", + "LIST_APPEND", + "MAP_ADD", + "LOAD_ATTR", + ) + + +def _check_lineno(lineno): + if not isinstance(lineno, int): + raise TypeError("lineno must be an int") + if lineno < 1: + raise ValueError("invalid lineno") -# : Placeholder label temporarily used when performing some conversions -# : concrete -> bytecode -PLACEHOLDER_LABEL = Label() +class SetLineno: + __slots__ = ("_lineno",) + + def __init__(self, lineno): + _check_lineno(lineno) + self._lineno = lineno + + @property + def lineno(self): + return self._lineno + + def __eq__(self, other): + if not isinstance(other, SetLineno): + return False + return self._lineno == other._lineno + + +class Label: + __slots__ = () class _Variable: __slots__ = ("name",) - def __init__(self, name: str) -> None: - self.name: str = name + def __init__(self, name): + self.name = name - def __eq__(self, other: Any) -> bool: - if type(self) is not type(other): + def __eq__(self, other): + if type(self) != type(other): return False return self.name == other.name - def __str__(self) -> str: + def __str__(self): return self.name - def __repr__(self) -> str: + def __repr__(self): return "<%s %r>" % (self.__class__.__name__, self.name) @@ -238,7 +121,7 @@ class FreeVar(_Variable): __slots__ = () -def _check_arg_int(arg: Any, name: str) -> TypeGuard[int]: +def _check_arg_int(name, arg): if not isinstance(arg, int): raise TypeError( "operation %s argument must be an int, " @@ -250,325 +133,143 @@ def _check_arg_int(arg: Any, name: str) -> TypeGuard[int]: "operation %s argument must be in " "the range 0..2,147,483,647" % name ) - return True - - -if sys.version_info >= (3, 12): - - def opcode_has_argument(opcode: int) -> bool: - return opcode in dis.hasarg - -else: - - def opcode_has_argument(opcode: int) -> bool: - return opcode >= dis.HAVE_ARGUMENT - -# --- Instruction stack effect impact - -# We split the stack effect between the manipulations done on the stack before -# executing the instruction (fetching the elements that are going to be used) -# and what is pushed back on the stack after the execution is complete. - -# Stack effects that do not depend on the argument of the instruction -STATIC_STACK_EFFECTS: Dict[str, Tuple[int, int]] = { - "ROT_TWO": (-2, 2), - "ROT_THREE": (-3, 3), - "ROT_FOUR": (-4, 4), - "DUP_TOP": (-1, 2), - "DUP_TOP_TWO": (-2, 4), - "GET_LEN": (-1, 2), - "GET_ITER": (-1, 1), - "GET_YIELD_FROM_ITER": (-1, 1), - "GET_AWAITABLE": (-1, 1), - "GET_AITER": (-1, 1), - "GET_ANEXT": (-1, 2), - "LIST_TO_TUPLE": (-1, 1), - "LIST_EXTEND": (-2, 1), - "SET_UPDATE": (-2, 1), - "DICT_UPDATE": (-2, 1), - "DICT_MERGE": (-2, 1), - "COMPARE_OP": (-2, 1), - "IS_OP": (-2, 1), - "CONTAINS_OP": (-2, 1), - "IMPORT_NAME": (-2, 1), - "ASYNC_GEN_WRAP": (-1, 1), - "PUSH_EXC_INFO": (-1, 2), - # Pop TOS and push TOS.__aexit__ and result of TOS.__aenter__() - "BEFORE_ASYNC_WITH": (-1, 2), - # Replace TOS based on TOS and TOS1 - "IMPORT_FROM": (-1, 2), - "COPY_DICT_WITHOUT_KEYS": (-2, 2), - # Call a function at position 7 (4 3.11+) on the stack and push the return value - "WITH_EXCEPT_START": (-4, 5) if sys.version_info >= (3, 11) else (-7, 8), - # Starting with Python 3.11 MATCH_CLASS does not push a boolean anymore - "MATCH_CLASS": (-3, 1 if sys.version_info >= (3, 11) else 2), - "MATCH_MAPPING": (-1, 2), - "MATCH_SEQUENCE": (-1, 2), - "MATCH_KEYS": (-2, 3 if sys.version_info >= (3, 11) else 4), - "CHECK_EXC_MATCH": (-2, 2), # (TOS1, TOS) -> (TOS1, bool) - "CHECK_EG_MATCH": (-2, 2), # (TOS, TOS1) -> non-matched, matched or TOS1, None) - "PREP_RERAISE_STAR": (-2, 1), # (TOS1, TOS) -> new exception group) - ** {k: (-1, 1) for k in (o for o in _opcode.opmap if (o.startswith("UNARY_")))}, - **{ - k: (-2, 1) - for k in ( - o - for o in _opcode.opmap - if (o.startswith("BINARY_") or o.startswith("INPLACE_")) - ) - }, - # Python 3.12 changes not covered by dis.stack_effect - "BINARY_SLICE": (-3, 1), - # "STORE_SLICE" handled by dis.stack_effect - "LOAD_FROM_DICT_OR_GLOBALS": (-1, 1), - "LOAD_FROM_DICT_OR_DEREF": (-1, 1), - "LOAD_INTRISIC_1": (-1, 1), - "LOAD_INTRISIC_2": (-2, 1), -} - -DYNAMIC_STACK_EFFECTS: Dict[ - str, Callable[[int, Any, Optional[bool]], Tuple[int, int]] -] = { - # PRECALL pops all arguments (as per its stack effect) and leaves - # the callable and either self or NULL - # CALL pops the 2 above items and push the return - # (when PRECALL does not exist it pops more as encoded by the effect) - "CALL": lambda effect, arg, jump: ( - -2 - arg if sys.version_info >= (3, 12) else -2, - 1, - ), - # 3.12 changed the behavior of LOAD_ATTR - "LOAD_ATTR": lambda effect, arg, jump: (-1, 1 + effect), - "LOAD_SUPER_ATTR": lambda effect, arg, jump: (-3, 3 + effect), - "SWAP": lambda effect, arg, jump: (-arg, arg), - "COPY": lambda effect, arg, jump: (-arg, arg + effect), - "ROT_N": lambda effect, arg, jump: (-arg, arg), - "SET_ADD": lambda effect, arg, jump: (-arg, arg - 1), - "LIST_APPEND": lambda effect, arg, jump: (-arg, arg - 1), - "MAP_ADD": lambda effect, arg, jump: (-arg, arg - 2), - "FORMAT_VALUE": lambda effect, arg, jump: (effect - 1, 1), - # FOR_ITER needs TOS to be an iterator, hence a prerequisite of 1 on the stack - "FOR_ITER": lambda effect, arg, jump: (effect, 0) if jump else (-1, 2), - **{ - # Instr(UNPACK_* , n) pops 1 and pushes n - k: lambda effect, arg, jump: (-1, effect + 1) - for k in ( - "UNPACK_SEQUENCE", - "UNPACK_EX", - ) - }, - **{ - k: lambda effect, arg, jump: (effect - 1, 1) - for k in ( - "MAKE_FUNCTION", - "CALL_FUNCTION", - "CALL_FUNCTION_EX", - "CALL_FUNCTION_KW", - "CALL_METHOD", - *(o for o in _opcode.opmap if o.startswith("BUILD_")), - ) - }, -} - -# --- Instruction location - -def _check_location( - location: Optional[int], location_name: str, min_value: int -) -> None: - if location is None: - return - if not isinstance(location, int): - raise TypeError(f"{location_name} must be an int, got {type(location)}") - if location < min_value: - raise ValueError( - f"invalid {location_name}, expected >= {min_value}, got {location}" +if sys.version_info < (3, 8): + _stack_effects = { + # NOTE: the entries are all 2-tuples. Entry[0/False] is non-taken jumps. + # Entry[1/True] is for taken jumps. + # opcodes not in dis.stack_effect + _opcode.opmap["EXTENDED_ARG"]: (0, 0), + _opcode.opmap["NOP"]: (0, 0), + # Jump taken/not-taken are different: + _opcode.opmap["JUMP_IF_TRUE_OR_POP"]: (-1, 0), + _opcode.opmap["JUMP_IF_FALSE_OR_POP"]: (-1, 0), + _opcode.opmap["FOR_ITER"]: (1, -1), + _opcode.opmap["SETUP_WITH"]: (1, 6), + _opcode.opmap["SETUP_ASYNC_WITH"]: (0, 5), + _opcode.opmap["SETUP_EXCEPT"]: (0, 6), # as of 3.7, below for <=3.6 + _opcode.opmap["SETUP_FINALLY"]: (0, 6), # as of 3.7, below for <=3.6 + } + + # More stack effect values that are unique to the version of Python. + if sys.version_info < (3, 7): + _stack_effects.update( + { + _opcode.opmap["SETUP_WITH"]: (7, 7), + _opcode.opmap["SETUP_EXCEPT"]: (6, 9), + _opcode.opmap["SETUP_FINALLY"]: (6, 9), + } ) -@dataclass(frozen=True) -class InstrLocation: - """Location information for an instruction.""" - - # : Lineno at which the instruction corresponds. - # : Optional so that a location of None in an instruction encode an unset value. - lineno: Optional[int] - - # : End lineno at which the instruction corresponds (Python 3.11+ only) - end_lineno: Optional[int] - - # : Column offset at which the instruction corresponds (Python 3.11+ only) - col_offset: Optional[int] - - # : End column offset at which the instruction corresponds (Python 3.11+ only) - end_col_offset: Optional[int] - - __slots__ = ["lineno", "end_lineno", "col_offset", "end_col_offset"] - - def __init__( - self, - lineno: Optional[int], - end_lineno: Optional[int], - col_offset: Optional[int], - end_col_offset: Optional[int], - ) -> None: - # Needed because we want the class to be frozen - object.__setattr__(self, "lineno", lineno) - object.__setattr__(self, "end_lineno", end_lineno) - object.__setattr__(self, "col_offset", col_offset) - object.__setattr__(self, "end_col_offset", end_col_offset) - # In Python 3.11 0 is a valid lineno for some instructions (RESUME for example) - _check_location(lineno, "lineno", 0 if sys.version_info >= (3, 11) else 1) - _check_location(end_lineno, "end_lineno", 1) - _check_location(col_offset, "col_offset", 0) - _check_location(end_col_offset, "end_col_offset", 0) - if end_lineno: - if lineno is None: - raise ValueError("End lineno specified with no lineno.") - elif lineno > end_lineno: - raise ValueError( - f"End lineno {end_lineno} cannot be smaller than lineno {lineno}." - ) - - if col_offset is not None or end_col_offset is not None: - if lineno is None or end_lineno is None: - raise ValueError( - "Column offsets were specified but lineno information are " - f"incomplete. Lineno: {lineno}, end lineno: {end_lineno}." - ) - if end_col_offset is not None: - if col_offset is None: - raise ValueError( - "End column offset specified with no column offset." - ) - # Column offset must be increasing inside a signle line but - # have no relations between different lines. - elif lineno == end_lineno and col_offset > end_col_offset: - raise ValueError( - f"End column offset {end_col_offset} cannot be smaller than " - f"column offset: {col_offset}." - ) - else: - raise ValueError( - "No end column offset was specified but a column offset was given." - ) - - @classmethod - def from_positions(cls, position: "dis.Positions") -> "InstrLocation": # type: ignore - return InstrLocation( - position.lineno, - position.end_lineno, - position.col_offset, - position.end_col_offset, - ) - - -class SetLineno: - __slots__ = ("_lineno",) - - def __init__(self, lineno: int) -> None: - # In Python 3.11 0 is a valid lineno for some instructions (RESUME for example) - _check_location(lineno, "lineno", 0 if sys.version_info >= (3, 11) else 1) - self._lineno: int = lineno - - @property - def lineno(self) -> int: - return self._lineno - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, SetLineno): - return False - return self._lineno == other._lineno +class Instr: + """Abstract instruction.""" -# --- Pseudo instructions used to represent exception handling (3.11+) + __slots__ = ("_name", "_opcode", "_arg", "_lineno", "offset") + def __init__(self, name, arg=UNSET, *, lineno=None, offset=None): + self._set(name, arg, lineno) + self.offset = offset -class TryBegin: - __slots__ = ("target", "push_lasti", "stack_depth") + def _check_arg(self, name, opcode, arg): + if name == "EXTENDED_ARG": + raise ValueError( + "only concrete instruction can contain EXTENDED_ARG, " + "highlevel instruction can represent arbitrary argument without it" + ) - def __init__( - self, - target: Union[Label, "_bytecode.BasicBlock"], - push_lasti: bool, - stack_depth: Union[int, _UNSET]=UNSET, - ) -> None: - self.target: Union[Label, "_bytecode.BasicBlock"] = target - self.push_lasti: bool = push_lasti - self.stack_depth: Union[int, _UNSET] = stack_depth + if opcode >= _opcode.HAVE_ARGUMENT: + if arg is UNSET: + raise ValueError("operation %s requires an argument" % name) + else: + if arg is not UNSET: + raise ValueError("operation %s has no argument" % name) - def copy(self) -> "TryBegin": - return TryBegin(self.target, self.push_lasti, self.stack_depth) + if self._has_jump(opcode): + if not isinstance(arg, (Label, _bytecode.BasicBlock)): + raise TypeError( + "operation %s argument type must be " + "Label or BasicBlock, got %s" % (name, type(arg).__name__) + ) + elif opcode in _opcode.hasfree: + if not isinstance(arg, (CellVar, FreeVar)): + raise TypeError( + "operation %s argument must be CellVar " + "or FreeVar, got %s" % (name, type(arg).__name__) + ) -class TryEnd: - __slots__ = "entry" + elif opcode in _opcode.haslocal or opcode in _opcode.hasname: + if not isinstance(arg, str): + raise TypeError( + "operation %s argument must be a str, " + "got %s" % (name, type(arg).__name__) + ) - def __init__(self, entry: TryBegin) -> None: - self.entry: TryBegin = entry + elif opcode in _opcode.hasconst: + if isinstance(arg, Label): + raise ValueError( + "label argument cannot be used " "in %s operation" % name + ) + if isinstance(arg, _bytecode.BasicBlock): + raise ValueError( + "block argument cannot be used " "in %s operation" % name + ) - def copy(self) -> "TryEnd": - return TryEnd(self.entry) + elif opcode in _opcode.hascompare: + if not isinstance(arg, Compare): + raise TypeError( + "operation %s argument type must be " + "Compare, got %s" % (name, type(arg).__name__) + ) + elif opcode >= _opcode.HAVE_ARGUMENT: + _check_arg_int(name, arg) -T = TypeVar("T", bound="BaseInstr") -A = TypeVar("A", bound=object) + def _set(self, name, arg, lineno): + if not isinstance(name, str): + raise TypeError("operation name must be a str") + try: + opcode = _opcode.opmap[name] + except KeyError: + raise ValueError("invalid operation name") + # check lineno + if lineno is not None: + _check_lineno(lineno) -class BaseInstr(Generic[A]): - """Abstract instruction.""" + self._check_arg(name, opcode, arg) - __slots__ = ("_name", "_opcode", "_arg", "_location", "_offset") - - # Work around an issue with the default value of arg - def __init__( - self, - name: str, - arg: A=UNSET, # type: ignore - * , - lineno: Union[int, None, _UNSET]=UNSET, - location: Optional[InstrLocation]=None, - offset=None, - ) -> None: - self._set(name, arg) - if location: - self._location = location - elif lineno is UNSET: - self._location = None - else: - self._location = InstrLocation(lineno, None, None, None) - self._offset = offset + self._name = name + self._opcode = opcode + self._arg = arg + self._lineno = lineno - # Work around an issue with the default value of arg - def set(self, name: str, arg: A=UNSET) -> None: # type: ignore + def set(self, name, arg=UNSET): """Modify the instruction in-place. Replace name and arg attributes. Don't modify lineno. - """ - self._set(name, arg) + self._set(name, arg, self._lineno) - def require_arg(self) -> bool: + def require_arg(self): """Does the instruction require an argument?""" - return opcode_has_argument(self._opcode) + return self._opcode >= _opcode.HAVE_ARGUMENT @property - def name(self) -> str: + def name(self): return self._name - @property - def offset(self) -> int: - return self._offset - @name.setter - def name(self, name: str) -> None: - self._set(name, self._arg) + def name(self, name): + self._set(name, self._arg, self._lineno) @property - def opcode(self) -> int: + def opcode(self): return self._opcode @opcode.setter - def opcode(self, op: int) -> None: + def opcode(self, op): if not isinstance(op, int): raise TypeError("operator code must be an int") if 0 <= op <= 255: @@ -579,62 +280,27 @@ def opcode(self, op: int) -> None: if not valid: raise ValueError("invalid operator code") - self._set(name, self._arg) + self._set(name, self._arg, self._lineno) @property - def arg(self) -> A: + def arg(self): return self._arg @arg.setter - def arg(self, arg: A): - self._set(self._name, arg) + def arg(self, arg): + self._set(self._name, arg, self._lineno) @property - def lineno(self) -> Union[int, _UNSET, None]: - return self._location.lineno if self._location is not None else UNSET + def lineno(self): + return self._lineno @lineno.setter - def lineno(self, lineno: Union[int, _UNSET, None]) -> None: - loc = self._location - if loc and ( - loc.end_lineno is not None - or loc.col_offset is not None - or loc.end_col_offset is not None - ): - raise RuntimeError( - "The lineno of an instruction with detailed location information " - "cannot be set." - ) - - if lineno is UNSET: - self._location = None - else: - self._location = InstrLocation(lineno, None, None, None) - - @property - def location(self) -> Optional[InstrLocation]: - return self._location - - @location.setter - def location(self, location: Optional[InstrLocation]) -> None: - if location and not isinstance(location, InstrLocation): - raise TypeError( - "The instr location must be an instance of InstrLocation or None." - ) - self._location = location + def lineno(self, lineno): + self._set(self._name, self._arg, lineno) - def stack_effect(self, jump: Optional[bool]=None) -> int: - if not self.require_arg(): + def stack_effect(self, jump=None): + if self._opcode < _opcode.HAVE_ARGUMENT: arg = None - # 3.11 where LOAD_GLOBAL arg encode whether or we push a null - # 3.12 does the same for LOAD_ATTR - elif self.name in BITFLAG_INSTRUCTIONS and isinstance(self._arg, tuple): - assert len(self._arg) == 2 - arg = self._arg[0] - # 3.12 does a similar trick for LOAD_SUPER_ATTR - elif self.name in BITFLAG2_INSTRUCTIONS and isinstance(self._arg, tuple): - assert len(self._arg) == 3 - arg = self._arg[0] elif not isinstance(self._arg, int) or self._opcode in _opcode.hasconst: # Argument is either a non-integer or an integer constant, # not oparg. @@ -642,62 +308,83 @@ def stack_effect(self, jump: Optional[bool]=None) -> int: else: arg = self._arg - return dis.stack_effect(self._opcode, arg, jump=jump) + if sys.version_info < (3, 8): + effect = _stack_effects.get(self._opcode, None) + if effect is not None: + return max(effect) if jump is None else effect[jump] + return dis.stack_effect(self._opcode, arg) + else: + return dis.stack_effect(self._opcode, arg, jump=jump) - def pre_and_post_stack_effect(self, jump: Optional[bool]=None) -> Tuple[int, int]: - # Allow to check that execution will not cause a stack underflow + def pre_and_post_stack_effect(self, jump=None): _effect = self.stack_effect(jump=jump) - n = self.name - if n in STATIC_STACK_EFFECTS: - return STATIC_STACK_EFFECTS[n] - elif n in DYNAMIC_STACK_EFFECTS: - return DYNAMIC_STACK_EFFECTS[n](_effect, self.arg, jump) + # To compute pre size and post size to avoid segfault cause by not enough + # stack element + _opname = _opcode.opname[self._opcode] + if _opname.startswith("DUP_TOP"): + return _effect * -1, _effect * 2 + if _pushes_back(_opname): + # if the op pushes value back to the stack, then the stack effect given + # by dis.stack_effect actually equals pre + post effect, therefore we need + # -1 from the stack effect as a pre condition + return _effect - 1, 1 + if _opname.startswith("UNPACK_"): + # Instr(UNPACK_* , n) pops 1 and pushes n + # _effect = n - 1 + # hence we return -1, _effect + 1 + return -1, _effect + 1 + if _opname == "FOR_ITER" and not jump: + # Since FOR_ITER needs TOS to be an iterator, which basically means + # a prerequisite of 1 on the stack + return -1, 2 + if _opname == "ROT_N": + return (-self._arg, self._arg) + return {"ROT_TWO": (-2, 2), "ROT_THREE": (-3, 3), "ROT_FOUR": (-4, 4)}.get( + _opname, (_effect, 0) + ) + + def copy(self): + return self.__class__(self._name, self._arg, lineno=self._lineno, offset=self.offset) + + def __repr__(self): + if self._arg is not UNSET: + return "<%s arg=%r lineno=%s>" % (self._name, self._arg, self._lineno) else: - # For instruction with no special value we simply consider the effect apply - # before execution - return (_effect, 0) + return "<%s lineno=%s>" % (self._name, self._lineno) - def copy(self: T) -> T: - return self.__class__(self._name, self._arg, location=self._location, offset=self._offset) + def _cmp_key(self, labels=None): + arg = self._arg + if self._opcode in _opcode.hasconst: + arg = const_key(arg) + elif isinstance(arg, Label) and labels is not None: + arg = labels[arg] + return (self._lineno, self._name, arg) - def has_jump(self) -> bool: + def __eq__(self, other): + if type(self) != type(other): + return False + return self._cmp_key() == other._cmp_key() + + @staticmethod + def _has_jump(opcode): + return opcode in _opcode.hasjrel or opcode in _opcode.hasjabs + + def has_jump(self): return self._has_jump(self._opcode) - def is_cond_jump(self) -> bool: + def is_cond_jump(self): """Is a conditional jump?""" # Ex: POP_JUMP_IF_TRUE, JUMP_IF_FALSE_OR_POP - # IN 3.11+ the JUMP and the IF are no necessary adjacent in the name. - name = self._name - return "JUMP_" in name and "IF_" in name + return "JUMP_IF_" in self._name - def is_uncond_jump(self) -> bool: + def is_uncond_jump(self): """Is an unconditional jump?""" - # JUMP_BACKWARD has been introduced in 3.11+ - # JUMP_ABSOLUTE was removed in 3.11+ - return self.name in { - "JUMP_FORWARD", - "JUMP_ABSOLUTE", - "JUMP_BACKWARD", - "JUMP_BACKWARD_NO_INTERRUPT", - } - - def is_abs_jump(self) -> bool: - """Is an absolute jump.""" - return self._opcode in _opcode.hasjabs - - def is_forward_rel_jump(self) -> bool: - """Is a forward relative jump.""" - return self._opcode in _opcode.hasjrel and "BACKWARD" not in self._name - - def is_backward_rel_jump(self) -> bool: - """Is a backward relative jump.""" - return self._opcode in _opcode.hasjrel and "BACKWARD" in self._name - - def is_final(self) -> bool: + return self.name in {"JUMP_FORWARD", "JUMP_ABSOLUTE"} + + def is_final(self): if self._name in { "RETURN_VALUE", - "RETURN_CONST", "RAISE_VARARGS", "RERAISE", "BREAK_LOOP", @@ -707,173 +394,3 @@ def is_final(self) -> bool: if self.is_uncond_jump(): return True return False - - def __repr__(self) -> str: - if self._arg is not UNSET: - return "<%s arg=%r location=%s>" % (self._name, self._arg, self._location) - else: - return "<%s location=%s>" % (self._name, self._location) - - def __eq__(self, other: Any) -> bool: - if type(self) is not type(other): - return False - return self._cmp_key() == other._cmp_key() - - # --- Private API - - _name: str - - _location: Optional[InstrLocation] - - _opcode: int - - _arg: A - - def _set(self, name: str, arg: A) -> None: - if not isinstance(name, str): - raise TypeError("operation name must be a str") - try: - opcode = _opcode.opmap[name] - except KeyError: - raise ValueError(f"invalid operation name: {name}") # noqa - - if opcode >= MIN_INSTRUMENTED_OPCODE: - raise ValueError( - f"operation {name} is an instrumented or pseudo opcode. " - "Only base opcodes are supported" - ) - - self._check_arg(name, opcode, arg) - - self._name = name - self._opcode = opcode - self._arg = arg - - @staticmethod - def _has_jump(opcode) -> bool: - return opcode in _opcode.hasjrel or opcode in _opcode.hasjabs - - @abstractmethod - def _check_arg(self, name: str, opcode: int, arg: A) -> None: - pass - - @abstractmethod - def _cmp_key(self) -> Tuple[Optional[InstrLocation], str, Any]: - pass - - -InstrArg = Union[ - int, - str, - Label, - CellVar, - FreeVar, - "_bytecode.BasicBlock", - Compare, - Tuple[bool, str], - Tuple[bool, bool, str], -] - - -class Instr(BaseInstr[InstrArg]): - __slots__ = () - - def _cmp_key(self) -> Tuple[Optional[InstrLocation], str, Any]: - arg: Any = self._arg - if self._opcode in _opcode.hasconst: - arg = const_key(arg) - return (self._location, self._name, arg) - - def _check_arg(self, name: str, opcode: int, arg: InstrArg) -> None: - if name == "EXTENDED_ARG": - raise ValueError( - "only concrete instruction can contain EXTENDED_ARG, " - "highlevel instruction can represent arbitrary argument without it" - ) - - if opcode_has_argument(opcode): - if arg is UNSET: - raise ValueError("operation %s requires an argument" % name) - else: - if arg is not UNSET: - raise ValueError("operation %s has no argument" % name) - - if self._has_jump(opcode): - if not isinstance(arg, (Label, _bytecode.BasicBlock)): - raise TypeError( - "operation %s argument type must be " - "Label or BasicBlock, got %s" % (name, type(arg).__name__) - ) - - elif opcode in _opcode.hasfree: - if not isinstance(arg, (CellVar, FreeVar)): - raise TypeError( - "operation %s argument must be CellVar " - "or FreeVar, got %s" % (name, type(arg).__name__) - ) - - elif opcode in _opcode.haslocal or opcode in _opcode.hasname: - if name in BITFLAG_INSTRUCTIONS: - if not ( - isinstance(arg, tuple) - and len(arg) == 2 - and isinstance(arg[0], bool) - and isinstance(arg[1], str) - ): - raise TypeError( - "operation %s argument must be a tuple[bool, str], " - "got %s (value=%s)" % (name, type(arg).__name__, str(arg)) - ) - - elif name in BITFLAG2_INSTRUCTIONS: - if not ( - isinstance(arg, tuple) - and len(arg) == 3 - and isinstance(arg[0], bool) - and isinstance(arg[1], bool) - and isinstance(arg[2], str) - ): - raise TypeError( - "operation %s argument must be a tuple[bool, bool, str], " - "got %s (value=%s)" % (name, type(arg).__name__, str(arg)) - ) - - elif not isinstance(arg, str): - raise TypeError( - "operation %s argument must be a str, " - "got %s" % (name, type(arg).__name__) - ) - - elif opcode in _opcode.hasconst: - if isinstance(arg, Label): - raise ValueError( - "label argument cannot be used " "in %s operation" % name - ) - if isinstance(arg, _bytecode.BasicBlock): - raise ValueError( - "block argument cannot be used " "in %s operation" % name - ) - - elif opcode in _opcode.hascompare: - if not isinstance(arg, Compare): - raise TypeError( - "operation %s argument type must be " - "Compare, got %s" % (name, type(arg).__name__) - ) - - elif opcode in INTRINSIC_1OP: - if not isinstance(arg, Intrinsic1Op): - raise TypeError( - "operation %s argument type must be " - "Intrinsic1Op, got %s" % (name, type(arg).__name__) - ) - - elif opcode in INTRINSIC_2OP: - if not isinstance(arg, Intrinsic2Op): - raise TypeError( - "operation %s argument type must be " - "Intrinsic2Op, got %s" % (name, type(arg).__name__) - ) - - elif opcode_has_argument(opcode): - _check_arg_int(arg, name) diff --git a/_pydevd_frame_eval/vendored/bytecode/peephole_opt.py b/_pydevd_frame_eval/vendored/bytecode/peephole_opt.py new file mode 100644 index 000000000..9ece96bf0 --- /dev/null +++ b/_pydevd_frame_eval/vendored/bytecode/peephole_opt.py @@ -0,0 +1,491 @@ +""" +Peephole optimizer of CPython 3.6 reimplemented in pure Python using +the bytecode module. +""" +import opcode +import operator +import sys +from _pydevd_frame_eval.vendored.bytecode import Instr, Bytecode, ControlFlowGraph, BasicBlock, Compare + +JUMPS_ON_TRUE = frozenset( + ( + "POP_JUMP_IF_TRUE", + "JUMP_IF_TRUE_OR_POP", + ) +) + +NOT_COMPARE = { + Compare.IN: Compare.NOT_IN, + Compare.NOT_IN: Compare.IN, + Compare.IS: Compare.IS_NOT, + Compare.IS_NOT: Compare.IS, +} + +MAX_SIZE = 20 + + +class ExitUnchanged(Exception): + """Exception used to skip the peephole optimizer""" + + pass + + +class PeepholeOptimizer: + """Python reimplementation of the peephole optimizer. + + Copy of the C comment: + + Perform basic peephole optimizations to components of a code object. + The consts object should still be in list form to allow new constants + to be appended. + + To keep the optimizer simple, it bails out (does nothing) for code that + has a length over 32,700, and does not calculate extended arguments. + That allows us to avoid overflow and sign issues. Likewise, it bails when + the lineno table has complex encoding for gaps >= 255. EXTENDED_ARG can + appear before MAKE_FUNCTION; in this case both opcodes are skipped. + EXTENDED_ARG preceding any other opcode causes the optimizer to bail. + + Optimizations are restricted to simple transformations occuring within a + single basic block. All transformations keep the code size the same or + smaller. For those that reduce size, the gaps are initially filled with + NOPs. Later those NOPs are removed and the jump addresses retargeted in + a single pass. Code offset is adjusted accordingly. + """ + + def __init__(self): + # bytecode.ControlFlowGraph instance + self.code = None + self.const_stack = None + self.block_index = None + self.block = None + # index of the current instruction in self.block instructions + self.index = None + # whether we are in a LOAD_CONST sequence + self.in_consts = False + + def check_result(self, value): + try: + size = len(value) + except TypeError: + return True + return size <= MAX_SIZE + + def replace_load_const(self, nconst, instr, result): + # FIXME: remove temporary computed constants? + # FIXME: or at least reuse existing constants? + + self.in_consts = True + + load_const = Instr("LOAD_CONST", result, lineno=instr.lineno) + start = self.index - nconst - 1 + self.block[start : self.index] = (load_const,) + self.index -= nconst + + if nconst: + del self.const_stack[-nconst:] + self.const_stack.append(result) + self.in_consts = True + + def eval_LOAD_CONST(self, instr): + self.in_consts = True + value = instr.arg + self.const_stack.append(value) + self.in_consts = True + + def unaryop(self, op, instr): + try: + value = self.const_stack[-1] + result = op(value) + except IndexError: + return + + if not self.check_result(result): + return + + self.replace_load_const(1, instr, result) + + def eval_UNARY_POSITIVE(self, instr): + return self.unaryop(operator.pos, instr) + + def eval_UNARY_NEGATIVE(self, instr): + return self.unaryop(operator.neg, instr) + + def eval_UNARY_INVERT(self, instr): + return self.unaryop(operator.invert, instr) + + def get_next_instr(self, name): + try: + next_instr = self.block[self.index] + except IndexError: + return None + if next_instr.name == name: + return next_instr + return None + + def eval_UNARY_NOT(self, instr): + # Note: UNARY_NOT is not optimized + + next_instr = self.get_next_instr("POP_JUMP_IF_FALSE") + if next_instr is None: + return None + + # Replace UNARY_NOT+POP_JUMP_IF_FALSE with POP_JUMP_IF_TRUE + instr.set("POP_JUMP_IF_TRUE", next_instr.arg) + del self.block[self.index] + + def binop(self, op, instr): + try: + left = self.const_stack[-2] + right = self.const_stack[-1] + except IndexError: + return + + try: + result = op(left, right) + except Exception: + return + + if not self.check_result(result): + return + + self.replace_load_const(2, instr, result) + + def eval_BINARY_ADD(self, instr): + return self.binop(operator.add, instr) + + def eval_BINARY_SUBTRACT(self, instr): + return self.binop(operator.sub, instr) + + def eval_BINARY_MULTIPLY(self, instr): + return self.binop(operator.mul, instr) + + def eval_BINARY_TRUE_DIVIDE(self, instr): + return self.binop(operator.truediv, instr) + + def eval_BINARY_FLOOR_DIVIDE(self, instr): + return self.binop(operator.floordiv, instr) + + def eval_BINARY_MODULO(self, instr): + return self.binop(operator.mod, instr) + + def eval_BINARY_POWER(self, instr): + return self.binop(operator.pow, instr) + + def eval_BINARY_LSHIFT(self, instr): + return self.binop(operator.lshift, instr) + + def eval_BINARY_RSHIFT(self, instr): + return self.binop(operator.rshift, instr) + + def eval_BINARY_AND(self, instr): + return self.binop(operator.and_, instr) + + def eval_BINARY_OR(self, instr): + return self.binop(operator.or_, instr) + + def eval_BINARY_XOR(self, instr): + return self.binop(operator.xor, instr) + + def eval_BINARY_SUBSCR(self, instr): + return self.binop(operator.getitem, instr) + + def replace_container_of_consts(self, instr, container_type): + items = self.const_stack[-instr.arg :] + value = container_type(items) + self.replace_load_const(instr.arg, instr, value) + + def build_tuple_unpack_seq(self, instr): + next_instr = self.get_next_instr("UNPACK_SEQUENCE") + if next_instr is None or next_instr.arg != instr.arg: + return + + if instr.arg < 1: + return + + if self.const_stack and instr.arg <= len(self.const_stack): + nconst = instr.arg + start = self.index - 1 + + # Rewrite LOAD_CONST instructions in the reverse order + load_consts = self.block[start - nconst : start] + self.block[start - nconst : start] = reversed(load_consts) + + # Remove BUILD_TUPLE+UNPACK_SEQUENCE + self.block[start : start + 2] = () + self.index -= 2 + self.const_stack.clear() + return + + if instr.arg == 1: + # Replace BUILD_TUPLE 1 + UNPACK_SEQUENCE 1 with NOP + del self.block[self.index - 1 : self.index + 1] + elif instr.arg == 2: + # Replace BUILD_TUPLE 2 + UNPACK_SEQUENCE 2 with ROT_TWO + rot2 = Instr("ROT_TWO", lineno=instr.lineno) + self.block[self.index - 1 : self.index + 1] = (rot2,) + self.index -= 1 + self.const_stack.clear() + elif instr.arg == 3: + # Replace BUILD_TUPLE 3 + UNPACK_SEQUENCE 3 + # with ROT_THREE + ROT_TWO + rot3 = Instr("ROT_THREE", lineno=instr.lineno) + rot2 = Instr("ROT_TWO", lineno=instr.lineno) + self.block[self.index - 1 : self.index + 1] = (rot3, rot2) + self.index -= 1 + self.const_stack.clear() + + def build_tuple(self, instr, container_type): + if instr.arg > len(self.const_stack): + return + + next_instr = self.get_next_instr("COMPARE_OP") + if next_instr is None or next_instr.arg not in (Compare.IN, Compare.NOT_IN): + return + + self.replace_container_of_consts(instr, container_type) + return True + + def eval_BUILD_TUPLE(self, instr): + if not instr.arg: + return + + if instr.arg <= len(self.const_stack): + self.replace_container_of_consts(instr, tuple) + else: + self.build_tuple_unpack_seq(instr) + + def eval_BUILD_LIST(self, instr): + if not instr.arg: + return + + if not self.build_tuple(instr, tuple): + self.build_tuple_unpack_seq(instr) + + def eval_BUILD_SET(self, instr): + if not instr.arg: + return + + self.build_tuple(instr, frozenset) + + # Note: BUILD_SLICE is not optimized + + def eval_COMPARE_OP(self, instr): + # Note: COMPARE_OP: 2 < 3 is not optimized + + try: + new_arg = NOT_COMPARE[instr.arg] + except KeyError: + return + + if self.get_next_instr("UNARY_NOT") is None: + return + + # not (a is b) --> a is not b + # not (a in b) --> a not in b + # not (a is not b) --> a is b + # not (a not in b) --> a in b + instr.arg = new_arg + self.block[self.index - 1 : self.index + 1] = (instr,) + + def jump_if_or_pop(self, instr): + # Simplify conditional jump to conditional jump where the + # result of the first test implies the success of a similar + # test or the failure of the opposite test. + # + # Arises in code like: + # "if a and b:" + # "if a or b:" + # "a and b or c" + # "(a and b) and c" + # + # x:JUMP_IF_FALSE_OR_POP y y:JUMP_IF_FALSE_OR_POP z + # --> x:JUMP_IF_FALSE_OR_POP z + # + # x:JUMP_IF_FALSE_OR_POP y y:JUMP_IF_TRUE_OR_POP z + # --> x:POP_JUMP_IF_FALSE y+3 + # where y+3 is the instruction following the second test. + target_block = instr.arg + try: + target_instr = target_block[0] + except IndexError: + return + + if not target_instr.is_cond_jump(): + self.optimize_jump_to_cond_jump(instr) + return + + if (target_instr.name in JUMPS_ON_TRUE) == (instr.name in JUMPS_ON_TRUE): + # The second jump will be taken iff the first is. + + target2 = target_instr.arg + # The current opcode inherits its target's stack behaviour + instr.name = target_instr.name + instr.arg = target2 + self.block[self.index - 1] = instr + self.index -= 1 + else: + # The second jump is not taken if the first is (so jump past it), + # and all conditional jumps pop their argument when they're not + # taken (so change the first jump to pop its argument when it's + # taken). + if instr.name in JUMPS_ON_TRUE: + name = "POP_JUMP_IF_TRUE" + else: + name = "POP_JUMP_IF_FALSE" + + new_label = self.code.split_block(target_block, 1) + + instr.name = name + instr.arg = new_label + self.block[self.index - 1] = instr + self.index -= 1 + + def eval_JUMP_IF_FALSE_OR_POP(self, instr): + self.jump_if_or_pop(instr) + + def eval_JUMP_IF_TRUE_OR_POP(self, instr): + self.jump_if_or_pop(instr) + + def eval_NOP(self, instr): + # Remove NOP + del self.block[self.index - 1] + self.index -= 1 + + def optimize_jump_to_cond_jump(self, instr): + # Replace jumps to unconditional jumps + jump_label = instr.arg + assert isinstance(jump_label, BasicBlock), jump_label + + try: + target_instr = jump_label[0] + except IndexError: + return + + if instr.is_uncond_jump() and target_instr.name == "RETURN_VALUE": + # Replace JUMP_ABSOLUTE => RETURN_VALUE with RETURN_VALUE + self.block[self.index - 1] = target_instr + + elif target_instr.is_uncond_jump(): + # Replace JUMP_FORWARD t1 jumping to JUMP_FORWARD t2 + # with JUMP_ABSOLUTE t2 + jump_target2 = target_instr.arg + + name = instr.name + if instr.name == "JUMP_FORWARD": + name = "JUMP_ABSOLUTE" + else: + # FIXME: reimplement this check + # if jump_target2 < 0: + # # No backward relative jumps + # return + + # FIXME: remove this workaround and implement comment code ^^ + if instr.opcode in opcode.hasjrel: + return + + instr.name = name + instr.arg = jump_target2 + self.block[self.index - 1] = instr + + def optimize_jump(self, instr): + if instr.is_uncond_jump() and self.index == len(self.block): + # JUMP_ABSOLUTE at the end of a block which points to the + # following block: remove the jump, link the current block + # to the following block + block_index = self.block_index + target_block = instr.arg + target_block_index = self.code.get_block_index(target_block) + if target_block_index == block_index: + del self.block[self.index - 1] + self.block.next_block = target_block + return + + self.optimize_jump_to_cond_jump(instr) + + def iterblock(self, block): + self.block = block + self.index = 0 + while self.index < len(block): + instr = self.block[self.index] + self.index += 1 + yield instr + + def optimize_block(self, block): + self.const_stack.clear() + self.in_consts = False + + for instr in self.iterblock(block): + if not self.in_consts: + self.const_stack.clear() + self.in_consts = False + + meth_name = "eval_%s" % instr.name + meth = getattr(self, meth_name, None) + if meth is not None: + meth(instr) + elif instr.has_jump(): + self.optimize_jump(instr) + + # Note: Skipping over LOAD_CONST trueconst; POP_JUMP_IF_FALSE + # is not implemented, since it looks like the optimization + # is never trigerred in practice. The compiler already optimizes if + # and while statements. + + def remove_dead_blocks(self): + # FIXME: remove empty blocks? + + used_blocks = {id(self.code[0])} + for block in self.code: + if block.next_block is not None: + used_blocks.add(id(block.next_block)) + for instr in block: + if isinstance(instr, Instr) and isinstance(instr.arg, BasicBlock): + used_blocks.add(id(instr.arg)) + + block_index = 0 + while block_index < len(self.code): + block = self.code[block_index] + if id(block) not in used_blocks: + del self.code[block_index] + else: + block_index += 1 + + # FIXME: merge following blocks if block1 does not contain any + # jump and block1.next_block is block2 + + def optimize_cfg(self, cfg): + self.code = cfg + self.const_stack = [] + + self.remove_dead_blocks() + + self.block_index = 0 + while self.block_index < len(self.code): + block = self.code[self.block_index] + self.block_index += 1 + self.optimize_block(block) + + def optimize(self, code_obj): + bytecode = Bytecode.from_code(code_obj) + cfg = ControlFlowGraph.from_bytecode(bytecode) + + self.optimize_cfg(cfg) + + bytecode = cfg.to_bytecode() + code = bytecode.to_code() + return code + + +# Code transformer for the PEP 511 +class CodeTransformer: + name = "pyopt" + + def code_transformer(self, code, context): + if sys.flags.verbose: + print( + "Optimize %s:%s: %s" + % (code.co_filename, code.co_firstlineno, code.co_name) + ) + optimizer = PeepholeOptimizer() + return optimizer.optimize(code) diff --git a/_pydevd_frame_eval/vendored/bytecode/py.typed b/_pydevd_frame_eval/vendored/bytecode/py.typed deleted file mode 100644 index e69de29bb..000000000 diff --git a/_pydevd_frame_eval/vendored/bytecode/tests/__init__.py b/_pydevd_frame_eval/vendored/bytecode/tests/__init__.py new file mode 100644 index 000000000..ee0f7d1b5 --- /dev/null +++ b/_pydevd_frame_eval/vendored/bytecode/tests/__init__.py @@ -0,0 +1,154 @@ +import sys +import textwrap +import types +import unittest + +from _pydevd_frame_eval.vendored.bytecode import ( + UNSET, + Label, + Instr, + ConcreteInstr, + BasicBlock, # noqa + Bytecode, + ControlFlowGraph, + ConcreteBytecode, +) + + +def _format_instr_list(block, labels, lineno): + instr_list = [] + for instr in block: + if not isinstance(instr, Label): + if isinstance(instr, ConcreteInstr): + cls_name = "ConcreteInstr" + else: + cls_name = "Instr" + arg = instr.arg + if arg is not UNSET: + if isinstance(arg, Label): + arg = labels[arg] + elif isinstance(arg, BasicBlock): + arg = labels[id(arg)] + else: + arg = repr(arg) + if lineno: + text = "%s(%r, %s, lineno=%s)" % ( + cls_name, + instr.name, + arg, + instr.lineno, + ) + else: + text = "%s(%r, %s)" % (cls_name, instr.name, arg) + else: + if lineno: + text = "%s(%r, lineno=%s)" % (cls_name, instr.name, instr.lineno) + else: + text = "%s(%r)" % (cls_name, instr.name) + else: + text = labels[instr] + instr_list.append(text) + return "[%s]" % ",\n ".join(instr_list) + + +def dump_bytecode(code, lineno=False): + """ + Use this function to write unit tests: copy/paste its output to + write a self.assertBlocksEqual() check. + """ + print() + + if isinstance(code, (Bytecode, ConcreteBytecode)): + is_concrete = isinstance(code, ConcreteBytecode) + if is_concrete: + block = list(code) + else: + block = code + + indent = " " * 8 + labels = {} + for index, instr in enumerate(block): + if isinstance(instr, Label): + name = "label_instr%s" % index + labels[instr] = name + + if is_concrete: + name = "ConcreteBytecode" + print(indent + "code = %s()" % name) + if code.argcount: + print(indent + "code.argcount = %s" % code.argcount) + if sys.version_info > (3, 8): + if code.posonlyargcount: + print(indent + "code.posonlyargcount = %s" % code.posonlyargcount) + if code.kwonlyargcount: + print(indent + "code.kwargonlycount = %s" % code.kwonlyargcount) + print(indent + "code.flags = %#x" % code.flags) + if code.consts: + print(indent + "code.consts = %r" % code.consts) + if code.names: + print(indent + "code.names = %r" % code.names) + if code.varnames: + print(indent + "code.varnames = %r" % code.varnames) + + for name in sorted(labels.values()): + print(indent + "%s = Label()" % name) + + if is_concrete: + text = indent + "code.extend(" + indent = " " * len(text) + else: + text = indent + "code = Bytecode(" + indent = " " * len(text) + + lines = _format_instr_list(code, labels, lineno).splitlines() + last_line = len(lines) - 1 + for index, line in enumerate(lines): + if index == 0: + print(text + lines[0]) + elif index == last_line: + print(indent + line + ")") + else: + print(indent + line) + + print() + else: + assert isinstance(code, ControlFlowGraph) + labels = {} + for block_index, block in enumerate(code): + labels[id(block)] = "code[%s]" % block_index + + for block_index, block in enumerate(code): + text = _format_instr_list(block, labels, lineno) + if block_index != len(code) - 1: + text += "," + print(text) + print() + + +def get_code(source, *, filename="", function=False): + source = textwrap.dedent(source).strip() + code = compile(source, filename, "exec") + if function: + sub_code = [ + const for const in code.co_consts if isinstance(const, types.CodeType) + ] + if len(sub_code) != 1: + raise ValueError("unable to find function code") + code = sub_code[0] + return code + + +def disassemble(source, *, filename="", function=False): + code = get_code(source, filename=filename, function=function) + return Bytecode.from_code(code) + + +class TestCase(unittest.TestCase): + def assertBlocksEqual(self, code, *expected_blocks): + self.assertEqual(len(code), len(expected_blocks)) + + for block1, block2 in zip(code, expected_blocks): + block_index = code.get_block_index(block1) + self.assertListEqual( + list(block1), block2, "Block #%s is different" % block_index + ) diff --git a/_pydevd_frame_eval/vendored/bytecode/tests/test_bytecode.py b/_pydevd_frame_eval/vendored/bytecode/tests/test_bytecode.py new file mode 100644 index 000000000..c629f75e9 --- /dev/null +++ b/_pydevd_frame_eval/vendored/bytecode/tests/test_bytecode.py @@ -0,0 +1,488 @@ + +import pytest +from tests_python.debugger_unittest import IS_PY36_OR_GREATER, IS_CPYTHON +from tests_python.debug_constants import TEST_CYTHON +pytestmark = pytest.mark.skipif(not IS_PY36_OR_GREATER or not IS_CPYTHON or not TEST_CYTHON, reason='Requires CPython >= 3.6') +#!/usr/bin/env python3 +import sys +import textwrap +import unittest +from _pydevd_frame_eval.vendored.bytecode import Label, Instr, FreeVar, Bytecode, SetLineno, ConcreteInstr +from _pydevd_frame_eval.vendored.bytecode.tests import TestCase, get_code + + +class BytecodeTests(TestCase): + maxDiff = 80 * 100 + + def test_constructor(self): + code = Bytecode() + self.assertEqual(code.name, "") + self.assertEqual(code.filename, "") + self.assertEqual(code.flags, 0) + self.assertEqual(code, []) + + def test_invalid_types(self): + code = Bytecode() + code.append(123) + with self.assertRaises(ValueError): + list(code) + with self.assertRaises(ValueError): + code.legalize() + with self.assertRaises(ValueError): + Bytecode([123]) + + def test_legalize(self): + code = Bytecode() + code.first_lineno = 3 + code.extend( + [ + Instr("LOAD_CONST", 7), + Instr("STORE_NAME", "x"), + Instr("LOAD_CONST", 8, lineno=4), + Instr("STORE_NAME", "y"), + Label(), + SetLineno(5), + Instr("LOAD_CONST", 9, lineno=6), + Instr("STORE_NAME", "z"), + ] + ) + + code.legalize() + self.assertListEqual( + code, + [ + Instr("LOAD_CONST", 7, lineno=3), + Instr("STORE_NAME", "x", lineno=3), + Instr("LOAD_CONST", 8, lineno=4), + Instr("STORE_NAME", "y", lineno=4), + Label(), + Instr("LOAD_CONST", 9, lineno=5), + Instr("STORE_NAME", "z", lineno=5), + ], + ) + + def test_slice(self): + code = Bytecode() + code.first_lineno = 3 + code.extend( + [ + Instr("LOAD_CONST", 7), + Instr("STORE_NAME", "x"), + SetLineno(4), + Instr("LOAD_CONST", 8), + Instr("STORE_NAME", "y"), + SetLineno(5), + Instr("LOAD_CONST", 9), + Instr("STORE_NAME", "z"), + ] + ) + sliced_code = code[:] + self.assertEqual(code, sliced_code) + for name in ( + "argcount", + "posonlyargcount", + "kwonlyargcount", + "first_lineno", + "name", + "filename", + "docstring", + "cellvars", + "freevars", + "argnames", + ): + self.assertEqual( + getattr(code, name, None), getattr(sliced_code, name, None) + ) + + def test_copy(self): + code = Bytecode() + code.first_lineno = 3 + code.extend( + [ + Instr("LOAD_CONST", 7), + Instr("STORE_NAME", "x"), + SetLineno(4), + Instr("LOAD_CONST", 8), + Instr("STORE_NAME", "y"), + SetLineno(5), + Instr("LOAD_CONST", 9), + Instr("STORE_NAME", "z"), + ] + ) + + copy_code = code.copy() + self.assertEqual(code, copy_code) + for name in ( + "argcount", + "posonlyargcount", + "kwonlyargcount", + "first_lineno", + "name", + "filename", + "docstring", + "cellvars", + "freevars", + "argnames", + ): + self.assertEqual(getattr(code, name, None), getattr(copy_code, name, None)) + + def test_from_code(self): + code = get_code( + """ + if test: + x = 1 + else: + x = 2 + """ + ) + bytecode = Bytecode.from_code(code) + label_else = Label() + label_exit = Label() + if sys.version_info < (3, 10): + self.assertEqual( + bytecode, + [ + Instr("LOAD_NAME", "test", lineno=1), + Instr("POP_JUMP_IF_FALSE", label_else, lineno=1), + Instr("LOAD_CONST", 1, lineno=2), + Instr("STORE_NAME", "x", lineno=2), + Instr("JUMP_FORWARD", label_exit, lineno=2), + label_else, + Instr("LOAD_CONST", 2, lineno=4), + Instr("STORE_NAME", "x", lineno=4), + label_exit, + Instr("LOAD_CONST", None, lineno=4), + Instr("RETURN_VALUE", lineno=4), + ], + ) + # Control flow handling appears to have changed under Python 3.10 + else: + self.assertEqual( + bytecode, + [ + Instr("LOAD_NAME", "test", lineno=1), + Instr("POP_JUMP_IF_FALSE", label_else, lineno=1), + Instr("LOAD_CONST", 1, lineno=2), + Instr("STORE_NAME", "x", lineno=2), + Instr("LOAD_CONST", None, lineno=2), + Instr("RETURN_VALUE", lineno=2), + label_else, + Instr("LOAD_CONST", 2, lineno=4), + Instr("STORE_NAME", "x", lineno=4), + Instr("LOAD_CONST", None, lineno=4), + Instr("RETURN_VALUE", lineno=4), + ], + ) + + def test_from_code_freevars(self): + ns = {} + exec( + textwrap.dedent( + """ + def create_func(): + x = 1 + def func(): + return x + return func + + func = create_func() + """ + ), + ns, + ns, + ) + code = ns["func"].__code__ + + bytecode = Bytecode.from_code(code) + self.assertEqual( + bytecode, + [ + Instr("LOAD_DEREF", FreeVar("x"), lineno=5), + Instr("RETURN_VALUE", lineno=5), + ], + ) + + def test_from_code_load_fast(self): + code = get_code( + """ + def func(): + x = 33 + y = x + """, + function=True, + ) + code = Bytecode.from_code(code) + self.assertEqual( + code, + [ + Instr("LOAD_CONST", 33, lineno=2), + Instr("STORE_FAST", "x", lineno=2), + Instr("LOAD_FAST", "x", lineno=3), + Instr("STORE_FAST", "y", lineno=3), + Instr("LOAD_CONST", None, lineno=3), + Instr("RETURN_VALUE", lineno=3), + ], + ) + + def test_setlineno(self): + # x = 7 + # y = 8 + # z = 9 + code = Bytecode() + code.first_lineno = 3 + code.extend( + [ + Instr("LOAD_CONST", 7), + Instr("STORE_NAME", "x"), + SetLineno(4), + Instr("LOAD_CONST", 8), + Instr("STORE_NAME", "y"), + SetLineno(5), + Instr("LOAD_CONST", 9), + Instr("STORE_NAME", "z"), + ] + ) + + concrete = code.to_concrete_bytecode() + self.assertEqual(concrete.consts, [7, 8, 9]) + self.assertEqual(concrete.names, ["x", "y", "z"]) + self.assertListEqual( + list(concrete), + [ + ConcreteInstr("LOAD_CONST", 0, lineno=3), + ConcreteInstr("STORE_NAME", 0, lineno=3), + ConcreteInstr("LOAD_CONST", 1, lineno=4), + ConcreteInstr("STORE_NAME", 1, lineno=4), + ConcreteInstr("LOAD_CONST", 2, lineno=5), + ConcreteInstr("STORE_NAME", 2, lineno=5), + ], + ) + + def test_to_code(self): + code = Bytecode() + code.first_lineno = 50 + code.extend( + [ + Instr("LOAD_NAME", "print"), + Instr("LOAD_CONST", "%s"), + Instr("LOAD_GLOBAL", "a"), + Instr("BINARY_MODULO"), + Instr("CALL_FUNCTION", 1), + Instr("RETURN_VALUE"), + ] + ) + co = code.to_code() + # hopefully this is obvious from inspection? :-) + self.assertEqual(co.co_stacksize, 3) + + co = code.to_code(stacksize=42) + self.assertEqual(co.co_stacksize, 42) + + def test_negative_size_unary(self): + opnames = ( + "UNARY_POSITIVE", + "UNARY_NEGATIVE", + "UNARY_NOT", + "UNARY_INVERT", + ) + for opname in opnames: + with self.subTest(): + code = Bytecode() + code.first_lineno = 1 + code.extend([Instr(opname)]) + with self.assertRaises(RuntimeError): + code.compute_stacksize() + + def test_negative_size_unary_with_disable_check_of_pre_and_post(self): + opnames = ( + "UNARY_POSITIVE", + "UNARY_NEGATIVE", + "UNARY_NOT", + "UNARY_INVERT", + ) + for opname in opnames: + with self.subTest(): + code = Bytecode() + code.first_lineno = 1 + code.extend([Instr(opname)]) + co = code.to_code(check_pre_and_post=False) + self.assertEqual(co.co_stacksize, 0) + + def test_negative_size_binary(self): + opnames = ( + "BINARY_POWER", + "BINARY_MULTIPLY", + "BINARY_MATRIX_MULTIPLY", + "BINARY_FLOOR_DIVIDE", + "BINARY_TRUE_DIVIDE", + "BINARY_MODULO", + "BINARY_ADD", + "BINARY_SUBTRACT", + "BINARY_SUBSCR", + "BINARY_LSHIFT", + "BINARY_RSHIFT", + "BINARY_AND", + "BINARY_XOR", + "BINARY_OR", + ) + for opname in opnames: + with self.subTest(): + code = Bytecode() + code.first_lineno = 1 + code.extend([Instr("LOAD_CONST", 1), Instr(opname)]) + with self.assertRaises(RuntimeError): + code.compute_stacksize() + + def test_negative_size_binary_with_disable_check_of_pre_and_post(self): + opnames = ( + "BINARY_POWER", + "BINARY_MULTIPLY", + "BINARY_MATRIX_MULTIPLY", + "BINARY_FLOOR_DIVIDE", + "BINARY_TRUE_DIVIDE", + "BINARY_MODULO", + "BINARY_ADD", + "BINARY_SUBTRACT", + "BINARY_SUBSCR", + "BINARY_LSHIFT", + "BINARY_RSHIFT", + "BINARY_AND", + "BINARY_XOR", + "BINARY_OR", + ) + for opname in opnames: + with self.subTest(): + code = Bytecode() + code.first_lineno = 1 + code.extend([Instr("LOAD_CONST", 1), Instr(opname)]) + co = code.to_code(check_pre_and_post=False) + self.assertEqual(co.co_stacksize, 1) + + def test_negative_size_call(self): + code = Bytecode() + code.first_lineno = 1 + code.extend([Instr("CALL_FUNCTION", 0)]) + with self.assertRaises(RuntimeError): + code.compute_stacksize() + + def test_negative_size_unpack(self): + opnames = ( + "UNPACK_SEQUENCE", + "UNPACK_EX", + ) + for opname in opnames: + with self.subTest(): + code = Bytecode() + code.first_lineno = 1 + code.extend([Instr(opname, 1)]) + with self.assertRaises(RuntimeError): + code.compute_stacksize() + + def test_negative_size_build(self): + opnames = ( + "BUILD_TUPLE", + "BUILD_LIST", + "BUILD_SET", + ) + if sys.version_info >= (3, 6): + opnames = (*opnames, "BUILD_STRING") + + for opname in opnames: + with self.subTest(): + code = Bytecode() + code.first_lineno = 1 + code.extend([Instr(opname, 1)]) + with self.assertRaises(RuntimeError): + code.compute_stacksize() + + def test_negative_size_build_map(self): + code = Bytecode() + code.first_lineno = 1 + code.extend([Instr("LOAD_CONST", 1), Instr("BUILD_MAP", 1)]) + with self.assertRaises(RuntimeError): + code.compute_stacksize() + + def test_negative_size_build_map_with_disable_check_of_pre_and_post(self): + code = Bytecode() + code.first_lineno = 1 + code.extend([Instr("LOAD_CONST", 1), Instr("BUILD_MAP", 1)]) + co = code.to_code(check_pre_and_post=False) + self.assertEqual(co.co_stacksize, 1) + + @unittest.skipIf(sys.version_info < (3, 6), "Inexistent opcode") + def test_negative_size_build_const_map(self): + code = Bytecode() + code.first_lineno = 1 + code.extend([Instr("LOAD_CONST", ("a",)), Instr("BUILD_CONST_KEY_MAP", 1)]) + with self.assertRaises(RuntimeError): + code.compute_stacksize() + + @unittest.skipIf(sys.version_info < (3, 6), "Inexistent opcode") + def test_negative_size_build_const_map_with_disable_check_of_pre_and_post(self): + code = Bytecode() + code.first_lineno = 1 + code.extend([Instr("LOAD_CONST", ("a",)), Instr("BUILD_CONST_KEY_MAP", 1)]) + co = code.to_code(check_pre_and_post=False) + self.assertEqual(co.co_stacksize, 1) + + def test_empty_dup(self): + code = Bytecode() + code.first_lineno = 1 + code.extend([Instr("DUP_TOP")]) + with self.assertRaises(RuntimeError): + code.compute_stacksize() + + def test_not_enough_dup(self): + code = Bytecode() + code.first_lineno = 1 + code.extend([Instr("LOAD_CONST", 1), Instr("DUP_TOP_TWO")]) + with self.assertRaises(RuntimeError): + code.compute_stacksize() + + def test_not_enough_rot(self): + opnames = ["ROT_TWO", "ROT_THREE"] + if sys.version_info >= (3, 8): + opnames.append("ROT_FOUR") + for opname in opnames: + with self.subTest(): + code = Bytecode() + code.first_lineno = 1 + code.extend([Instr("LOAD_CONST", 1), Instr(opname)]) + with self.assertRaises(RuntimeError): + code.compute_stacksize() + + def test_not_enough_rot_with_disable_check_of_pre_and_post(self): + opnames = ["ROT_TWO", "ROT_THREE"] + if sys.version_info >= (3, 8): + opnames.append("ROT_FOUR") + for opname in opnames: + with self.subTest(): + code = Bytecode() + code.first_lineno = 1 + code.extend([Instr("LOAD_CONST", 1), Instr(opname)]) + co = code.to_code(check_pre_and_post=False) + self.assertEqual(co.co_stacksize, 1) + + def test_for_iter_stack_effect_computation(self): + with self.subTest(): + code = Bytecode() + code.first_lineno = 1 + lab1 = Label() + lab2 = Label() + code.extend( + [ + lab1, + Instr("FOR_ITER", lab2), + Instr("STORE_FAST", "i"), + Instr("JUMP_ABSOLUTE", lab1), + lab2, + ] + ) + with self.assertRaises(RuntimeError): + # Use compute_stacksize since the code is so broken that conversion + # to from concrete is actually broken + code.compute_stacksize(check_pre_and_post=False) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/_pydevd_frame_eval/vendored/bytecode/tests/test_cfg.py b/_pydevd_frame_eval/vendored/bytecode/tests/test_cfg.py new file mode 100644 index 000000000..9b5b07b1c --- /dev/null +++ b/_pydevd_frame_eval/vendored/bytecode/tests/test_cfg.py @@ -0,0 +1,836 @@ + +import pytest +from tests_python.debugger_unittest import IS_PY36_OR_GREATER, IS_CPYTHON +from tests_python.debug_constants import TEST_CYTHON +pytestmark = pytest.mark.skipif(not IS_PY36_OR_GREATER or not IS_CPYTHON or not TEST_CYTHON, reason='Requires CPython >= 3.6') +#!/usr/bin/env python3 +import io +import sys +import unittest +import contextlib +from _pydevd_frame_eval.vendored.bytecode import ( + Label, + Compare, + SetLineno, + Instr, + Bytecode, + BasicBlock, + ControlFlowGraph, +) +from _pydevd_frame_eval.vendored.bytecode.concrete import OFFSET_AS_INSTRUCTION +from _pydevd_frame_eval.vendored.bytecode.tests import disassemble as _disassemble, TestCase + + +def disassemble( + source, *, filename="", function=False, remove_last_return_none=False +): + code = _disassemble(source, filename=filename, function=function) + blocks = ControlFlowGraph.from_bytecode(code) + if remove_last_return_none: + # drop LOAD_CONST+RETURN_VALUE to only keep 2 instructions, + # to make unit tests shorter + block = blocks[-1] + test = ( + block[-2].name == "LOAD_CONST" + and block[-2].arg is None + and block[-1].name == "RETURN_VALUE" + ) + if not test: + raise ValueError( + "unable to find implicit RETURN_VALUE : %s" % block[-2:] + ) + del block[-2:] + return blocks + + +class BlockTests(unittest.TestCase): + def test_iter_invalid_types(self): + # Labels are not allowed in basic blocks + block = BasicBlock() + block.append(Label()) + with self.assertRaises(ValueError): + list(block) + with self.assertRaises(ValueError): + block.legalize(1) + + # Only one jump allowed and only at the end + block = BasicBlock() + block2 = BasicBlock() + block.extend([Instr("JUMP_ABSOLUTE", block2), Instr("NOP")]) + with self.assertRaises(ValueError): + list(block) + with self.assertRaises(ValueError): + block.legalize(1) + + # jump target must be a BasicBlock + block = BasicBlock() + label = Label() + block.extend([Instr("JUMP_ABSOLUTE", label)]) + with self.assertRaises(ValueError): + list(block) + with self.assertRaises(ValueError): + block.legalize(1) + + def test_slice(self): + block = BasicBlock([Instr("NOP")]) + next_block = BasicBlock() + block.next_block = next_block + self.assertEqual(block, block[:]) + self.assertIs(next_block, block[:].next_block) + + def test_copy(self): + block = BasicBlock([Instr("NOP")]) + next_block = BasicBlock() + block.next_block = next_block + self.assertEqual(block, block.copy()) + self.assertIs(next_block, block.copy().next_block) + + +class BytecodeBlocksTests(TestCase): + maxDiff = 80 * 100 + + def test_constructor(self): + code = ControlFlowGraph() + self.assertEqual(code.name, "") + self.assertEqual(code.filename, "") + self.assertEqual(code.flags, 0) + self.assertBlocksEqual(code, []) + + def test_attr(self): + source = """ + first_line = 1 + + def func(arg1, arg2, *, arg3): + x = 1 + y = 2 + return arg1 + """ + code = disassemble(source, filename="hello.py", function=True) + self.assertEqual(code.argcount, 2) + self.assertEqual(code.filename, "hello.py") + self.assertEqual(code.first_lineno, 3) + if sys.version_info > (3, 8): + self.assertEqual(code.posonlyargcount, 0) + self.assertEqual(code.kwonlyargcount, 1) + self.assertEqual(code.name, "func") + self.assertEqual(code.cellvars, []) + + code.name = "name" + code.filename = "filename" + code.flags = 123 + self.assertEqual(code.name, "name") + self.assertEqual(code.filename, "filename") + self.assertEqual(code.flags, 123) + + # FIXME: test non-empty cellvars + + def test_add_del_block(self): + code = ControlFlowGraph() + code[0].append(Instr("LOAD_CONST", 0)) + + block = code.add_block() + self.assertEqual(len(code), 2) + self.assertIs(block, code[1]) + + code[1].append(Instr("LOAD_CONST", 2)) + self.assertBlocksEqual(code, [Instr("LOAD_CONST", 0)], [Instr("LOAD_CONST", 2)]) + + del code[0] + self.assertBlocksEqual(code, [Instr("LOAD_CONST", 2)]) + + del code[0] + self.assertEqual(len(code), 0) + + def test_setlineno(self): + # x = 7 + # y = 8 + # z = 9 + code = Bytecode() + code.first_lineno = 3 + code.extend( + [ + Instr("LOAD_CONST", 7), + Instr("STORE_NAME", "x"), + SetLineno(4), + Instr("LOAD_CONST", 8), + Instr("STORE_NAME", "y"), + SetLineno(5), + Instr("LOAD_CONST", 9), + Instr("STORE_NAME", "z"), + ] + ) + + blocks = ControlFlowGraph.from_bytecode(code) + self.assertBlocksEqual( + blocks, + [ + Instr("LOAD_CONST", 7), + Instr("STORE_NAME", "x"), + SetLineno(4), + Instr("LOAD_CONST", 8), + Instr("STORE_NAME", "y"), + SetLineno(5), + Instr("LOAD_CONST", 9), + Instr("STORE_NAME", "z"), + ], + ) + + def test_legalize(self): + code = Bytecode() + code.first_lineno = 3 + code.extend( + [ + Instr("LOAD_CONST", 7), + Instr("STORE_NAME", "x"), + Instr("LOAD_CONST", 8, lineno=4), + Instr("STORE_NAME", "y"), + SetLineno(5), + Instr("LOAD_CONST", 9, lineno=6), + Instr("STORE_NAME", "z"), + ] + ) + + blocks = ControlFlowGraph.from_bytecode(code) + blocks.legalize() + self.assertBlocksEqual( + blocks, + [ + Instr("LOAD_CONST", 7, lineno=3), + Instr("STORE_NAME", "x", lineno=3), + Instr("LOAD_CONST", 8, lineno=4), + Instr("STORE_NAME", "y", lineno=4), + Instr("LOAD_CONST", 9, lineno=5), + Instr("STORE_NAME", "z", lineno=5), + ], + ) + + def test_repr(self): + r = repr(ControlFlowGraph()) + self.assertIn("ControlFlowGraph", r) + self.assertIn("1", r) + + def test_to_bytecode(self): + # if test: + # x = 2 + # x = 5 + blocks = ControlFlowGraph() + blocks.add_block() + blocks.add_block() + blocks[0].extend( + [ + Instr("LOAD_NAME", "test", lineno=1), + Instr("POP_JUMP_IF_FALSE", blocks[2], lineno=1), + ] + ) + + blocks[1].extend( + [ + Instr("LOAD_CONST", 5, lineno=2), + Instr("STORE_NAME", "x", lineno=2), + Instr("JUMP_FORWARD", blocks[2], lineno=2), + ] + ) + + blocks[2].extend( + [ + Instr("LOAD_CONST", 7, lineno=3), + Instr("STORE_NAME", "x", lineno=3), + Instr("LOAD_CONST", None, lineno=3), + Instr("RETURN_VALUE", lineno=3), + ] + ) + + bytecode = blocks.to_bytecode() + label = Label() + self.assertEqual( + bytecode, + [ + Instr("LOAD_NAME", "test", lineno=1), + Instr("POP_JUMP_IF_FALSE", label, lineno=1), + Instr("LOAD_CONST", 5, lineno=2), + Instr("STORE_NAME", "x", lineno=2), + Instr("JUMP_FORWARD", label, lineno=2), + label, + Instr("LOAD_CONST", 7, lineno=3), + Instr("STORE_NAME", "x", lineno=3), + Instr("LOAD_CONST", None, lineno=3), + Instr("RETURN_VALUE", lineno=3), + ], + ) + # FIXME: test other attributes + + def test_label_at_the_end(self): + label = Label() + code = Bytecode( + [ + Instr("LOAD_NAME", "x"), + Instr("UNARY_NOT"), + Instr("POP_JUMP_IF_FALSE", label), + Instr("LOAD_CONST", 9), + Instr("STORE_NAME", "y"), + label, + ] + ) + + cfg = ControlFlowGraph.from_bytecode(code) + self.assertBlocksEqual( + cfg, + [ + Instr("LOAD_NAME", "x"), + Instr("UNARY_NOT"), + Instr("POP_JUMP_IF_FALSE", cfg[2]), + ], + [Instr("LOAD_CONST", 9), Instr("STORE_NAME", "y")], + [], + ) + + def test_from_bytecode(self): + bytecode = Bytecode() + label = Label() + bytecode.extend( + [ + Instr("LOAD_NAME", "test", lineno=1), + Instr("POP_JUMP_IF_FALSE", label, lineno=1), + Instr("LOAD_CONST", 5, lineno=2), + Instr("STORE_NAME", "x", lineno=2), + Instr("JUMP_FORWARD", label, lineno=2), + # dead code! + Instr("LOAD_CONST", 7, lineno=4), + Instr("STORE_NAME", "x", lineno=4), + Label(), # unused label + label, + Label(), # unused label + Instr("LOAD_CONST", None, lineno=4), + Instr("RETURN_VALUE", lineno=4), + ] + ) + + blocks = ControlFlowGraph.from_bytecode(bytecode) + label2 = blocks[3] + self.assertBlocksEqual( + blocks, + [ + Instr("LOAD_NAME", "test", lineno=1), + Instr("POP_JUMP_IF_FALSE", label2, lineno=1), + ], + [ + Instr("LOAD_CONST", 5, lineno=2), + Instr("STORE_NAME", "x", lineno=2), + Instr("JUMP_FORWARD", label2, lineno=2), + ], + [Instr("LOAD_CONST", 7, lineno=4), Instr("STORE_NAME", "x", lineno=4)], + [Instr("LOAD_CONST", None, lineno=4), Instr("RETURN_VALUE", lineno=4)], + ) + # FIXME: test other attributes + + def test_from_bytecode_loop(self): + # for x in (1, 2, 3): + # if x == 2: + # break + # continue + + if sys.version_info < (3, 8): + label_loop_start = Label() + label_loop_exit = Label() + label_loop_end = Label() + + code = Bytecode() + code.extend( + ( + Instr("SETUP_LOOP", label_loop_end, lineno=1), + Instr("LOAD_CONST", (1, 2, 3), lineno=1), + Instr("GET_ITER", lineno=1), + label_loop_start, + Instr("FOR_ITER", label_loop_exit, lineno=1), + Instr("STORE_NAME", "x", lineno=1), + Instr("LOAD_NAME", "x", lineno=2), + Instr("LOAD_CONST", 2, lineno=2), + Instr("COMPARE_OP", Compare.EQ, lineno=2), + Instr("POP_JUMP_IF_FALSE", label_loop_start, lineno=2), + Instr("BREAK_LOOP", lineno=3), + Instr("JUMP_ABSOLUTE", label_loop_start, lineno=4), + Instr("JUMP_ABSOLUTE", label_loop_start, lineno=4), + label_loop_exit, + Instr("POP_BLOCK", lineno=4), + label_loop_end, + Instr("LOAD_CONST", None, lineno=4), + Instr("RETURN_VALUE", lineno=4), + ) + ) + blocks = ControlFlowGraph.from_bytecode(code) + + expected = [ + [Instr("SETUP_LOOP", blocks[8], lineno=1)], + [Instr("LOAD_CONST", (1, 2, 3), lineno=1), Instr("GET_ITER", lineno=1)], + [Instr("FOR_ITER", blocks[7], lineno=1)], + [ + Instr("STORE_NAME", "x", lineno=1), + Instr("LOAD_NAME", "x", lineno=2), + Instr("LOAD_CONST", 2, lineno=2), + Instr("COMPARE_OP", Compare.EQ, lineno=2), + Instr("POP_JUMP_IF_FALSE", blocks[2], lineno=2), + ], + [Instr("BREAK_LOOP", lineno=3)], + [Instr("JUMP_ABSOLUTE", blocks[2], lineno=4)], + [Instr("JUMP_ABSOLUTE", blocks[2], lineno=4)], + [Instr("POP_BLOCK", lineno=4)], + [Instr("LOAD_CONST", None, lineno=4), Instr("RETURN_VALUE", lineno=4)], + ] + self.assertBlocksEqual(blocks, *expected) + else: + label_loop_start = Label() + label_loop_exit = Label() + + code = Bytecode() + code.extend( + ( + Instr("LOAD_CONST", (1, 2, 3), lineno=1), + Instr("GET_ITER", lineno=1), + label_loop_start, + Instr("FOR_ITER", label_loop_exit, lineno=1), + Instr("STORE_NAME", "x", lineno=1), + Instr("LOAD_NAME", "x", lineno=2), + Instr("LOAD_CONST", 2, lineno=2), + Instr("COMPARE_OP", Compare.EQ, lineno=2), + Instr("POP_JUMP_IF_FALSE", label_loop_start, lineno=2), + Instr("JUMP_ABSOLUTE", label_loop_exit, lineno=3), + Instr("JUMP_ABSOLUTE", label_loop_start, lineno=4), + Instr("JUMP_ABSOLUTE", label_loop_start, lineno=4), + label_loop_exit, + Instr("LOAD_CONST", None, lineno=4), + Instr("RETURN_VALUE", lineno=4), + ) + ) + blocks = ControlFlowGraph.from_bytecode(code) + + expected = [ + [Instr("LOAD_CONST", (1, 2, 3), lineno=1), Instr("GET_ITER", lineno=1)], + [Instr("FOR_ITER", blocks[6], lineno=1)], + [ + Instr("STORE_NAME", "x", lineno=1), + Instr("LOAD_NAME", "x", lineno=2), + Instr("LOAD_CONST", 2, lineno=2), + Instr("COMPARE_OP", Compare.EQ, lineno=2), + Instr("POP_JUMP_IF_FALSE", blocks[1], lineno=2), + ], + [Instr("JUMP_ABSOLUTE", blocks[6], lineno=3)], + [Instr("JUMP_ABSOLUTE", blocks[1], lineno=4)], + [Instr("JUMP_ABSOLUTE", blocks[1], lineno=4)], + [Instr("LOAD_CONST", None, lineno=4), Instr("RETURN_VALUE", lineno=4)], + ] + self.assertBlocksEqual(blocks, *expected) + + +class BytecodeBlocksFunctionalTests(TestCase): + def test_eq(self): + # compare codes with multiple blocks and labels, + # Code.__eq__() renumbers labels to get equal labels + source = "x = 1 if test else 2" + code1 = disassemble(source) + code2 = disassemble(source) + self.assertEqual(code1, code2) + + # Type mismatch + self.assertFalse(code1 == 1) + + # argnames mismatch + cfg = ControlFlowGraph() + cfg.argnames = 10 + self.assertFalse(code1 == cfg) + + # instr mismatch + cfg = ControlFlowGraph() + cfg.argnames = code1.argnames + self.assertFalse(code1 == cfg) + + def check_getitem(self, code): + # check internal Code block indexes (index by index, index by label) + for block_index, block in enumerate(code): + self.assertIs(code[block_index], block) + self.assertIs(code[block], block) + self.assertEqual(code.get_block_index(block), block_index) + + def test_delitem(self): + cfg = ControlFlowGraph() + b = cfg.add_block() + del cfg[b] + self.assertEqual(len(cfg.get_instructions()), 0) + + def sample_code(self): + code = disassemble("x = 1", remove_last_return_none=True) + self.assertBlocksEqual( + code, [Instr("LOAD_CONST", 1, lineno=1), Instr("STORE_NAME", "x", lineno=1)] + ) + return code + + def test_split_block(self): + code = self.sample_code() + code[0].append(Instr("NOP", lineno=1)) + + label = code.split_block(code[0], 2) + self.assertIs(label, code[1]) + self.assertBlocksEqual( + code, + [Instr("LOAD_CONST", 1, lineno=1), Instr("STORE_NAME", "x", lineno=1)], + [Instr("NOP", lineno=1)], + ) + self.check_getitem(code) + + label2 = code.split_block(code[0], 1) + self.assertIs(label2, code[1]) + self.assertBlocksEqual( + code, + [Instr("LOAD_CONST", 1, lineno=1)], + [Instr("STORE_NAME", "x", lineno=1)], + [Instr("NOP", lineno=1)], + ) + self.check_getitem(code) + + with self.assertRaises(TypeError): + code.split_block(1, 1) + + with self.assertRaises(ValueError) as e: + code.split_block(code[0], -2) + self.assertIn("positive", e.exception.args[0]) + + def test_split_block_end(self): + code = self.sample_code() + + # split at the end of the last block requires to add a new empty block + label = code.split_block(code[0], 2) + self.assertIs(label, code[1]) + self.assertBlocksEqual( + code, + [Instr("LOAD_CONST", 1, lineno=1), Instr("STORE_NAME", "x", lineno=1)], + [], + ) + self.check_getitem(code) + + # split at the end of a block which is not the end doesn't require to + # add a new block + label = code.split_block(code[0], 2) + self.assertIs(label, code[1]) + self.assertBlocksEqual( + code, + [Instr("LOAD_CONST", 1, lineno=1), Instr("STORE_NAME", "x", lineno=1)], + [], + ) + + def test_split_block_dont_split(self): + code = self.sample_code() + + # FIXME: is it really useful to support that? + block = code.split_block(code[0], 0) + self.assertIs(block, code[0]) + self.assertBlocksEqual( + code, [Instr("LOAD_CONST", 1, lineno=1), Instr("STORE_NAME", "x", lineno=1)] + ) + + def test_split_block_error(self): + code = self.sample_code() + + with self.assertRaises(ValueError): + # invalid index + code.split_block(code[0], 3) + + def test_to_code(self): + # test resolution of jump labels + bytecode = ControlFlowGraph() + bytecode.first_lineno = 3 + bytecode.argcount = 3 + if sys.version_info > (3, 8): + bytecode.posonlyargcount = 0 + bytecode.kwonlyargcount = 2 + bytecode.name = "func" + bytecode.filename = "hello.py" + bytecode.flags = 0x43 + bytecode.argnames = ("arg", "arg2", "arg3", "kwonly", "kwonly2") + bytecode.docstring = None + block0 = bytecode[0] + block1 = bytecode.add_block() + block2 = bytecode.add_block() + block0.extend( + [ + Instr("LOAD_FAST", "x", lineno=4), + Instr("POP_JUMP_IF_FALSE", block2, lineno=4), + ] + ) + block1.extend( + [Instr("LOAD_FAST", "arg", lineno=5), Instr("STORE_FAST", "x", lineno=5)] + ) + block2.extend( + [ + Instr("LOAD_CONST", 3, lineno=6), + Instr("STORE_FAST", "x", lineno=6), + Instr("LOAD_FAST", "x", lineno=7), + Instr("RETURN_VALUE", lineno=7), + ] + ) + + if OFFSET_AS_INSTRUCTION: + # The argument of the jump is divided by 2 + expected = ( + b"|\x05" b"r\x04" b"|\x00" b"}\x05" b"d\x01" b"}\x05" b"|\x05" b"S\x00" + ) + else: + expected = ( + b"|\x05" b"r\x08" b"|\x00" b"}\x05" b"d\x01" b"}\x05" b"|\x05" b"S\x00" + ) + + code = bytecode.to_code() + self.assertEqual(code.co_consts, (None, 3)) + self.assertEqual(code.co_argcount, 3) + if sys.version_info > (3, 8): + self.assertEqual(code.co_posonlyargcount, 0) + self.assertEqual(code.co_kwonlyargcount, 2) + self.assertEqual(code.co_nlocals, 6) + self.assertEqual(code.co_stacksize, 1) + # FIXME: don't use hardcoded constants + self.assertEqual(code.co_flags, 0x43) + self.assertEqual(code.co_code, expected) + self.assertEqual(code.co_names, ()) + self.assertEqual( + code.co_varnames, ("arg", "arg2", "arg3", "kwonly", "kwonly2", "x") + ) + self.assertEqual(code.co_filename, "hello.py") + self.assertEqual(code.co_name, "func") + self.assertEqual(code.co_firstlineno, 3) + + # verify stacksize argument is honored + explicit_stacksize = code.co_stacksize + 42 + code = bytecode.to_code(stacksize=explicit_stacksize) + self.assertEqual(code.co_stacksize, explicit_stacksize) + + def test_get_block_index(self): + blocks = ControlFlowGraph() + block0 = blocks[0] + block1 = blocks.add_block() + block2 = blocks.add_block() + self.assertEqual(blocks.get_block_index(block0), 0) + self.assertEqual(blocks.get_block_index(block1), 1) + self.assertEqual(blocks.get_block_index(block2), 2) + + other_block = BasicBlock() + self.assertRaises(ValueError, blocks.get_block_index, other_block) + + +class CFGStacksizeComputationTests(TestCase): + def check_stack_size(self, func): + code = func.__code__ + bytecode = Bytecode.from_code(code) + cfg = ControlFlowGraph.from_bytecode(bytecode) + self.assertEqual(code.co_stacksize, cfg.compute_stacksize()) + + def test_empty_code(self): + cfg = ControlFlowGraph() + del cfg[0] + self.assertEqual(cfg.compute_stacksize(), 0) + + def test_handling_of_set_lineno(self): + code = Bytecode() + code.first_lineno = 3 + code.extend( + [ + Instr("LOAD_CONST", 7), + Instr("STORE_NAME", "x"), + SetLineno(4), + Instr("LOAD_CONST", 8), + Instr("STORE_NAME", "y"), + SetLineno(5), + Instr("LOAD_CONST", 9), + Instr("STORE_NAME", "z"), + ] + ) + self.assertEqual(code.compute_stacksize(), 1) + + def test_invalid_stacksize(self): + code = Bytecode() + code.extend([Instr("STORE_NAME", "x")]) + with self.assertRaises(RuntimeError): + code.compute_stacksize() + + def test_stack_size_computation_and(self): + def test(arg1, *args, **kwargs): # pragma: no cover + return arg1 and args # Test JUMP_IF_FALSE_OR_POP + + self.check_stack_size(test) + + def test_stack_size_computation_or(self): + def test(arg1, *args, **kwargs): # pragma: no cover + return arg1 or args # Test JUMP_IF_TRUE_OR_POP + + self.check_stack_size(test) + + def test_stack_size_computation_if_else(self): + def test(arg1, *args, **kwargs): # pragma: no cover + if args: + return 0 + elif kwargs: + return 1 + else: + return 2 + + self.check_stack_size(test) + + def test_stack_size_computation_for_loop_continue(self): + def test(arg1, *args, **kwargs): # pragma: no cover + for k in kwargs: + if k in args: + continue + else: + return 1 + + self.check_stack_size(test) + + def test_stack_size_computation_while_loop_break(self): + def test(arg1, *args, **kwargs): # pragma: no cover + while True: + if arg1: + break + + self.check_stack_size(test) + + def test_stack_size_computation_with(self): + def test(arg1, *args, **kwargs): # pragma: no cover + with open(arg1) as f: + return f.read() + + self.check_stack_size(test) + + def test_stack_size_computation_try_except(self): + def test(arg1, *args, **kwargs): # pragma: no cover + try: + return args[0] + except Exception: + return 2 + + self.check_stack_size(test) + + def test_stack_size_computation_try_finally(self): + def test(arg1, *args, **kwargs): # pragma: no cover + try: + return args[0] + finally: + return 2 + + self.check_stack_size(test) + + def test_stack_size_computation_try_except_finally(self): + def test(arg1, *args, **kwargs): # pragma: no cover + try: + return args[0] + except Exception: + return 2 + finally: + print("Interrupt") + + self.check_stack_size(test) + + def test_stack_size_computation_try_except_else_finally(self): + def test(arg1, *args, **kwargs): # pragma: no cover + try: + return args[0] + except Exception: + return 2 + else: + return arg1 + finally: + print("Interrupt") + + self.check_stack_size(test) + + def test_stack_size_computation_nested_try_except_finally(self): + def test(arg1, *args, **kwargs): # pragma: no cover + k = 1 + try: + getattr(arg1, k) + except AttributeError: + pass + except Exception: + try: + assert False + except Exception: + return 2 + finally: + print("unexpected") + finally: + print("attempted to get {}".format(k)) + + self.check_stack_size(test) + + def test_stack_size_computation_nested_try_except_else_finally(self): + def test(*args, **kwargs): + try: + v = args[1] + except IndexError: + try: + w = kwargs["value"] + except KeyError: + return -1 + else: + return w + finally: + print("second finally") + else: + return v + finally: + print("first finally") + + # A direct comparison of the stack depth fails because CPython + # generate dead code that is used in stack computation. + cpython_stacksize = test.__code__.co_stacksize + test.__code__ = Bytecode.from_code(test.__code__).to_code() + self.assertLessEqual(test.__code__.co_stacksize, cpython_stacksize) + with contextlib.redirect_stdout(io.StringIO()) as stdout: + self.assertEqual(test(1, 4), 4) + self.assertEqual(stdout.getvalue(), "first finally\n") + + with contextlib.redirect_stdout(io.StringIO()) as stdout: + self.assertEqual(test([], value=3), 3) + self.assertEqual(stdout.getvalue(), "second finally\nfirst finally\n") + + with contextlib.redirect_stdout(io.StringIO()) as stdout: + self.assertEqual(test([], name=None), -1) + self.assertEqual(stdout.getvalue(), "second finally\nfirst finally\n") + + def test_stack_size_with_dead_code(self): + # Simply demonstrate more directly the previously mentioned issue. + def test(*args): # pragma: no cover + return 0 + try: + a = args[0] + except IndexError: + return -1 + else: + return a + + test.__code__ = Bytecode.from_code(test.__code__).to_code() + self.assertEqual(test.__code__.co_stacksize, 1) + self.assertEqual(test(1), 0) + + def test_huge_code_with_numerous_blocks(self): + def base_func(x): + pass + + def mk_if_then_else(depth): + instructions = [] + for i in range(depth): + label_else = Label() + instructions.extend( + [ + Instr("LOAD_FAST", "x"), + Instr("POP_JUMP_IF_FALSE", label_else), + Instr("LOAD_GLOBAL", "f{}".format(i)), + Instr("RETURN_VALUE"), + label_else, + ] + ) + instructions.extend([Instr("LOAD_CONST", None), Instr("RETURN_VALUE")]) + return instructions + + bytecode = Bytecode(mk_if_then_else(5000)) + bytecode.compute_stacksize() + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/_pydevd_frame_eval/vendored/bytecode/tests/test_code.py b/_pydevd_frame_eval/vendored/bytecode/tests/test_code.py new file mode 100644 index 000000000..6938bd1bf --- /dev/null +++ b/_pydevd_frame_eval/vendored/bytecode/tests/test_code.py @@ -0,0 +1,93 @@ + +import pytest +from tests_python.debugger_unittest import IS_PY36_OR_GREATER, IS_CPYTHON +from tests_python.debug_constants import TEST_CYTHON +pytestmark = pytest.mark.skipif(not IS_PY36_OR_GREATER or not IS_CPYTHON or not TEST_CYTHON, reason='Requires CPython >= 3.6') +import unittest + +from _pydevd_frame_eval.vendored.bytecode import ConcreteBytecode, Bytecode, ControlFlowGraph +from _pydevd_frame_eval.vendored.bytecode.tests import get_code + + +class CodeTests(unittest.TestCase): + """Check that bytecode.from_code(code).to_code() returns code.""" + + def check(self, source, function=False): + ref_code = get_code(source, function=function) + + code = ConcreteBytecode.from_code(ref_code).to_code() + self.assertEqual(code, ref_code) + + code = Bytecode.from_code(ref_code).to_code() + self.assertEqual(code, ref_code) + + bytecode = Bytecode.from_code(ref_code) + blocks = ControlFlowGraph.from_bytecode(bytecode) + code = blocks.to_bytecode().to_code() + self.assertEqual(code, ref_code) + + def test_loop(self): + self.check( + """ + for x in range(1, 10): + x += 1 + if x == 3: + continue + x -= 1 + if x > 7: + break + x = 0 + print(x) + """ + ) + + def test_varargs(self): + self.check( + """ + def func(a, b, *varargs): + pass + """, + function=True, + ) + + def test_kwargs(self): + self.check( + """ + def func(a, b, **kwargs): + pass + """, + function=True, + ) + + def test_kwonlyargs(self): + self.check( + """ + def func(*, arg, arg2): + pass + """, + function=True, + ) + + # Added because Python 3.10 added some special beahavior with respect to + # generators in term of stack size + def test_generator_func(self): + self.check( + """ + def func(arg, arg2): + yield + """, + function=True, + ) + + def test_async_func(self): + self.check( + """ + async def func(arg, arg2): + pass + """, + function=True, + ) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/_pydevd_frame_eval/vendored/bytecode/tests/test_concrete.py b/_pydevd_frame_eval/vendored/bytecode/tests/test_concrete.py new file mode 100644 index 000000000..510f6cd55 --- /dev/null +++ b/_pydevd_frame_eval/vendored/bytecode/tests/test_concrete.py @@ -0,0 +1,1513 @@ + +import pytest +from tests_python.debugger_unittest import IS_PY36_OR_GREATER, IS_CPYTHON +from tests_python.debug_constants import TEST_CYTHON +pytestmark = pytest.mark.skipif(not IS_PY36_OR_GREATER or not IS_CPYTHON or not TEST_CYTHON, reason='Requires CPython >= 3.6') +#!/usr/bin/env python3 +import opcode +import sys +import textwrap +import types +import unittest + +from _pydevd_frame_eval.vendored.bytecode import ( + UNSET, + Label, + Instr, + SetLineno, + Bytecode, + CellVar, + FreeVar, + CompilerFlags, + ConcreteInstr, + ConcreteBytecode, +) +from _pydevd_frame_eval.vendored.bytecode.concrete import OFFSET_AS_INSTRUCTION +from _pydevd_frame_eval.vendored.bytecode.tests import get_code, TestCase + + +class ConcreteInstrTests(TestCase): + def test_constructor(self): + with self.assertRaises(ValueError): + # need an argument + ConcreteInstr("LOAD_CONST") + with self.assertRaises(ValueError): + # must not have an argument + ConcreteInstr("ROT_TWO", 33) + + # invalid argument + with self.assertRaises(TypeError): + ConcreteInstr("LOAD_CONST", 1.0) + with self.assertRaises(ValueError): + ConcreteInstr("LOAD_CONST", -1) + with self.assertRaises(TypeError): + ConcreteInstr("LOAD_CONST", 5, lineno=1.0) + with self.assertRaises(ValueError): + ConcreteInstr("LOAD_CONST", 5, lineno=-1) + + # test maximum argument + with self.assertRaises(ValueError): + ConcreteInstr("LOAD_CONST", 2147483647 + 1) + instr = ConcreteInstr("LOAD_CONST", 2147483647) + self.assertEqual(instr.arg, 2147483647) + + # test meaningless extended args + instr = ConcreteInstr("LOAD_FAST", 8, lineno=3, extended_args=1) + self.assertEqual(instr.name, "LOAD_FAST") + self.assertEqual(instr.arg, 8) + self.assertEqual(instr.lineno, 3) + self.assertEqual(instr.size, 4) + + def test_attr(self): + instr = ConcreteInstr("LOAD_CONST", 5, lineno=12) + self.assertEqual(instr.name, "LOAD_CONST") + self.assertEqual(instr.opcode, 100) + self.assertEqual(instr.arg, 5) + self.assertEqual(instr.lineno, 12) + self.assertEqual(instr.size, 2) + + def test_set(self): + instr = ConcreteInstr("LOAD_CONST", 5, lineno=3) + + instr.set("NOP") + self.assertEqual(instr.name, "NOP") + self.assertIs(instr.arg, UNSET) + self.assertEqual(instr.lineno, 3) + + instr.set("LOAD_FAST", 8) + self.assertEqual(instr.name, "LOAD_FAST") + self.assertEqual(instr.arg, 8) + self.assertEqual(instr.lineno, 3) + + # invalid + with self.assertRaises(ValueError): + instr.set("LOAD_CONST") + with self.assertRaises(ValueError): + instr.set("NOP", 5) + + def test_set_attr(self): + instr = ConcreteInstr("LOAD_CONST", 5, lineno=12) + + # operator name + instr.name = "LOAD_FAST" + self.assertEqual(instr.name, "LOAD_FAST") + self.assertEqual(instr.opcode, 124) + self.assertRaises(TypeError, setattr, instr, "name", 3) + self.assertRaises(ValueError, setattr, instr, "name", "xxx") + + # operator code + instr.opcode = 100 + self.assertEqual(instr.name, "LOAD_CONST") + self.assertEqual(instr.opcode, 100) + self.assertRaises(ValueError, setattr, instr, "opcode", -12) + self.assertRaises(TypeError, setattr, instr, "opcode", "abc") + + # extended argument + instr.arg = 0x1234ABCD + self.assertEqual(instr.arg, 0x1234ABCD) + self.assertEqual(instr.size, 8) + + # small argument + instr.arg = 0 + self.assertEqual(instr.arg, 0) + self.assertEqual(instr.size, 2) + + # invalid argument + self.assertRaises(ValueError, setattr, instr, "arg", -1) + self.assertRaises(ValueError, setattr, instr, "arg", 2147483647 + 1) + + # size attribute is read-only + self.assertRaises(AttributeError, setattr, instr, "size", 3) + + # lineno + instr.lineno = 33 + self.assertEqual(instr.lineno, 33) + self.assertRaises(TypeError, setattr, instr, "lineno", 1.0) + self.assertRaises(ValueError, setattr, instr, "lineno", -1) + + def test_size(self): + self.assertEqual(ConcreteInstr("ROT_TWO").size, 2) + self.assertEqual(ConcreteInstr("LOAD_CONST", 3).size, 2) + self.assertEqual(ConcreteInstr("LOAD_CONST", 0x1234ABCD).size, 8) + + def test_disassemble(self): + code = b"\t\x00d\x03" + instr = ConcreteInstr.disassemble(1, code, 0) + self.assertEqual(instr, ConcreteInstr("NOP", lineno=1)) + + instr = ConcreteInstr.disassemble(2, code, 1 if OFFSET_AS_INSTRUCTION else 2) + self.assertEqual(instr, ConcreteInstr("LOAD_CONST", 3, lineno=2)) + + code = b"\x90\x12\x904\x90\xabd\xcd" + + instr = ConcreteInstr.disassemble(3, code, 0) + self.assertEqual(instr, ConcreteInstr("EXTENDED_ARG", 0x12, lineno=3)) + + def test_assemble(self): + instr = ConcreteInstr("NOP") + self.assertEqual(instr.assemble(), b"\t\x00") + + instr = ConcreteInstr("LOAD_CONST", 3) + self.assertEqual(instr.assemble(), b"d\x03") + + instr = ConcreteInstr("LOAD_CONST", 0x1234ABCD) + self.assertEqual( + instr.assemble(), + (b"\x90\x12\x904\x90\xabd\xcd"), + ) + + instr = ConcreteInstr("LOAD_CONST", 3, extended_args=1) + self.assertEqual( + instr.assemble(), + (b"\x90\x00d\x03"), + ) + + def test_get_jump_target(self): + jump_abs = ConcreteInstr("JUMP_ABSOLUTE", 3) + self.assertEqual(jump_abs.get_jump_target(100), 3) + + jump_forward = ConcreteInstr("JUMP_FORWARD", 5) + self.assertEqual( + jump_forward.get_jump_target(10), 16 if OFFSET_AS_INSTRUCTION else 17 + ) + + +class ConcreteBytecodeTests(TestCase): + def test_repr(self): + r = repr(ConcreteBytecode()) + self.assertIn("ConcreteBytecode", r) + self.assertIn("0", r) + + def test_eq(self): + code = ConcreteBytecode() + self.assertFalse(code == 1) + + for name, val in ( + ("names", ["a"]), + ("varnames", ["a"]), + ("consts", [1]), + ("argcount", 1), + ("kwonlyargcount", 2), + ("flags", CompilerFlags(CompilerFlags.GENERATOR)), + ("first_lineno", 10), + ("filename", "xxxx.py"), + ("name", "__x"), + ("docstring", "x-x-x"), + ("cellvars", [CellVar("x")]), + ("freevars", [FreeVar("x")]), + ): + c = ConcreteBytecode() + setattr(c, name, val) + # For obscure reasons using assertNotEqual here fail + self.assertFalse(code == c) + + if sys.version_info > (3, 8): + c = ConcreteBytecode() + c.posonlyargcount = 10 + self.assertFalse(code == c) + + c = ConcreteBytecode() + c.consts = [1] + code.consts = [1] + c.append(ConcreteInstr("LOAD_CONST", 0)) + self.assertFalse(code == c) + + def test_attr(self): + code_obj = get_code("x = 5") + code = ConcreteBytecode.from_code(code_obj) + self.assertEqual(code.consts, [5, None]) + self.assertEqual(code.names, ["x"]) + self.assertEqual(code.varnames, []) + self.assertEqual(code.freevars, []) + self.assertListEqual( + list(code), + [ + ConcreteInstr("LOAD_CONST", 0, lineno=1), + ConcreteInstr("STORE_NAME", 0, lineno=1), + ConcreteInstr("LOAD_CONST", 1, lineno=1), + ConcreteInstr("RETURN_VALUE", lineno=1), + ], + ) + # FIXME: test other attributes + + def test_invalid_types(self): + code = ConcreteBytecode() + code.append(Label()) + with self.assertRaises(ValueError): + list(code) + with self.assertRaises(ValueError): + code.legalize() + with self.assertRaises(ValueError): + ConcreteBytecode([Label()]) + + def test_to_code_lnotab(self): + + # We use an actual function for the simple case to + # ensure we get lnotab right + def f(): + # + # + x = 7 # noqa + y = 8 # noqa + z = 9 # noqa + + fl = f.__code__.co_firstlineno + concrete = ConcreteBytecode() + concrete.consts = [None, 7, 8, 9] + concrete.varnames = ["x", "y", "z"] + concrete.first_lineno = fl + concrete.extend( + [ + SetLineno(fl + 3), + ConcreteInstr("LOAD_CONST", 1), + ConcreteInstr("STORE_FAST", 0), + SetLineno(fl + 4), + ConcreteInstr("LOAD_CONST", 2), + ConcreteInstr("STORE_FAST", 1), + SetLineno(fl + 5), + ConcreteInstr("LOAD_CONST", 3), + ConcreteInstr("STORE_FAST", 2), + ConcreteInstr("LOAD_CONST", 0), + ConcreteInstr("RETURN_VALUE"), + ] + ) + + code = concrete.to_code() + self.assertEqual(code.co_code, f.__code__.co_code) + self.assertEqual(code.co_lnotab, f.__code__.co_lnotab) + if sys.version_info >= (3, 10): + self.assertEqual(code.co_linetable, f.__code__.co_linetable) + + def test_negative_lnotab(self): + # x = 7 + # y = 8 + concrete = ConcreteBytecode( + [ + ConcreteInstr("LOAD_CONST", 0), + ConcreteInstr("STORE_NAME", 0), + # line number goes backward! + SetLineno(2), + ConcreteInstr("LOAD_CONST", 1), + ConcreteInstr("STORE_NAME", 1), + ] + ) + concrete.consts = [7, 8] + concrete.names = ["x", "y"] + concrete.first_lineno = 5 + + code = concrete.to_code() + expected = b"d\x00Z\x00d\x01Z\x01" + self.assertEqual(code.co_code, expected) + self.assertEqual(code.co_firstlineno, 5) + self.assertEqual(code.co_lnotab, b"\x04\xfd") + + def test_extended_lnotab(self): + # x = 7 + # 200 blank lines + # y = 8 + concrete = ConcreteBytecode( + [ + ConcreteInstr("LOAD_CONST", 0), + SetLineno(1 + 128), + ConcreteInstr("STORE_NAME", 0), + # line number goes backward! + SetLineno(1 + 129), + ConcreteInstr("LOAD_CONST", 1), + SetLineno(1), + ConcreteInstr("STORE_NAME", 1), + ] + ) + concrete.consts = [7, 8] + concrete.names = ["x", "y"] + concrete.first_lineno = 1 + + code = concrete.to_code() + expected = b"d\x00Z\x00d\x01Z\x01" + self.assertEqual(code.co_code, expected) + self.assertEqual(code.co_firstlineno, 1) + self.assertEqual(code.co_lnotab, b"\x02\x7f\x00\x01\x02\x01\x02\x80\x00\xff") + + def test_extended_lnotab2(self): + # x = 7 + # 200 blank lines + # y = 8 + base_code = compile("x = 7" + "\n" * 200 + "y = 8", "", "exec") + concrete = ConcreteBytecode( + [ + ConcreteInstr("LOAD_CONST", 0), + ConcreteInstr("STORE_NAME", 0), + SetLineno(201), + ConcreteInstr("LOAD_CONST", 1), + ConcreteInstr("STORE_NAME", 1), + ConcreteInstr("LOAD_CONST", 2), + ConcreteInstr("RETURN_VALUE"), + ] + ) + concrete.consts = [None, 7, 8] + concrete.names = ["x", "y"] + concrete.first_lineno = 1 + + code = concrete.to_code() + self.assertEqual(code.co_code, base_code.co_code) + self.assertEqual(code.co_firstlineno, base_code.co_firstlineno) + self.assertEqual(code.co_lnotab, base_code.co_lnotab) + if sys.version_info >= (3, 10): + self.assertEqual(code.co_linetable, base_code.co_linetable) + + def test_to_bytecode_consts(self): + # x = -0.0 + # x = +0.0 + # + # code optimized by the CPython 3.6 peephole optimizer which emits + # duplicated constants (0.0 is twice in consts). + code = ConcreteBytecode() + code.consts = [0.0, None, -0.0, 0.0] + code.names = ["x", "y"] + code.extend( + [ + ConcreteInstr("LOAD_CONST", 2, lineno=1), + ConcreteInstr("STORE_NAME", 0, lineno=1), + ConcreteInstr("LOAD_CONST", 3, lineno=2), + ConcreteInstr("STORE_NAME", 1, lineno=2), + ConcreteInstr("LOAD_CONST", 1, lineno=2), + ConcreteInstr("RETURN_VALUE", lineno=2), + ] + ) + + code = code.to_bytecode().to_concrete_bytecode() + # the conversion changes the constant order: the order comes from + # the order of LOAD_CONST instructions + self.assertEqual(code.consts, [-0.0, 0.0, None]) + code.names = ["x", "y"] + self.assertListEqual( + list(code), + [ + ConcreteInstr("LOAD_CONST", 0, lineno=1), + ConcreteInstr("STORE_NAME", 0, lineno=1), + ConcreteInstr("LOAD_CONST", 1, lineno=2), + ConcreteInstr("STORE_NAME", 1, lineno=2), + ConcreteInstr("LOAD_CONST", 2, lineno=2), + ConcreteInstr("RETURN_VALUE", lineno=2), + ], + ) + + def test_cellvar(self): + concrete = ConcreteBytecode() + concrete.cellvars = ["x"] + concrete.append(ConcreteInstr("LOAD_DEREF", 0)) + code = concrete.to_code() + + concrete = ConcreteBytecode.from_code(code) + self.assertEqual(concrete.cellvars, ["x"]) + self.assertEqual(concrete.freevars, []) + self.assertEqual(list(concrete), [ConcreteInstr("LOAD_DEREF", 0, lineno=1)]) + + bytecode = concrete.to_bytecode() + self.assertEqual(bytecode.cellvars, ["x"]) + self.assertEqual(list(bytecode), [Instr("LOAD_DEREF", CellVar("x"), lineno=1)]) + + def test_freevar(self): + concrete = ConcreteBytecode() + concrete.freevars = ["x"] + concrete.append(ConcreteInstr("LOAD_DEREF", 0)) + code = concrete.to_code() + + concrete = ConcreteBytecode.from_code(code) + self.assertEqual(concrete.cellvars, []) + self.assertEqual(concrete.freevars, ["x"]) + self.assertEqual(list(concrete), [ConcreteInstr("LOAD_DEREF", 0, lineno=1)]) + + bytecode = concrete.to_bytecode() + self.assertEqual(bytecode.cellvars, []) + self.assertEqual(list(bytecode), [Instr("LOAD_DEREF", FreeVar("x"), lineno=1)]) + + def test_cellvar_freevar(self): + concrete = ConcreteBytecode() + concrete.cellvars = ["cell"] + concrete.freevars = ["free"] + concrete.append(ConcreteInstr("LOAD_DEREF", 0)) + concrete.append(ConcreteInstr("LOAD_DEREF", 1)) + code = concrete.to_code() + + concrete = ConcreteBytecode.from_code(code) + self.assertEqual(concrete.cellvars, ["cell"]) + self.assertEqual(concrete.freevars, ["free"]) + self.assertEqual( + list(concrete), + [ + ConcreteInstr("LOAD_DEREF", 0, lineno=1), + ConcreteInstr("LOAD_DEREF", 1, lineno=1), + ], + ) + + bytecode = concrete.to_bytecode() + self.assertEqual(bytecode.cellvars, ["cell"]) + self.assertEqual( + list(bytecode), + [ + Instr("LOAD_DEREF", CellVar("cell"), lineno=1), + Instr("LOAD_DEREF", FreeVar("free"), lineno=1), + ], + ) + + def test_load_classderef(self): + concrete = ConcreteBytecode() + concrete.cellvars = ["__class__"] + concrete.freevars = ["__class__"] + concrete.extend( + [ConcreteInstr("LOAD_CLASSDEREF", 1), ConcreteInstr("STORE_DEREF", 1)] + ) + + bytecode = concrete.to_bytecode() + self.assertEqual(bytecode.freevars, ["__class__"]) + self.assertEqual(bytecode.cellvars, ["__class__"]) + self.assertEqual( + list(bytecode), + [ + Instr("LOAD_CLASSDEREF", FreeVar("__class__"), lineno=1), + Instr("STORE_DEREF", FreeVar("__class__"), lineno=1), + ], + ) + + concrete = bytecode.to_concrete_bytecode() + self.assertEqual(concrete.freevars, ["__class__"]) + self.assertEqual(concrete.cellvars, ["__class__"]) + self.assertEqual( + list(concrete), + [ + ConcreteInstr("LOAD_CLASSDEREF", 1, lineno=1), + ConcreteInstr("STORE_DEREF", 1, lineno=1), + ], + ) + + code = concrete.to_code() + self.assertEqual(code.co_freevars, ("__class__",)) + self.assertEqual(code.co_cellvars, ("__class__",)) + self.assertEqual( + code.co_code, + b"\x94\x01\x89\x01", + ) + + def test_explicit_stacksize(self): + # Passing stacksize=... to ConcreteBytecode.to_code should result in a + # code object with the specified stacksize. We pass some silly values + # and assert that they are honored. + code_obj = get_code("print('%s' % (a,b,c))") + original_stacksize = code_obj.co_stacksize + concrete = ConcreteBytecode.from_code(code_obj) + + # First with something bigger than necessary. + explicit_stacksize = original_stacksize + 42 + new_code_obj = concrete.to_code(stacksize=explicit_stacksize) + self.assertEqual(new_code_obj.co_stacksize, explicit_stacksize) + + # Then with something bogus. We probably don't want to advertise this + # in the documentation. If this fails then decide if it's for good + # reason, and remove if so. + explicit_stacksize = 0 + new_code_obj = concrete.to_code(stacksize=explicit_stacksize) + self.assertEqual(new_code_obj.co_stacksize, explicit_stacksize) + + def test_legalize(self): + concrete = ConcreteBytecode() + concrete.first_lineno = 3 + concrete.consts = [7, 8, 9] + concrete.names = ["x", "y", "z"] + concrete.extend( + [ + ConcreteInstr("LOAD_CONST", 0), + ConcreteInstr("STORE_NAME", 0), + ConcreteInstr("LOAD_CONST", 1, lineno=4), + ConcreteInstr("STORE_NAME", 1), + SetLineno(5), + ConcreteInstr("LOAD_CONST", 2, lineno=6), + ConcreteInstr("STORE_NAME", 2), + ] + ) + + concrete.legalize() + self.assertListEqual( + list(concrete), + [ + ConcreteInstr("LOAD_CONST", 0, lineno=3), + ConcreteInstr("STORE_NAME", 0, lineno=3), + ConcreteInstr("LOAD_CONST", 1, lineno=4), + ConcreteInstr("STORE_NAME", 1, lineno=4), + ConcreteInstr("LOAD_CONST", 2, lineno=5), + ConcreteInstr("STORE_NAME", 2, lineno=5), + ], + ) + + def test_slice(self): + concrete = ConcreteBytecode() + concrete.first_lineno = 3 + concrete.consts = [7, 8, 9] + concrete.names = ["x", "y", "z"] + concrete.extend( + [ + ConcreteInstr("LOAD_CONST", 0), + ConcreteInstr("STORE_NAME", 0), + SetLineno(4), + ConcreteInstr("LOAD_CONST", 1), + ConcreteInstr("STORE_NAME", 1), + SetLineno(5), + ConcreteInstr("LOAD_CONST", 2), + ConcreteInstr("STORE_NAME", 2), + ] + ) + self.assertEqual(concrete, concrete[:]) + + def test_copy(self): + concrete = ConcreteBytecode() + concrete.first_lineno = 3 + concrete.consts = [7, 8, 9] + concrete.names = ["x", "y", "z"] + concrete.extend( + [ + ConcreteInstr("LOAD_CONST", 0), + ConcreteInstr("STORE_NAME", 0), + SetLineno(4), + ConcreteInstr("LOAD_CONST", 1), + ConcreteInstr("STORE_NAME", 1), + SetLineno(5), + ConcreteInstr("LOAD_CONST", 2), + ConcreteInstr("STORE_NAME", 2), + ] + ) + self.assertEqual(concrete, concrete.copy()) + + +class ConcreteFromCodeTests(TestCase): + def test_extended_arg(self): + # Create a code object from arbitrary bytecode + co_code = b"\x90\x12\x904\x90\xabd\xcd" + code = get_code("x=1") + args = ( + (code.co_argcount,) + if sys.version_info < (3, 8) + else (code.co_argcount, code.co_posonlyargcount) + ) + args += ( + code.co_kwonlyargcount, + code.co_nlocals, + code.co_stacksize, + code.co_flags, + co_code, + code.co_consts, + code.co_names, + code.co_varnames, + code.co_filename, + code.co_name, + code.co_firstlineno, + code.co_linetable if sys.version_info >= (3, 10) else code.co_lnotab, + code.co_freevars, + code.co_cellvars, + ) + + code = types.CodeType(*args) + + # without EXTENDED_ARG opcode + bytecode = ConcreteBytecode.from_code(code) + self.assertListEqual( + list(bytecode), [ConcreteInstr("LOAD_CONST", 0x1234ABCD, lineno=1)] + ) + + # with EXTENDED_ARG opcode + bytecode = ConcreteBytecode.from_code(code, extended_arg=True) + expected = [ + ConcreteInstr("EXTENDED_ARG", 0x12, lineno=1), + ConcreteInstr("EXTENDED_ARG", 0x34, lineno=1), + ConcreteInstr("EXTENDED_ARG", 0xAB, lineno=1), + ConcreteInstr("LOAD_CONST", 0xCD, lineno=1), + ] + self.assertListEqual(list(bytecode), expected) + + def test_extended_arg_make_function(self): + if (3, 9) <= sys.version_info < (3, 10): + from _pydevd_frame_eval.vendored.bytecode.tests.util_annotation import get_code as get_code_future + + code_obj = get_code_future( + """ + def foo(x: int, y: int): + pass + """ + ) + else: + code_obj = get_code( + """ + def foo(x: int, y: int): + pass + """ + ) + + # without EXTENDED_ARG + concrete = ConcreteBytecode.from_code(code_obj) + if sys.version_info >= (3, 10): + func_code = concrete.consts[2] + names = ["int", "foo"] + consts = ["x", "y", func_code, "foo", None] + const_offset = 1 + name_offset = 1 + first_instrs = [ + ConcreteInstr("LOAD_CONST", 0, lineno=1), + ConcreteInstr("LOAD_NAME", 0, lineno=1), + ConcreteInstr("LOAD_CONST", 1, lineno=1), + ConcreteInstr("LOAD_NAME", 0, lineno=1), + ConcreteInstr("BUILD_TUPLE", 4, lineno=1), + ] + elif ( + sys.version_info >= (3, 7) + and concrete.flags & CompilerFlags.FUTURE_ANNOTATIONS + ): + func_code = concrete.consts[2] + names = ["foo"] + consts = ["int", ("x", "y"), func_code, "foo", None] + const_offset = 1 + name_offset = 0 + first_instrs = [ + ConcreteInstr("LOAD_CONST", 0, lineno=1), + ConcreteInstr("LOAD_CONST", 0, lineno=1), + ConcreteInstr("LOAD_CONST", 0 + const_offset, lineno=1), + ConcreteInstr("BUILD_CONST_KEY_MAP", 2, lineno=1), + ] + else: + func_code = concrete.consts[1] + names = ["int", "foo"] + consts = [("x", "y"), func_code, "foo", None] + const_offset = 0 + name_offset = 1 + first_instrs = [ + ConcreteInstr("LOAD_NAME", 0, lineno=1), + ConcreteInstr("LOAD_NAME", 0, lineno=1), + ConcreteInstr("LOAD_CONST", 0 + const_offset, lineno=1), + ConcreteInstr("BUILD_CONST_KEY_MAP", 2, lineno=1), + ] + + self.assertEqual(concrete.names, names) + self.assertEqual(concrete.consts, consts) + expected = first_instrs + [ + ConcreteInstr("LOAD_CONST", 1 + const_offset, lineno=1), + ConcreteInstr("LOAD_CONST", 2 + const_offset, lineno=1), + ConcreteInstr("MAKE_FUNCTION", 4, lineno=1), + ConcreteInstr("STORE_NAME", name_offset, lineno=1), + ConcreteInstr("LOAD_CONST", 3 + const_offset, lineno=1), + ConcreteInstr("RETURN_VALUE", lineno=1), + ] + self.assertListEqual(list(concrete), expected) + + # with EXTENDED_ARG + concrete = ConcreteBytecode.from_code(code_obj, extended_arg=True) + # With future annotation the int annotation is stringified and + # stored as constant this the default behavior under Python 3.10 + if sys.version_info >= (3, 10): + func_code = concrete.consts[2] + names = ["int", "foo"] + consts = ["x", "y", func_code, "foo", None] + elif concrete.flags & CompilerFlags.FUTURE_ANNOTATIONS: + func_code = concrete.consts[2] + names = ["foo"] + consts = ["int", ("x", "y"), func_code, "foo", None] + else: + func_code = concrete.consts[1] + names = ["int", "foo"] + consts = [("x", "y"), func_code, "foo", None] + + self.assertEqual(concrete.names, names) + self.assertEqual(concrete.consts, consts) + self.assertListEqual(list(concrete), expected) + + # The next three tests ensure we can round trip ConcreteBytecode generated + # with extended_args=True + + def test_extended_arg_unpack_ex(self): + def test(): + p = [1, 2, 3, 4, 5, 6] + q, r, *s, t = p + return q, r, s, t + + cpython_stacksize = test.__code__.co_stacksize + test.__code__ = ConcreteBytecode.from_code( + test.__code__, extended_arg=True + ).to_code() + self.assertEqual(test.__code__.co_stacksize, cpython_stacksize) + self.assertEqual(test(), (1, 2, [3, 4, 5], 6)) + + def test_expected_arg_with_many_consts(self): + def test(): + var = 0 + var = 1 + var = 2 + var = 3 + var = 4 + var = 5 + var = 6 + var = 7 + var = 8 + var = 9 + var = 10 + var = 11 + var = 12 + var = 13 + var = 14 + var = 15 + var = 16 + var = 17 + var = 18 + var = 19 + var = 20 + var = 21 + var = 22 + var = 23 + var = 24 + var = 25 + var = 26 + var = 27 + var = 28 + var = 29 + var = 30 + var = 31 + var = 32 + var = 33 + var = 34 + var = 35 + var = 36 + var = 37 + var = 38 + var = 39 + var = 40 + var = 41 + var = 42 + var = 43 + var = 44 + var = 45 + var = 46 + var = 47 + var = 48 + var = 49 + var = 50 + var = 51 + var = 52 + var = 53 + var = 54 + var = 55 + var = 56 + var = 57 + var = 58 + var = 59 + var = 60 + var = 61 + var = 62 + var = 63 + var = 64 + var = 65 + var = 66 + var = 67 + var = 68 + var = 69 + var = 70 + var = 71 + var = 72 + var = 73 + var = 74 + var = 75 + var = 76 + var = 77 + var = 78 + var = 79 + var = 80 + var = 81 + var = 82 + var = 83 + var = 84 + var = 85 + var = 86 + var = 87 + var = 88 + var = 89 + var = 90 + var = 91 + var = 92 + var = 93 + var = 94 + var = 95 + var = 96 + var = 97 + var = 98 + var = 99 + var = 100 + var = 101 + var = 102 + var = 103 + var = 104 + var = 105 + var = 106 + var = 107 + var = 108 + var = 109 + var = 110 + var = 111 + var = 112 + var = 113 + var = 114 + var = 115 + var = 116 + var = 117 + var = 118 + var = 119 + var = 120 + var = 121 + var = 122 + var = 123 + var = 124 + var = 125 + var = 126 + var = 127 + var = 128 + var = 129 + var = 130 + var = 131 + var = 132 + var = 133 + var = 134 + var = 135 + var = 136 + var = 137 + var = 138 + var = 139 + var = 140 + var = 141 + var = 142 + var = 143 + var = 144 + var = 145 + var = 146 + var = 147 + var = 148 + var = 149 + var = 150 + var = 151 + var = 152 + var = 153 + var = 154 + var = 155 + var = 156 + var = 157 + var = 158 + var = 159 + var = 160 + var = 161 + var = 162 + var = 163 + var = 164 + var = 165 + var = 166 + var = 167 + var = 168 + var = 169 + var = 170 + var = 171 + var = 172 + var = 173 + var = 174 + var = 175 + var = 176 + var = 177 + var = 178 + var = 179 + var = 180 + var = 181 + var = 182 + var = 183 + var = 184 + var = 185 + var = 186 + var = 187 + var = 188 + var = 189 + var = 190 + var = 191 + var = 192 + var = 193 + var = 194 + var = 195 + var = 196 + var = 197 + var = 198 + var = 199 + var = 200 + var = 201 + var = 202 + var = 203 + var = 204 + var = 205 + var = 206 + var = 207 + var = 208 + var = 209 + var = 210 + var = 211 + var = 212 + var = 213 + var = 214 + var = 215 + var = 216 + var = 217 + var = 218 + var = 219 + var = 220 + var = 221 + var = 222 + var = 223 + var = 224 + var = 225 + var = 226 + var = 227 + var = 228 + var = 229 + var = 230 + var = 231 + var = 232 + var = 233 + var = 234 + var = 235 + var = 236 + var = 237 + var = 238 + var = 239 + var = 240 + var = 241 + var = 242 + var = 243 + var = 244 + var = 245 + var = 246 + var = 247 + var = 248 + var = 249 + var = 250 + var = 251 + var = 252 + var = 253 + var = 254 + var = 255 + var = 256 + var = 257 + var = 258 + var = 259 + + return var + + test.__code__ = ConcreteBytecode.from_code( + test.__code__, extended_arg=True + ).to_code() + self.assertEqual(test.__code__.co_stacksize, 1) + self.assertEqual(test(), 259) + + if sys.version_info >= (3, 6): + + def test_fail_extended_arg_jump(self): + def test(): + var = None + for _ in range(0, 1): + var = 0 + var = 1 + var = 2 + var = 3 + var = 4 + var = 5 + var = 6 + var = 7 + var = 8 + var = 9 + var = 10 + var = 11 + var = 12 + var = 13 + var = 14 + var = 15 + var = 16 + var = 17 + var = 18 + var = 19 + var = 20 + var = 21 + var = 22 + var = 23 + var = 24 + var = 25 + var = 26 + var = 27 + var = 28 + var = 29 + var = 30 + var = 31 + var = 32 + var = 33 + var = 34 + var = 35 + var = 36 + var = 37 + var = 38 + var = 39 + var = 40 + var = 41 + var = 42 + var = 43 + var = 44 + var = 45 + var = 46 + var = 47 + var = 48 + var = 49 + var = 50 + var = 51 + var = 52 + var = 53 + var = 54 + var = 55 + var = 56 + var = 57 + var = 58 + var = 59 + var = 60 + var = 61 + var = 62 + var = 63 + var = 64 + var = 65 + var = 66 + var = 67 + var = 68 + var = 69 + var = 70 + return var + + # Generate the bytecode with extended arguments + bytecode = ConcreteBytecode.from_code(test.__code__, extended_arg=True) + bytecode.to_code() + + +class BytecodeToConcreteTests(TestCase): + def test_label(self): + code = Bytecode() + label = Label() + code.extend( + [ + Instr("LOAD_CONST", "hello", lineno=1), + Instr("JUMP_FORWARD", label, lineno=1), + label, + Instr("POP_TOP", lineno=1), + ] + ) + + code = code.to_concrete_bytecode() + expected = [ + ConcreteInstr("LOAD_CONST", 0, lineno=1), + ConcreteInstr("JUMP_FORWARD", 0, lineno=1), + ConcreteInstr("POP_TOP", lineno=1), + ] + self.assertListEqual(list(code), expected) + self.assertListEqual(code.consts, ["hello"]) + + def test_label2(self): + bytecode = Bytecode() + label = Label() + bytecode.extend( + [ + Instr("LOAD_NAME", "test", lineno=1), + Instr("POP_JUMP_IF_FALSE", label), + Instr("LOAD_CONST", 5, lineno=2), + Instr("STORE_NAME", "x"), + Instr("JUMP_FORWARD", label), + Instr("LOAD_CONST", 7, lineno=4), + Instr("STORE_NAME", "x"), + label, + Instr("LOAD_CONST", None), + Instr("RETURN_VALUE"), + ] + ) + + concrete = bytecode.to_concrete_bytecode() + expected = [ + ConcreteInstr("LOAD_NAME", 0, lineno=1), + ConcreteInstr( + "POP_JUMP_IF_FALSE", 7 if OFFSET_AS_INSTRUCTION else 14, lineno=1 + ), + ConcreteInstr("LOAD_CONST", 0, lineno=2), + ConcreteInstr("STORE_NAME", 1, lineno=2), + ConcreteInstr("JUMP_FORWARD", 2 if OFFSET_AS_INSTRUCTION else 4, lineno=2), + ConcreteInstr("LOAD_CONST", 1, lineno=4), + ConcreteInstr("STORE_NAME", 1, lineno=4), + ConcreteInstr("LOAD_CONST", 2, lineno=4), + ConcreteInstr("RETURN_VALUE", lineno=4), + ] + self.assertListEqual(list(concrete), expected) + self.assertListEqual(concrete.consts, [5, 7, None]) + self.assertListEqual(concrete.names, ["test", "x"]) + self.assertListEqual(concrete.varnames, []) + + def test_label3(self): + """ + CPython generates useless EXTENDED_ARG 0 in some cases. We need to + properly track them as otherwise we can end up with broken offset for + jumps. + """ + source = """ + def func(x): + if x == 1: + return x + 0 + elif x == 2: + return x + 1 + elif x == 3: + return x + 2 + elif x == 4: + return x + 3 + elif x == 5: + return x + 4 + elif x == 6: + return x + 5 + elif x == 7: + return x + 6 + elif x == 8: + return x + 7 + elif x == 9: + return x + 8 + elif x == 10: + return x + 9 + elif x == 11: + return x + 10 + elif x == 12: + return x + 11 + elif x == 13: + return x + 12 + elif x == 14: + return x + 13 + elif x == 15: + return x + 14 + elif x == 16: + return x + 15 + elif x == 17: + return x + 16 + return -1 + """ + code = get_code(source, function=True) + bcode = Bytecode.from_code(code) + concrete = bcode.to_concrete_bytecode() + self.assertIsInstance(concrete, ConcreteBytecode) + + # Ensure that we do not generate broken code + loc = {} + exec(textwrap.dedent(source), loc) + func = loc["func"] + func.__code__ = bcode.to_code() + for i, x in enumerate(range(1, 18)): + self.assertEqual(func(x), x + i) + self.assertEqual(func(18), -1) + + # Ensure that we properly round trip in such cases + self.assertEqual( + ConcreteBytecode.from_code(code).to_code().co_code, code.co_code + ) + + def test_setlineno(self): + # x = 7 + # y = 8 + # z = 9 + concrete = ConcreteBytecode() + concrete.consts = [7, 8, 9] + concrete.names = ["x", "y", "z"] + concrete.first_lineno = 3 + concrete.extend( + [ + ConcreteInstr("LOAD_CONST", 0), + ConcreteInstr("STORE_NAME", 0), + SetLineno(4), + ConcreteInstr("LOAD_CONST", 1), + ConcreteInstr("STORE_NAME", 1), + SetLineno(5), + ConcreteInstr("LOAD_CONST", 2), + ConcreteInstr("STORE_NAME", 2), + ] + ) + + code = concrete.to_bytecode() + self.assertEqual( + code, + [ + Instr("LOAD_CONST", 7, lineno=3), + Instr("STORE_NAME", "x", lineno=3), + Instr("LOAD_CONST", 8, lineno=4), + Instr("STORE_NAME", "y", lineno=4), + Instr("LOAD_CONST", 9, lineno=5), + Instr("STORE_NAME", "z", lineno=5), + ], + ) + + def test_extended_jump(self): + NOP = bytes((opcode.opmap["NOP"],)) + + class BigInstr(ConcreteInstr): + def __init__(self, size): + super().__init__("NOP") + self._size = size + + def copy(self): + return self + + def assemble(self): + return NOP * self._size + + # (invalid) code using jumps > 0xffff to test extended arg + label = Label() + nb_nop = 2 ** 16 + code = Bytecode( + [ + Instr("JUMP_ABSOLUTE", label), + BigInstr(nb_nop), + label, + Instr("LOAD_CONST", None), + Instr("RETURN_VALUE"), + ] + ) + + code_obj = code.to_code() + if OFFSET_AS_INSTRUCTION: + expected = b"\x90\x80q\x02" + NOP * nb_nop + b"d\x00S\x00" + else: + expected = b"\x90\x01\x90\x00q\x06" + NOP * nb_nop + b"d\x00S\x00" + self.assertEqual(code_obj.co_code, expected) + + def test_jumps(self): + # if test: + # x = 12 + # else: + # x = 37 + code = Bytecode() + label_else = Label() + label_return = Label() + code.extend( + [ + Instr("LOAD_NAME", "test", lineno=1), + Instr("POP_JUMP_IF_FALSE", label_else), + Instr("LOAD_CONST", 12, lineno=2), + Instr("STORE_NAME", "x"), + Instr("JUMP_FORWARD", label_return), + label_else, + Instr("LOAD_CONST", 37, lineno=4), + Instr("STORE_NAME", "x"), + label_return, + Instr("LOAD_CONST", None, lineno=4), + Instr("RETURN_VALUE"), + ] + ) + + code = code.to_concrete_bytecode() + expected = [ + ConcreteInstr("LOAD_NAME", 0, lineno=1), + ConcreteInstr( + "POP_JUMP_IF_FALSE", 5 if OFFSET_AS_INSTRUCTION else 10, lineno=1 + ), + ConcreteInstr("LOAD_CONST", 0, lineno=2), + ConcreteInstr("STORE_NAME", 1, lineno=2), + ConcreteInstr("JUMP_FORWARD", 2 if OFFSET_AS_INSTRUCTION else 4, lineno=2), + ConcreteInstr("LOAD_CONST", 1, lineno=4), + ConcreteInstr("STORE_NAME", 1, lineno=4), + ConcreteInstr("LOAD_CONST", 2, lineno=4), + ConcreteInstr("RETURN_VALUE", lineno=4), + ] + self.assertListEqual(list(code), expected) + self.assertListEqual(code.consts, [12, 37, None]) + self.assertListEqual(code.names, ["test", "x"]) + self.assertListEqual(code.varnames, []) + + def test_dont_merge_constants(self): + # test two constants which are equal but have a different type + code = Bytecode() + code.extend( + [ + Instr("LOAD_CONST", 5, lineno=1), + Instr("LOAD_CONST", 5.0, lineno=1), + Instr("LOAD_CONST", -0.0, lineno=1), + Instr("LOAD_CONST", +0.0, lineno=1), + ] + ) + + code = code.to_concrete_bytecode() + expected = [ + ConcreteInstr("LOAD_CONST", 0, lineno=1), + ConcreteInstr("LOAD_CONST", 1, lineno=1), + ConcreteInstr("LOAD_CONST", 2, lineno=1), + ConcreteInstr("LOAD_CONST", 3, lineno=1), + ] + self.assertListEqual(list(code), expected) + self.assertListEqual(code.consts, [5, 5.0, -0.0, +0.0]) + + def test_cellvars(self): + code = Bytecode() + code.cellvars = ["x"] + code.freevars = ["y"] + code.extend( + [ + Instr("LOAD_DEREF", CellVar("x"), lineno=1), + Instr("LOAD_DEREF", FreeVar("y"), lineno=1), + ] + ) + concrete = code.to_concrete_bytecode() + self.assertEqual(concrete.cellvars, ["x"]) + self.assertEqual(concrete.freevars, ["y"]) + code.extend( + [ + ConcreteInstr("LOAD_DEREF", 0, lineno=1), + ConcreteInstr("LOAD_DEREF", 1, lineno=1), + ] + ) + + def test_compute_jumps_convergence(self): + # Consider the following sequence of instructions: + # + # JUMP_ABSOLUTE Label1 + # JUMP_ABSOLUTE Label2 + # ...126 instructions... + # Label1: Offset 254 on first pass, 256 second pass + # NOP + # ... many more instructions ... + # Label2: Offset > 256 on first pass + # + # On first pass of compute_jumps(), Label2 will be at address 254, so + # that value encodes into the single byte arg of JUMP_ABSOLUTE. + # + # On second pass compute_jumps() the instr at Label1 will have offset + # of 256 so will also be given an EXTENDED_ARG. + # + # Thus we need to make an additional pass. This test only verifies + # case where 2 passes is insufficient but three is enough. + # + # On Python > 3.10 we need to double the number since the offset is now + # in term of instructions and not bytes. + + # Create code from comment above. + code = Bytecode() + label1 = Label() + label2 = Label() + nop = "NOP" + code.append(Instr("JUMP_ABSOLUTE", label1)) + code.append(Instr("JUMP_ABSOLUTE", label2)) + # Need 254 * 2 + 2 since the arg will change by 1 instruction rather than 2 + # bytes. + for x in range(4, 510 if OFFSET_AS_INSTRUCTION else 254, 2): + code.append(Instr(nop)) + code.append(label1) + code.append(Instr(nop)) + for x in range( + 514 if OFFSET_AS_INSTRUCTION else 256, + 600 if OFFSET_AS_INSTRUCTION else 300, + 2, + ): + code.append(Instr(nop)) + code.append(label2) + code.append(Instr(nop)) + + # This should pass by default. + code.to_code() + + # Try with max of two passes: it should raise + with self.assertRaises(RuntimeError): + code.to_code(compute_jumps_passes=2) + + def test_extreme_compute_jumps_convergence(self): + """Test of compute_jumps() requiring absurd number of passes. + + NOTE: This test also serves to demonstrate that there is no worst + case: the number of passes can be unlimited (or, actually, limited by + the size of the provided code). + + This is an extension of test_compute_jumps_convergence. Instead of + two jumps, where the earlier gets extended after the latter, we + instead generate a series of many jumps. Each pass of compute_jumps() + extends one more instruction, which in turn causes the one behind it + to be extended on the next pass. + + """ + + # N: the number of unextended instructions that can be squeezed into a + # set of bytes adressable by the arg of an unextended instruction. + # The answer is "128", but here's how we arrive at it. + max_unextended_offset = 1 << 8 + unextended_branch_instr_size = 2 + N = max_unextended_offset // unextended_branch_instr_size + + # When using instruction rather than bytes in the offset multiply by 2 + if OFFSET_AS_INSTRUCTION: + N *= 2 + + nop = "UNARY_POSITIVE" # don't use NOP, dis.stack_effect will raise + + # The number of jumps will be equal to the number of labels. The + # number of passes of compute_jumps() required will be one greater + # than this. + labels = [Label() for x in range(0, 3 * N)] + + code = Bytecode() + code.extend( + Instr("JUMP_FORWARD", labels[len(labels) - x - 1]) + for x in range(0, len(labels)) + ) + end_of_jumps = len(code) + code.extend(Instr(nop) for x in range(0, N)) + + # Now insert the labels. The first is N instructions (i.e. 256 + # bytes) after the last jump. Then they proceed to earlier positions + # 4 bytes at a time. While the targets are in the range of the nop + # instructions, 4 bytes is two instructions. When the targets are in + # the range of JUMP_FORWARD instructions we have to allow for the fact + # that the instructions will have been extended to four bytes each, so + # working backwards 4 bytes per label means just one instruction per + # label. + offset = end_of_jumps + N + for index in range(0, len(labels)): + code.insert(offset, labels[index]) + if offset <= end_of_jumps: + offset -= 1 + else: + offset -= 2 + + code.insert(0, Instr("LOAD_CONST", 0)) + del end_of_jumps + code.append(Instr("RETURN_VALUE")) + + code.to_code(compute_jumps_passes=(len(labels) + 1)) + + def test_general_constants(self): + """Test if general object could be linked as constants.""" + + class CustomObject: + pass + + class UnHashableCustomObject: + __hash__ = None + + obj1 = [1, 2, 3] + obj2 = {1, 2, 3} + obj3 = CustomObject() + obj4 = UnHashableCustomObject() + code = Bytecode( + [ + Instr("LOAD_CONST", obj1, lineno=1), + Instr("LOAD_CONST", obj2, lineno=1), + Instr("LOAD_CONST", obj3, lineno=1), + Instr("LOAD_CONST", obj4, lineno=1), + Instr("BUILD_TUPLE", 4, lineno=1), + Instr("RETURN_VALUE", lineno=1), + ] + ) + self.assertEqual(code.to_code().co_consts, (obj1, obj2, obj3, obj4)) + + def f(): + return # pragma: no cover + + f.__code__ = code.to_code() + self.assertEqual(f(), (obj1, obj2, obj3, obj4)) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/_pydevd_frame_eval/vendored/bytecode/tests/test_flags.py b/_pydevd_frame_eval/vendored/bytecode/tests/test_flags.py new file mode 100644 index 000000000..b7744fbd0 --- /dev/null +++ b/_pydevd_frame_eval/vendored/bytecode/tests/test_flags.py @@ -0,0 +1,159 @@ + +import pytest +from tests_python.debugger_unittest import IS_PY36_OR_GREATER, IS_CPYTHON +from tests_python.debug_constants import TEST_CYTHON +pytestmark = pytest.mark.skipif(not IS_PY36_OR_GREATER or not IS_CPYTHON or not TEST_CYTHON, reason='Requires CPython >= 3.6') +#!/usr/bin/env python3 +import unittest +from _pydevd_frame_eval.vendored.bytecode import ( + CompilerFlags, + ConcreteBytecode, + ConcreteInstr, + Bytecode, + ControlFlowGraph, +) +from _pydevd_frame_eval.vendored.bytecode.flags import infer_flags + + +class FlagsTests(unittest.TestCase): + def test_type_validation_on_inference(self): + with self.assertRaises(ValueError): + infer_flags(1) + + def test_flag_inference(self): + + # Check no loss of non-infered flags + code = ControlFlowGraph() + code.flags |= ( + CompilerFlags.NEWLOCALS + | CompilerFlags.VARARGS + | CompilerFlags.VARKEYWORDS + | CompilerFlags.NESTED + | CompilerFlags.FUTURE_GENERATOR_STOP + ) + code.update_flags() + for f in ( + CompilerFlags.NEWLOCALS, + CompilerFlags.VARARGS, + CompilerFlags.VARKEYWORDS, + CompilerFlags.NESTED, + CompilerFlags.NOFREE, + CompilerFlags.OPTIMIZED, + CompilerFlags.FUTURE_GENERATOR_STOP, + ): + self.assertTrue(bool(code.flags & f)) + + # Infer optimized and nofree + code = Bytecode() + flags = infer_flags(code) + self.assertTrue(bool(flags & CompilerFlags.OPTIMIZED)) + self.assertTrue(bool(flags & CompilerFlags.NOFREE)) + code.append(ConcreteInstr("STORE_NAME", 1)) + flags = infer_flags(code) + self.assertFalse(bool(flags & CompilerFlags.OPTIMIZED)) + self.assertTrue(bool(flags & CompilerFlags.NOFREE)) + code.append(ConcreteInstr("STORE_DEREF", 2)) + code.update_flags() + self.assertFalse(bool(code.flags & CompilerFlags.OPTIMIZED)) + self.assertFalse(bool(code.flags & CompilerFlags.NOFREE)) + + def test_async_gen_no_flag_is_async_None(self): + # Test inference in the absence of any flag set on the bytecode + + # Infer generator + code = ConcreteBytecode() + code.append(ConcreteInstr("YIELD_VALUE")) + code.update_flags() + self.assertTrue(bool(code.flags & CompilerFlags.GENERATOR)) + + # Infer coroutine + code = ConcreteBytecode() + code.append(ConcreteInstr("GET_AWAITABLE")) + code.update_flags() + self.assertTrue(bool(code.flags & CompilerFlags.COROUTINE)) + + # Infer coroutine or async generator + for i, expected in ( + ("YIELD_VALUE", CompilerFlags.ASYNC_GENERATOR), + ("YIELD_FROM", CompilerFlags.COROUTINE), + ): + code = ConcreteBytecode() + code.append(ConcreteInstr("GET_AWAITABLE")) + code.append(ConcreteInstr(i)) + code.update_flags() + self.assertTrue(bool(code.flags & expected)) + + def test_async_gen_no_flag_is_async_True(self): + # Test inference when we request an async function + + # Force coroutine + code = ConcreteBytecode() + code.update_flags(is_async=True) + self.assertTrue(bool(code.flags & CompilerFlags.COROUTINE)) + + # Infer coroutine or async generator + for i, expected in ( + ("YIELD_VALUE", CompilerFlags.ASYNC_GENERATOR), + ("YIELD_FROM", CompilerFlags.COROUTINE), + ): + code = ConcreteBytecode() + code.append(ConcreteInstr(i)) + code.update_flags(is_async=True) + self.assertTrue(bool(code.flags & expected)) + + def test_async_gen_no_flag_is_async_False(self): + # Test inference when we request a non-async function + + # Infer generator + code = ConcreteBytecode() + code.append(ConcreteInstr("YIELD_VALUE")) + code.flags = CompilerFlags(CompilerFlags.COROUTINE) + code.update_flags(is_async=False) + self.assertTrue(bool(code.flags & CompilerFlags.GENERATOR)) + + # Abort on coroutine + code = ConcreteBytecode() + code.append(ConcreteInstr("GET_AWAITABLE")) + code.flags = CompilerFlags(CompilerFlags.COROUTINE) + with self.assertRaises(ValueError): + code.update_flags(is_async=False) + + def test_async_gen_flags(self): + # Test inference in the presence of pre-existing flags + + for is_async in (None, True): + + # Infer generator + code = ConcreteBytecode() + code.append(ConcreteInstr("YIELD_VALUE")) + for f, expected in ( + (CompilerFlags.COROUTINE, CompilerFlags.ASYNC_GENERATOR), + (CompilerFlags.ASYNC_GENERATOR, CompilerFlags.ASYNC_GENERATOR), + (CompilerFlags.ITERABLE_COROUTINE, CompilerFlags.ITERABLE_COROUTINE), + ): + code.flags = CompilerFlags(f) + code.update_flags(is_async=is_async) + self.assertTrue(bool(code.flags & expected)) + + # Infer coroutine + code = ConcreteBytecode() + code.append(ConcreteInstr("YIELD_FROM")) + for f, expected in ( + (CompilerFlags.COROUTINE, CompilerFlags.COROUTINE), + (CompilerFlags.ASYNC_GENERATOR, CompilerFlags.COROUTINE), + (CompilerFlags.ITERABLE_COROUTINE, CompilerFlags.ITERABLE_COROUTINE), + ): + code.flags = CompilerFlags(f) + code.update_flags(is_async=is_async) + self.assertTrue(bool(code.flags & expected)) + + # Crash on ITERABLE_COROUTINE with async bytecode + code = ConcreteBytecode() + code.append(ConcreteInstr("GET_AWAITABLE")) + code.flags = CompilerFlags(CompilerFlags.ITERABLE_COROUTINE) + with self.assertRaises(ValueError): + code.update_flags(is_async=is_async) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/_pydevd_frame_eval/vendored/bytecode/tests/test_instr.py b/_pydevd_frame_eval/vendored/bytecode/tests/test_instr.py new file mode 100644 index 000000000..ca4a66a73 --- /dev/null +++ b/_pydevd_frame_eval/vendored/bytecode/tests/test_instr.py @@ -0,0 +1,362 @@ + +import pytest +from tests_python.debugger_unittest import IS_PY36_OR_GREATER, IS_CPYTHON +from tests_python.debug_constants import TEST_CYTHON +pytestmark = pytest.mark.skipif(not IS_PY36_OR_GREATER or not IS_CPYTHON or not TEST_CYTHON, reason='Requires CPython >= 3.6') +#!/usr/bin/env python3 +import opcode +import unittest +from _pydevd_frame_eval.vendored.bytecode import ( + UNSET, + Label, + Instr, + CellVar, + FreeVar, + BasicBlock, + SetLineno, + Compare, +) +from _pydevd_frame_eval.vendored.bytecode.tests import TestCase + + +class SetLinenoTests(TestCase): + def test_lineno(self): + lineno = SetLineno(1) + self.assertEqual(lineno.lineno, 1) + + def test_equality(self): + lineno = SetLineno(1) + self.assertNotEqual(lineno, 1) + self.assertEqual(lineno, SetLineno(1)) + self.assertNotEqual(lineno, SetLineno(2)) + + +class VariableTests(TestCase): + def test_str(self): + for cls in (CellVar, FreeVar): + var = cls("a") + self.assertEqual(str(var), "a") + + def test_repr(self): + for cls in (CellVar, FreeVar): + var = cls("_a_x_a_") + r = repr(var) + self.assertIn("_a_x_a_", r) + self.assertIn(cls.__name__, r) + + def test_eq(self): + f1 = FreeVar("a") + f2 = FreeVar("b") + c1 = CellVar("a") + c2 = CellVar("b") + + for v1, v2, eq in ( + (f1, f1, True), + (f1, f2, False), + (f1, c1, False), + (c1, c1, True), + (c1, c2, False), + ): + if eq: + self.assertEqual(v1, v2) + else: + self.assertNotEqual(v1, v2) + + +class InstrTests(TestCase): + def test_constructor(self): + # invalid line number + with self.assertRaises(TypeError): + Instr("NOP", lineno="x") + with self.assertRaises(ValueError): + Instr("NOP", lineno=0) + + # invalid name + with self.assertRaises(TypeError): + Instr(1) + with self.assertRaises(ValueError): + Instr("xxx") + + def test_repr(self): + + # No arg + r = repr(Instr("NOP", lineno=10)) + self.assertIn("NOP", r) + self.assertIn("10", r) + self.assertIn("lineno", r) + + # Arg + r = repr(Instr("LOAD_FAST", "_x_", lineno=10)) + self.assertIn("LOAD_FAST", r) + self.assertIn("lineno", r) + self.assertIn("10", r) + self.assertIn("arg", r) + self.assertIn("_x_", r) + + def test_invalid_arg(self): + label = Label() + block = BasicBlock() + + # EXTENDED_ARG + self.assertRaises(ValueError, Instr, "EXTENDED_ARG", 0) + + # has_jump() + self.assertRaises(TypeError, Instr, "JUMP_ABSOLUTE", 1) + self.assertRaises(TypeError, Instr, "JUMP_ABSOLUTE", 1.0) + Instr("JUMP_ABSOLUTE", label) + Instr("JUMP_ABSOLUTE", block) + + # hasfree + self.assertRaises(TypeError, Instr, "LOAD_DEREF", "x") + Instr("LOAD_DEREF", CellVar("x")) + Instr("LOAD_DEREF", FreeVar("x")) + + # haslocal + self.assertRaises(TypeError, Instr, "LOAD_FAST", 1) + Instr("LOAD_FAST", "x") + + # hasname + self.assertRaises(TypeError, Instr, "LOAD_NAME", 1) + Instr("LOAD_NAME", "x") + + # hasconst + self.assertRaises(ValueError, Instr, "LOAD_CONST") # UNSET + self.assertRaises(ValueError, Instr, "LOAD_CONST", label) + self.assertRaises(ValueError, Instr, "LOAD_CONST", block) + Instr("LOAD_CONST", 1.0) + Instr("LOAD_CONST", object()) + + # hascompare + self.assertRaises(TypeError, Instr, "COMPARE_OP", 1) + Instr("COMPARE_OP", Compare.EQ) + + # HAVE_ARGUMENT + self.assertRaises(ValueError, Instr, "CALL_FUNCTION", -1) + self.assertRaises(TypeError, Instr, "CALL_FUNCTION", 3.0) + Instr("CALL_FUNCTION", 3) + + # test maximum argument + self.assertRaises(ValueError, Instr, "CALL_FUNCTION", 2147483647 + 1) + instr = Instr("CALL_FUNCTION", 2147483647) + self.assertEqual(instr.arg, 2147483647) + + # not HAVE_ARGUMENT + self.assertRaises(ValueError, Instr, "NOP", 0) + Instr("NOP") + + def test_require_arg(self): + i = Instr("CALL_FUNCTION", 3) + self.assertTrue(i.require_arg()) + i = Instr("NOP") + self.assertFalse(i.require_arg()) + + def test_attr(self): + instr = Instr("LOAD_CONST", 3, lineno=5) + self.assertEqual(instr.name, "LOAD_CONST") + self.assertEqual(instr.opcode, 100) + self.assertEqual(instr.arg, 3) + self.assertEqual(instr.lineno, 5) + + # invalid values/types + self.assertRaises(ValueError, setattr, instr, "lineno", 0) + self.assertRaises(TypeError, setattr, instr, "lineno", 1.0) + self.assertRaises(TypeError, setattr, instr, "name", 5) + self.assertRaises(TypeError, setattr, instr, "opcode", 1.0) + self.assertRaises(ValueError, setattr, instr, "opcode", -1) + self.assertRaises(ValueError, setattr, instr, "opcode", 255) + + # arg can take any attribute but cannot be deleted + instr.arg = -8 + instr.arg = object() + self.assertRaises(AttributeError, delattr, instr, "arg") + + # no argument + instr = Instr("ROT_TWO") + self.assertIs(instr.arg, UNSET) + + def test_modify_op(self): + instr = Instr("LOAD_NAME", "x") + load_fast = opcode.opmap["LOAD_FAST"] + instr.opcode = load_fast + self.assertEqual(instr.name, "LOAD_FAST") + self.assertEqual(instr.opcode, load_fast) + + def test_extended_arg(self): + instr = Instr("LOAD_CONST", 0x1234ABCD) + self.assertEqual(instr.arg, 0x1234ABCD) + + def test_slots(self): + instr = Instr("NOP") + with self.assertRaises(AttributeError): + instr.myattr = 1 + + def test_compare(self): + instr = Instr("LOAD_CONST", 3, lineno=7) + self.assertEqual(instr, Instr("LOAD_CONST", 3, lineno=7)) + self.assertNotEqual(instr, 1) + + # different lineno + self.assertNotEqual(instr, Instr("LOAD_CONST", 3)) + self.assertNotEqual(instr, Instr("LOAD_CONST", 3, lineno=6)) + # different op + self.assertNotEqual(instr, Instr("LOAD_FAST", "x", lineno=7)) + # different arg + self.assertNotEqual(instr, Instr("LOAD_CONST", 4, lineno=7)) + + def test_has_jump(self): + label = Label() + jump = Instr("JUMP_ABSOLUTE", label) + self.assertTrue(jump.has_jump()) + + instr = Instr("LOAD_FAST", "x") + self.assertFalse(instr.has_jump()) + + def test_is_cond_jump(self): + label = Label() + jump = Instr("POP_JUMP_IF_TRUE", label) + self.assertTrue(jump.is_cond_jump()) + + instr = Instr("LOAD_FAST", "x") + self.assertFalse(instr.is_cond_jump()) + + def test_is_uncond_jump(self): + label = Label() + jump = Instr("JUMP_ABSOLUTE", label) + self.assertTrue(jump.is_uncond_jump()) + + instr = Instr("POP_JUMP_IF_TRUE", label) + self.assertFalse(instr.is_uncond_jump()) + + def test_const_key_not_equal(self): + def check(value): + self.assertEqual(Instr("LOAD_CONST", value), Instr("LOAD_CONST", value)) + + def func(): + pass + + check(None) + check(0) + check(0.0) + check(b"bytes") + check("text") + check(Ellipsis) + check((1, 2, 3)) + check(frozenset({1, 2, 3})) + check(func.__code__) + check(object()) + + def test_const_key_equal(self): + neg_zero = -0.0 + pos_zero = +0.0 + + # int and float: 0 == 0.0 + self.assertNotEqual(Instr("LOAD_CONST", 0), Instr("LOAD_CONST", 0.0)) + + # float: -0.0 == +0.0 + self.assertNotEqual( + Instr("LOAD_CONST", neg_zero), Instr("LOAD_CONST", pos_zero) + ) + + # complex + self.assertNotEqual( + Instr("LOAD_CONST", complex(neg_zero, 1.0)), + Instr("LOAD_CONST", complex(pos_zero, 1.0)), + ) + self.assertNotEqual( + Instr("LOAD_CONST", complex(1.0, neg_zero)), + Instr("LOAD_CONST", complex(1.0, pos_zero)), + ) + + # tuple + self.assertNotEqual(Instr("LOAD_CONST", (0,)), Instr("LOAD_CONST", (0.0,))) + nested_tuple1 = (0,) + nested_tuple1 = (nested_tuple1,) + nested_tuple2 = (0.0,) + nested_tuple2 = (nested_tuple2,) + self.assertNotEqual( + Instr("LOAD_CONST", nested_tuple1), Instr("LOAD_CONST", nested_tuple2) + ) + + # frozenset + self.assertNotEqual( + Instr("LOAD_CONST", frozenset({0})), Instr("LOAD_CONST", frozenset({0.0})) + ) + + def test_stack_effects(self): + # Verify all opcodes are handled and that "jump=None" really returns + # the max of the other cases. + from _pydevd_frame_eval.vendored.bytecode.concrete import ConcreteInstr + + def check(instr): + jump = instr.stack_effect(jump=True) + no_jump = instr.stack_effect(jump=False) + max_effect = instr.stack_effect(jump=None) + self.assertEqual(instr.stack_effect(), max_effect) + self.assertEqual(max_effect, max(jump, no_jump)) + + if not instr.has_jump(): + self.assertEqual(jump, no_jump) + + for name, op in opcode.opmap.items(): + with self.subTest(name): + # Use ConcreteInstr instead of Instr because it doesn't care + # what kind of argument it is constructed with. + if op < opcode.HAVE_ARGUMENT: + check(ConcreteInstr(name)) + else: + for arg in range(256): + check(ConcreteInstr(name, arg)) + + # LOAD_CONST uses a concrete python object as its oparg, however, in + # dis.stack_effect(opcode.opmap['LOAD_CONST'], oparg), + # oparg should be the index of that python object in the constants. + # + # Fortunately, for an instruction whose oparg isn't equivalent to its + # form in binary files(pyc format), the stack effect is a + # constant which does not depend on its oparg. + # + # The second argument of dis.stack_effect cannot be + # more than 2**31 - 1. If stack effect of an instruction is + # independent of its oparg, we pass 0 as the second argument + # of dis.stack_effect. + # (As a result we can calculate stack_effect for + # any LOAD_CONST instructions, even for large integers) + + for arg in 2 ** 31, 2 ** 32, 2 ** 63, 2 ** 64, -1: + self.assertEqual(Instr("LOAD_CONST", arg).stack_effect(), 1) + + def test_code_object_containing_mutable_data(self): + from _pydevd_frame_eval.vendored.bytecode import Bytecode, Instr + from types import CodeType + + def f(): + def g(): + return "value" + + return g + + f_code = Bytecode.from_code(f.__code__) + instr_load_code = None + mutable_datum = [4, 2] + + for each in f_code: + if ( + isinstance(each, Instr) + and each.name == "LOAD_CONST" + and isinstance(each.arg, CodeType) + ): + instr_load_code = each + break + + self.assertIsNotNone(instr_load_code) + + g_code = Bytecode.from_code(instr_load_code.arg) + g_code[0].arg = mutable_datum + instr_load_code.arg = g_code.to_code() + f.__code__ = f_code.to_code() + + self.assertIs(f()(), mutable_datum) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/_pydevd_frame_eval/vendored/bytecode/tests/test_misc.py b/_pydevd_frame_eval/vendored/bytecode/tests/test_misc.py new file mode 100644 index 000000000..5f4c0636a --- /dev/null +++ b/_pydevd_frame_eval/vendored/bytecode/tests/test_misc.py @@ -0,0 +1,270 @@ + +import pytest +from tests_python.debugger_unittest import IS_PY36_OR_GREATER, IS_CPYTHON +from tests_python.debug_constants import TEST_CYTHON +pytestmark = pytest.mark.skipif(not IS_PY36_OR_GREATER or not IS_CPYTHON or not TEST_CYTHON, reason='Requires CPython >= 3.6') +#!/usr/bin/env python3 +import contextlib +import io +import sys +import textwrap +import unittest + +from _pydevd_frame_eval.vendored import bytecode +from _pydevd_frame_eval.vendored.bytecode import Label, Instr, Bytecode, BasicBlock, ControlFlowGraph +from _pydevd_frame_eval.vendored.bytecode.concrete import OFFSET_AS_INSTRUCTION +from _pydevd_frame_eval.vendored.bytecode.tests import disassemble + + +class DumpCodeTests(unittest.TestCase): + maxDiff = 80 * 100 + + def check_dump_bytecode(self, code, expected, lineno=None): + with contextlib.redirect_stdout(io.StringIO()) as stderr: + if lineno is not None: + bytecode.dump_bytecode(code, lineno=True) + else: + bytecode.dump_bytecode(code) + output = stderr.getvalue() + + self.assertEqual(output, expected) + + def test_bytecode(self): + source = """ + def func(test): + if test == 1: + return 1 + elif test == 2: + return 2 + return 3 + """ + code = disassemble(source, function=True) + + # without line numbers + enum_repr = "" + expected = f""" + LOAD_FAST 'test' + LOAD_CONST 1 + COMPARE_OP {enum_repr} + POP_JUMP_IF_FALSE + LOAD_CONST 1 + RETURN_VALUE + +label_instr6: + LOAD_FAST 'test' + LOAD_CONST 2 + COMPARE_OP {enum_repr} + POP_JUMP_IF_FALSE + LOAD_CONST 2 + RETURN_VALUE + +label_instr13: + LOAD_CONST 3 + RETURN_VALUE + + """[ + 1: + ].rstrip( + " " + ) + self.check_dump_bytecode(code, expected) + + # with line numbers + expected = f""" + L. 2 0: LOAD_FAST 'test' + 1: LOAD_CONST 1 + 2: COMPARE_OP {enum_repr} + 3: POP_JUMP_IF_FALSE + L. 3 4: LOAD_CONST 1 + 5: RETURN_VALUE + +label_instr6: + L. 4 7: LOAD_FAST 'test' + 8: LOAD_CONST 2 + 9: COMPARE_OP {enum_repr} + 10: POP_JUMP_IF_FALSE + L. 5 11: LOAD_CONST 2 + 12: RETURN_VALUE + +label_instr13: + L. 6 14: LOAD_CONST 3 + 15: RETURN_VALUE + + """[ + 1: + ].rstrip( + " " + ) + self.check_dump_bytecode(code, expected, lineno=True) + + def test_bytecode_broken_label(self): + label = Label() + code = Bytecode([Instr("JUMP_ABSOLUTE", label)]) + + expected = " JUMP_ABSOLUTE \n\n" + self.check_dump_bytecode(code, expected) + + def test_blocks_broken_jump(self): + block = BasicBlock() + code = ControlFlowGraph() + code[0].append(Instr("JUMP_ABSOLUTE", block)) + + expected = textwrap.dedent( + """ + block1: + JUMP_ABSOLUTE + + """ + ).lstrip("\n") + self.check_dump_bytecode(code, expected) + + def test_bytecode_blocks(self): + source = """ + def func(test): + if test == 1: + return 1 + elif test == 2: + return 2 + return 3 + """ + code = disassemble(source, function=True) + code = ControlFlowGraph.from_bytecode(code) + + # without line numbers + enum_repr = "" + expected = textwrap.dedent( + f""" + block1: + LOAD_FAST 'test' + LOAD_CONST 1 + COMPARE_OP {enum_repr} + POP_JUMP_IF_FALSE + -> block2 + + block2: + LOAD_CONST 1 + RETURN_VALUE + + block3: + LOAD_FAST 'test' + LOAD_CONST 2 + COMPARE_OP {enum_repr} + POP_JUMP_IF_FALSE + -> block4 + + block4: + LOAD_CONST 2 + RETURN_VALUE + + block5: + LOAD_CONST 3 + RETURN_VALUE + + """ + ).lstrip() + self.check_dump_bytecode(code, expected) + + # with line numbers + expected = textwrap.dedent( + f""" + block1: + L. 2 0: LOAD_FAST 'test' + 1: LOAD_CONST 1 + 2: COMPARE_OP {enum_repr} + 3: POP_JUMP_IF_FALSE + -> block2 + + block2: + L. 3 0: LOAD_CONST 1 + 1: RETURN_VALUE + + block3: + L. 4 0: LOAD_FAST 'test' + 1: LOAD_CONST 2 + 2: COMPARE_OP {enum_repr} + 3: POP_JUMP_IF_FALSE + -> block4 + + block4: + L. 5 0: LOAD_CONST 2 + 1: RETURN_VALUE + + block5: + L. 6 0: LOAD_CONST 3 + 1: RETURN_VALUE + + """ + ).lstrip() + self.check_dump_bytecode(code, expected, lineno=True) + + def test_concrete_bytecode(self): + source = """ + def func(test): + if test == 1: + return 1 + elif test == 2: + return 2 + return 3 + """ + code = disassemble(source, function=True) + code = code.to_concrete_bytecode() + + # without line numbers + expected = f""" + 0 LOAD_FAST 0 + 2 LOAD_CONST 1 + 4 COMPARE_OP 2 + 6 POP_JUMP_IF_FALSE {6 if OFFSET_AS_INSTRUCTION else 12} + 8 LOAD_CONST 1 + 10 RETURN_VALUE + 12 LOAD_FAST 0 + 14 LOAD_CONST 2 + 16 COMPARE_OP 2 + 18 POP_JUMP_IF_FALSE {12 if OFFSET_AS_INSTRUCTION else 24} + 20 LOAD_CONST 2 + 22 RETURN_VALUE + 24 LOAD_CONST 3 + 26 RETURN_VALUE +""".lstrip( + "\n" + ) + self.check_dump_bytecode(code, expected) + + # with line numbers + expected = f""" +L. 2 0: LOAD_FAST 0 + 2: LOAD_CONST 1 + 4: COMPARE_OP 2 + 6: POP_JUMP_IF_FALSE {6 if OFFSET_AS_INSTRUCTION else 12} +L. 3 8: LOAD_CONST 1 + 10: RETURN_VALUE +L. 4 12: LOAD_FAST 0 + 14: LOAD_CONST 2 + 16: COMPARE_OP 2 + 18: POP_JUMP_IF_FALSE {12 if OFFSET_AS_INSTRUCTION else 24} +L. 5 20: LOAD_CONST 2 + 22: RETURN_VALUE +L. 6 24: LOAD_CONST 3 + 26: RETURN_VALUE +""".lstrip( + "\n" + ) + self.check_dump_bytecode(code, expected, lineno=True) + + def test_type_validation(self): + class T: + first_lineno = 1 + + with self.assertRaises(TypeError): + bytecode.dump_bytecode(T()) + + +class MiscTests(unittest.TestCase): + def skip_test_version(self): + import setup + + self.assertEqual(bytecode.__version__, setup.VERSION) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/_pydevd_frame_eval/vendored/bytecode/tests/test_peephole_opt.py b/_pydevd_frame_eval/vendored/bytecode/tests/test_peephole_opt.py new file mode 100644 index 000000000..387a7829f --- /dev/null +++ b/_pydevd_frame_eval/vendored/bytecode/tests/test_peephole_opt.py @@ -0,0 +1,985 @@ + +import pytest +from tests_python.debugger_unittest import IS_PY36_OR_GREATER, IS_CPYTHON +from tests_python.debug_constants import TEST_CYTHON +pytestmark = pytest.mark.skipif(not IS_PY36_OR_GREATER or not IS_CPYTHON or not TEST_CYTHON, reason='Requires CPython >= 3.6') +import sys +import unittest +from _pydevd_frame_eval.vendored.bytecode import Label, Instr, Compare, Bytecode, ControlFlowGraph +from _pydevd_frame_eval.vendored.bytecode import peephole_opt +from _pydevd_frame_eval.vendored.bytecode.tests import TestCase, dump_bytecode +from unittest import mock + + +class Tests(TestCase): + + maxDiff = 80 * 100 + + def optimize_blocks(self, code): + if isinstance(code, Bytecode): + code = ControlFlowGraph.from_bytecode(code) + optimizer = peephole_opt.PeepholeOptimizer() + optimizer.optimize_cfg(code) + return code + + def check(self, code, *expected): + if isinstance(code, Bytecode): + code = ControlFlowGraph.from_bytecode(code) + optimizer = peephole_opt.PeepholeOptimizer() + optimizer.optimize_cfg(code) + code = code.to_bytecode() + + try: + self.assertEqual(code, expected) + except AssertionError: + print("Optimized code:") + dump_bytecode(code) + + print("Expected code:") + for instr in expected: + print(instr) + + raise + + def check_dont_optimize(self, code): + code = ControlFlowGraph.from_bytecode(code) + noopt = code.to_bytecode() + + optim = self.optimize_blocks(code) + optim = optim.to_bytecode() + self.assertEqual(optim, noopt) + + def test_unary_op(self): + def check_unary_op(op, value, result): + code = Bytecode( + [Instr("LOAD_CONST", value), Instr(op), Instr("STORE_NAME", "x")] + ) + self.check(code, Instr("LOAD_CONST", result), Instr("STORE_NAME", "x")) + + check_unary_op("UNARY_POSITIVE", 2, 2) + check_unary_op("UNARY_NEGATIVE", 3, -3) + check_unary_op("UNARY_INVERT", 5, -6) + + def test_binary_op(self): + def check_bin_op(left, op, right, result): + code = Bytecode( + [ + Instr("LOAD_CONST", left), + Instr("LOAD_CONST", right), + Instr(op), + Instr("STORE_NAME", "x"), + ] + ) + self.check(code, Instr("LOAD_CONST", result), Instr("STORE_NAME", "x")) + + check_bin_op(10, "BINARY_ADD", 20, 30) + check_bin_op(5, "BINARY_SUBTRACT", 1, 4) + check_bin_op(5, "BINARY_MULTIPLY", 3, 15) + check_bin_op(10, "BINARY_TRUE_DIVIDE", 3, 10 / 3) + check_bin_op(10, "BINARY_FLOOR_DIVIDE", 3, 3) + check_bin_op(10, "BINARY_MODULO", 3, 1) + check_bin_op(2, "BINARY_POWER", 8, 256) + check_bin_op(1, "BINARY_LSHIFT", 3, 8) + check_bin_op(16, "BINARY_RSHIFT", 3, 2) + check_bin_op(10, "BINARY_AND", 3, 2) + check_bin_op(2, "BINARY_OR", 3, 3) + check_bin_op(2, "BINARY_XOR", 3, 1) + + def test_combined_unary_bin_ops(self): + # x = 1 + 3 + 7 + code = Bytecode( + [ + Instr("LOAD_CONST", 1), + Instr("LOAD_CONST", 3), + Instr("BINARY_ADD"), + Instr("LOAD_CONST", 7), + Instr("BINARY_ADD"), + Instr("STORE_NAME", "x"), + ] + ) + self.check(code, Instr("LOAD_CONST", 11), Instr("STORE_NAME", "x")) + + # x = ~(~(5)) + code = Bytecode( + [ + Instr("LOAD_CONST", 5), + Instr("UNARY_INVERT"), + Instr("UNARY_INVERT"), + Instr("STORE_NAME", "x"), + ] + ) + self.check(code, Instr("LOAD_CONST", 5), Instr("STORE_NAME", "x")) + + # "events = [(0, 'call'), (1, 'line'), (-(3), 'call')]" + code = Bytecode( + [ + Instr("LOAD_CONST", 0), + Instr("LOAD_CONST", "call"), + Instr("BUILD_TUPLE", 2), + Instr("LOAD_CONST", 1), + Instr("LOAD_CONST", "line"), + Instr("BUILD_TUPLE", 2), + Instr("LOAD_CONST", 3), + Instr("UNARY_NEGATIVE"), + Instr("LOAD_CONST", "call"), + Instr("BUILD_TUPLE", 2), + Instr("BUILD_LIST", 3), + Instr("STORE_NAME", "events"), + ] + ) + self.check( + code, + Instr("LOAD_CONST", (0, "call")), + Instr("LOAD_CONST", (1, "line")), + Instr("LOAD_CONST", (-3, "call")), + Instr("BUILD_LIST", 3), + Instr("STORE_NAME", "events"), + ) + + # 'x = (1,) + (0,) * 8' + code = Bytecode( + [ + Instr("LOAD_CONST", 1), + Instr("BUILD_TUPLE", 1), + Instr("LOAD_CONST", 0), + Instr("BUILD_TUPLE", 1), + Instr("LOAD_CONST", 8), + Instr("BINARY_MULTIPLY"), + Instr("BINARY_ADD"), + Instr("STORE_NAME", "x"), + ] + ) + zeros = (0,) * 8 + result = (1,) + zeros + self.check(code, Instr("LOAD_CONST", result), Instr("STORE_NAME", "x")) + + def test_max_size(self): + max_size = 3 + with mock.patch.object(peephole_opt, "MAX_SIZE", max_size): + # optimized binary operation: size <= maximum size + # + # (9,) * size + size = max_size + result = (9,) * size + code = Bytecode( + [ + Instr("LOAD_CONST", 9), + Instr("BUILD_TUPLE", 1), + Instr("LOAD_CONST", size), + Instr("BINARY_MULTIPLY"), + Instr("STORE_NAME", "x"), + ] + ) + self.check(code, Instr("LOAD_CONST", result), Instr("STORE_NAME", "x")) + + # don't optimize binary operation: size > maximum size + # + # x = (9,) * size + size = max_size + 1 + code = Bytecode( + [ + Instr("LOAD_CONST", 9), + Instr("BUILD_TUPLE", 1), + Instr("LOAD_CONST", size), + Instr("BINARY_MULTIPLY"), + Instr("STORE_NAME", "x"), + ] + ) + self.check( + code, + Instr("LOAD_CONST", (9,)), + Instr("LOAD_CONST", size), + Instr("BINARY_MULTIPLY"), + Instr("STORE_NAME", "x"), + ) + + def test_bin_op_dont_optimize(self): + # 1 / 0 + code = Bytecode( + [ + Instr("LOAD_CONST", 1), + Instr("LOAD_CONST", 0), + Instr("BINARY_TRUE_DIVIDE"), + Instr("POP_TOP"), + Instr("LOAD_CONST", None), + Instr("RETURN_VALUE"), + ] + ) + self.check_dont_optimize(code) + + # 1 // 0 + code = Bytecode( + [ + Instr("LOAD_CONST", 1), + Instr("LOAD_CONST", 0), + Instr("BINARY_FLOOR_DIVIDE"), + Instr("POP_TOP"), + Instr("LOAD_CONST", None), + Instr("RETURN_VALUE"), + ] + ) + self.check_dont_optimize(code) + + # 1 % 0 + code = Bytecode( + [ + Instr("LOAD_CONST", 1), + Instr("LOAD_CONST", 0), + Instr("BINARY_MODULO"), + Instr("POP_TOP"), + Instr("LOAD_CONST", None), + Instr("RETURN_VALUE"), + ] + ) + self.check_dont_optimize(code) + + # 1 % 1j + code = Bytecode( + [ + Instr("LOAD_CONST", 1), + Instr("LOAD_CONST", 1j), + Instr("BINARY_MODULO"), + Instr("POP_TOP"), + Instr("LOAD_CONST", None), + Instr("RETURN_VALUE"), + ] + ) + self.check_dont_optimize(code) + + def test_build_tuple(self): + # x = (1, 2, 3) + code = Bytecode( + [ + Instr("LOAD_CONST", 1), + Instr("LOAD_CONST", 2), + Instr("LOAD_CONST", 3), + Instr("BUILD_TUPLE", 3), + Instr("STORE_NAME", "x"), + ] + ) + self.check(code, Instr("LOAD_CONST", (1, 2, 3)), Instr("STORE_NAME", "x")) + + def test_build_list(self): + # test = x in [1, 2, 3] + code = Bytecode( + [ + Instr("LOAD_NAME", "x"), + Instr("LOAD_CONST", 1), + Instr("LOAD_CONST", 2), + Instr("LOAD_CONST", 3), + Instr("BUILD_LIST", 3), + Instr("COMPARE_OP", Compare.IN), + Instr("STORE_NAME", "test"), + ] + ) + + self.check( + code, + Instr("LOAD_NAME", "x"), + Instr("LOAD_CONST", (1, 2, 3)), + Instr("COMPARE_OP", Compare.IN), + Instr("STORE_NAME", "test"), + ) + + def test_build_list_unpack_seq(self): + for build_list in ("BUILD_TUPLE", "BUILD_LIST"): + # x, = [a] + code = Bytecode( + [ + Instr("LOAD_NAME", "a"), + Instr(build_list, 1), + Instr("UNPACK_SEQUENCE", 1), + Instr("STORE_NAME", "x"), + ] + ) + self.check(code, Instr("LOAD_NAME", "a"), Instr("STORE_NAME", "x")) + + # x, y = [a, b] + code = Bytecode( + [ + Instr("LOAD_NAME", "a"), + Instr("LOAD_NAME", "b"), + Instr(build_list, 2), + Instr("UNPACK_SEQUENCE", 2), + Instr("STORE_NAME", "x"), + Instr("STORE_NAME", "y"), + ] + ) + self.check( + code, + Instr("LOAD_NAME", "a"), + Instr("LOAD_NAME", "b"), + Instr("ROT_TWO"), + Instr("STORE_NAME", "x"), + Instr("STORE_NAME", "y"), + ) + + # x, y, z = [a, b, c] + code = Bytecode( + [ + Instr("LOAD_NAME", "a"), + Instr("LOAD_NAME", "b"), + Instr("LOAD_NAME", "c"), + Instr(build_list, 3), + Instr("UNPACK_SEQUENCE", 3), + Instr("STORE_NAME", "x"), + Instr("STORE_NAME", "y"), + Instr("STORE_NAME", "z"), + ] + ) + self.check( + code, + Instr("LOAD_NAME", "a"), + Instr("LOAD_NAME", "b"), + Instr("LOAD_NAME", "c"), + Instr("ROT_THREE"), + Instr("ROT_TWO"), + Instr("STORE_NAME", "x"), + Instr("STORE_NAME", "y"), + Instr("STORE_NAME", "z"), + ) + + def test_build_tuple_unpack_seq_const(self): + # x, y = (3, 4) + code = Bytecode( + [ + Instr("LOAD_CONST", 3), + Instr("LOAD_CONST", 4), + Instr("BUILD_TUPLE", 2), + Instr("UNPACK_SEQUENCE", 2), + Instr("STORE_NAME", "x"), + Instr("STORE_NAME", "y"), + ] + ) + self.check( + code, + Instr("LOAD_CONST", (3, 4)), + Instr("UNPACK_SEQUENCE", 2), + Instr("STORE_NAME", "x"), + Instr("STORE_NAME", "y"), + ) + + def test_build_list_unpack_seq_const(self): + # x, y, z = [3, 4, 5] + code = Bytecode( + [ + Instr("LOAD_CONST", 3), + Instr("LOAD_CONST", 4), + Instr("LOAD_CONST", 5), + Instr("BUILD_LIST", 3), + Instr("UNPACK_SEQUENCE", 3), + Instr("STORE_NAME", "x"), + Instr("STORE_NAME", "y"), + Instr("STORE_NAME", "z"), + ] + ) + self.check( + code, + Instr("LOAD_CONST", 5), + Instr("LOAD_CONST", 4), + Instr("LOAD_CONST", 3), + Instr("STORE_NAME", "x"), + Instr("STORE_NAME", "y"), + Instr("STORE_NAME", "z"), + ) + + def test_build_set(self): + # test = x in {1, 2, 3} + code = Bytecode( + [ + Instr("LOAD_NAME", "x"), + Instr("LOAD_CONST", 1), + Instr("LOAD_CONST", 2), + Instr("LOAD_CONST", 3), + Instr("BUILD_SET", 3), + Instr("COMPARE_OP", Compare.IN), + Instr("STORE_NAME", "test"), + ] + ) + + self.check( + code, + Instr("LOAD_NAME", "x"), + Instr("LOAD_CONST", frozenset((1, 2, 3))), + Instr("COMPARE_OP", Compare.IN), + Instr("STORE_NAME", "test"), + ) + + def test_compare_op_unary_not(self): + for op, not_op in ( + (Compare.IN, Compare.NOT_IN), # in => not in + (Compare.NOT_IN, Compare.IN), # not in => in + (Compare.IS, Compare.IS_NOT), # is => is not + (Compare.IS_NOT, Compare.IS), # is not => is + ): + code = Bytecode( + [ + Instr("LOAD_NAME", "a"), + Instr("LOAD_NAME", "b"), + Instr("COMPARE_OP", op), + Instr("UNARY_NOT"), + Instr("STORE_NAME", "x"), + ] + ) + self.check( + code, + Instr("LOAD_NAME", "a"), + Instr("LOAD_NAME", "b"), + Instr("COMPARE_OP", not_op), + Instr("STORE_NAME", "x"), + ) + + # don't optimize: + # x = not (a and b is True) + label_instr5 = Label() + code = Bytecode( + [ + Instr("LOAD_NAME", "a"), + Instr("JUMP_IF_FALSE_OR_POP", label_instr5), + Instr("LOAD_NAME", "b"), + Instr("LOAD_CONST", True), + Instr("COMPARE_OP", Compare.IS), + label_instr5, + Instr("UNARY_NOT"), + Instr("STORE_NAME", "x"), + Instr("LOAD_CONST", None), + Instr("RETURN_VALUE"), + ] + ) + self.check_dont_optimize(code) + + def test_dont_optimize(self): + # x = 3 < 5 + code = Bytecode( + [ + Instr("LOAD_CONST", 3), + Instr("LOAD_CONST", 5), + Instr("COMPARE_OP", Compare.LT), + Instr("STORE_NAME", "x"), + Instr("LOAD_CONST", None), + Instr("RETURN_VALUE"), + ] + ) + self.check_dont_optimize(code) + + # x = (10, 20, 30)[1:] + code = Bytecode( + [ + Instr("LOAD_CONST", (10, 20, 30)), + Instr("LOAD_CONST", 1), + Instr("LOAD_CONST", None), + Instr("BUILD_SLICE", 2), + Instr("BINARY_SUBSCR"), + Instr("STORE_NAME", "x"), + ] + ) + self.check_dont_optimize(code) + + def test_optimize_code_obj(self): + # Test optimize() method with a code object + # + # x = 3 + 5 => x = 8 + noopt = Bytecode( + [ + Instr("LOAD_CONST", 3), + Instr("LOAD_CONST", 5), + Instr("BINARY_ADD"), + Instr("STORE_NAME", "x"), + Instr("LOAD_CONST", None), + Instr("RETURN_VALUE"), + ] + ) + noopt = noopt.to_code() + + optimizer = peephole_opt.PeepholeOptimizer() + optim = optimizer.optimize(noopt) + + code = Bytecode.from_code(optim) + self.assertEqual( + code, + [ + Instr("LOAD_CONST", 8, lineno=1), + Instr("STORE_NAME", "x", lineno=1), + Instr("LOAD_CONST", None, lineno=1), + Instr("RETURN_VALUE", lineno=1), + ], + ) + + def test_return_value(self): + # return+return: remove second return + # + # def func(): + # return 4 + # return 5 + code = Bytecode( + [ + Instr("LOAD_CONST", 4, lineno=2), + Instr("RETURN_VALUE", lineno=2), + Instr("LOAD_CONST", 5, lineno=3), + Instr("RETURN_VALUE", lineno=3), + ] + ) + code = ControlFlowGraph.from_bytecode(code) + self.check( + code, Instr("LOAD_CONST", 4, lineno=2), Instr("RETURN_VALUE", lineno=2) + ) + + # return+return + return+return: remove second and fourth return + # + # def func(): + # return 4 + # return 5 + # return 6 + # return 7 + code = Bytecode( + [ + Instr("LOAD_CONST", 4, lineno=2), + Instr("RETURN_VALUE", lineno=2), + Instr("LOAD_CONST", 5, lineno=3), + Instr("RETURN_VALUE", lineno=3), + Instr("LOAD_CONST", 6, lineno=4), + Instr("RETURN_VALUE", lineno=4), + Instr("LOAD_CONST", 7, lineno=5), + Instr("RETURN_VALUE", lineno=5), + ] + ) + code = ControlFlowGraph.from_bytecode(code) + self.check( + code, Instr("LOAD_CONST", 4, lineno=2), Instr("RETURN_VALUE", lineno=2) + ) + + # return + JUMP_ABSOLUTE: remove JUMP_ABSOLUTE + # while 1: + # return 7 + if sys.version_info < (3, 8): + setup_loop = Label() + return_label = Label() + code = Bytecode( + [ + setup_loop, + Instr("SETUP_LOOP", return_label, lineno=2), + Instr("LOAD_CONST", 7, lineno=3), + Instr("RETURN_VALUE", lineno=3), + Instr("JUMP_ABSOLUTE", setup_loop, lineno=3), + Instr("POP_BLOCK", lineno=3), + return_label, + Instr("LOAD_CONST", None, lineno=3), + Instr("RETURN_VALUE", lineno=3), + ] + ) + code = ControlFlowGraph.from_bytecode(code) + + end_loop = Label() + self.check( + code, + Instr("SETUP_LOOP", end_loop, lineno=2), + Instr("LOAD_CONST", 7, lineno=3), + Instr("RETURN_VALUE", lineno=3), + end_loop, + Instr("LOAD_CONST", None, lineno=3), + Instr("RETURN_VALUE", lineno=3), + ) + else: + setup_loop = Label() + return_label = Label() + code = Bytecode( + [ + setup_loop, + Instr("LOAD_CONST", 7, lineno=3), + Instr("RETURN_VALUE", lineno=3), + Instr("JUMP_ABSOLUTE", setup_loop, lineno=3), + Instr("LOAD_CONST", None, lineno=3), + Instr("RETURN_VALUE", lineno=3), + ] + ) + code = ControlFlowGraph.from_bytecode(code) + + self.check( + code, Instr("LOAD_CONST", 7, lineno=3), Instr("RETURN_VALUE", lineno=3) + ) + + def test_not_jump_if_false(self): + # Replace UNARY_NOT+POP_JUMP_IF_FALSE with POP_JUMP_IF_TRUE + # + # if not x: + # y = 9 + label = Label() + code = Bytecode( + [ + Instr("LOAD_NAME", "x"), + Instr("UNARY_NOT"), + Instr("POP_JUMP_IF_FALSE", label), + Instr("LOAD_CONST", 9), + Instr("STORE_NAME", "y"), + label, + ] + ) + + code = self.optimize_blocks(code) + label = Label() + self.check( + code, + Instr("LOAD_NAME", "x"), + Instr("POP_JUMP_IF_TRUE", label), + Instr("LOAD_CONST", 9), + Instr("STORE_NAME", "y"), + label, + ) + + def test_unconditional_jump_to_return(self): + # def func(): + # if test: + # if test2: + # x = 10 + # else: + # x = 20 + # else: + # x = 30 + + label_instr11 = Label() + label_instr14 = Label() + label_instr7 = Label() + code = Bytecode( + [ + Instr("LOAD_GLOBAL", "test", lineno=2), + Instr("POP_JUMP_IF_FALSE", label_instr11, lineno=2), + Instr("LOAD_GLOBAL", "test2", lineno=3), + Instr("POP_JUMP_IF_FALSE", label_instr7, lineno=3), + Instr("LOAD_CONST", 10, lineno=4), + Instr("STORE_FAST", "x", lineno=4), + Instr("JUMP_ABSOLUTE", label_instr14, lineno=4), + label_instr7, + Instr("LOAD_CONST", 20, lineno=6), + Instr("STORE_FAST", "x", lineno=6), + Instr("JUMP_FORWARD", label_instr14, lineno=6), + label_instr11, + Instr("LOAD_CONST", 30, lineno=8), + Instr("STORE_FAST", "x", lineno=8), + label_instr14, + Instr("LOAD_CONST", None, lineno=8), + Instr("RETURN_VALUE", lineno=8), + ] + ) + + label1 = Label() + label3 = Label() + label4 = Label() + self.check( + code, + Instr("LOAD_GLOBAL", "test", lineno=2), + Instr("POP_JUMP_IF_FALSE", label3, lineno=2), + Instr("LOAD_GLOBAL", "test2", lineno=3), + Instr("POP_JUMP_IF_FALSE", label1, lineno=3), + Instr("LOAD_CONST", 10, lineno=4), + Instr("STORE_FAST", "x", lineno=4), + Instr("JUMP_ABSOLUTE", label4, lineno=4), + label1, + Instr("LOAD_CONST", 20, lineno=6), + Instr("STORE_FAST", "x", lineno=6), + Instr("JUMP_FORWARD", label4, lineno=6), + label3, + Instr("LOAD_CONST", 30, lineno=8), + Instr("STORE_FAST", "x", lineno=8), + label4, + Instr("LOAD_CONST", None, lineno=8), + Instr("RETURN_VALUE", lineno=8), + ) + + def test_unconditional_jumps(self): + # def func(): + # if x: + # if y: + # func() + label_instr7 = Label() + code = Bytecode( + [ + Instr("LOAD_GLOBAL", "x", lineno=2), + Instr("POP_JUMP_IF_FALSE", label_instr7, lineno=2), + Instr("LOAD_GLOBAL", "y", lineno=3), + Instr("POP_JUMP_IF_FALSE", label_instr7, lineno=3), + Instr("LOAD_GLOBAL", "func", lineno=4), + Instr("CALL_FUNCTION", 0, lineno=4), + Instr("POP_TOP", lineno=4), + label_instr7, + Instr("LOAD_CONST", None, lineno=4), + Instr("RETURN_VALUE", lineno=4), + ] + ) + + label_return = Label() + self.check( + code, + Instr("LOAD_GLOBAL", "x", lineno=2), + Instr("POP_JUMP_IF_FALSE", label_return, lineno=2), + Instr("LOAD_GLOBAL", "y", lineno=3), + Instr("POP_JUMP_IF_FALSE", label_return, lineno=3), + Instr("LOAD_GLOBAL", "func", lineno=4), + Instr("CALL_FUNCTION", 0, lineno=4), + Instr("POP_TOP", lineno=4), + label_return, + Instr("LOAD_CONST", None, lineno=4), + Instr("RETURN_VALUE", lineno=4), + ) + + def test_jump_to_return(self): + # def func(condition): + # return 'yes' if condition else 'no' + label_instr4 = Label() + label_instr6 = Label() + code = Bytecode( + [ + Instr("LOAD_FAST", "condition"), + Instr("POP_JUMP_IF_FALSE", label_instr4), + Instr("LOAD_CONST", "yes"), + Instr("JUMP_FORWARD", label_instr6), + label_instr4, + Instr("LOAD_CONST", "no"), + label_instr6, + Instr("RETURN_VALUE"), + ] + ) + + label = Label() + self.check( + code, + Instr("LOAD_FAST", "condition"), + Instr("POP_JUMP_IF_FALSE", label), + Instr("LOAD_CONST", "yes"), + Instr("RETURN_VALUE"), + label, + Instr("LOAD_CONST", "no"), + Instr("RETURN_VALUE"), + ) + + def test_jump_if_true_to_jump_if_false(self): + # Replace JUMP_IF_TRUE_OR_POP jumping to POP_JUMP_IF_FALSE + # with POP_JUMP_IF_TRUE + # + # if x or y: + # z = 1 + + label_instr3 = Label() + label_instr7 = Label() + code = Bytecode( + [ + Instr("LOAD_NAME", "x"), + Instr("JUMP_IF_TRUE_OR_POP", label_instr3), + Instr("LOAD_NAME", "y"), + label_instr3, + Instr("POP_JUMP_IF_FALSE", label_instr7), + Instr("LOAD_CONST", 1), + Instr("STORE_NAME", "z"), + label_instr7, + Instr("LOAD_CONST", None), + Instr("RETURN_VALUE"), + ] + ) + + label_instr4 = Label() + label_instr7 = Label() + self.check( + code, + Instr("LOAD_NAME", "x"), + Instr("POP_JUMP_IF_TRUE", label_instr4), + Instr("LOAD_NAME", "y"), + Instr("POP_JUMP_IF_FALSE", label_instr7), + label_instr4, + Instr("LOAD_CONST", 1), + Instr("STORE_NAME", "z"), + label_instr7, + Instr("LOAD_CONST", None), + Instr("RETURN_VALUE"), + ) + + def test_jump_if_false_to_jump_if_false(self): + # Replace JUMP_IF_FALSE_OR_POP jumping to POP_JUMP_IF_FALSE