diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6dca8cb9..37f4e1e0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -298,8 +298,10 @@ jobs: CIBW_MUSLLINUX_AARCH64_IMAGE: ${{ matrix.musllinux_img || 'musllinux_1_1' }} CIBW_PRERELEASE_PYTHONS: 'True' CIBW_FREE_THREADED_SUPPORT: 'True' - CIBW_TEST_REQUIRES: pytest setuptools # 3.12+ no longer includes distutils, just always ensure setuptools is present - CIBW_TEST_COMMAND: PYTHONUNBUFFERED=1 python -m pytest ${{ matrix.test_args || '{project}' }} # default to test all + # 3.12+ no longer includes distutils, just always ensure setuptools is present + CIBW_TEST_REQUIRES: pytest setuptools pytest-xdist filelock + # default to test all + CIBW_TEST_COMMAND: PYTHONUNBUFFERED=1 python -m pytest ${{ matrix.test_args || '{project}' }} -n 4 run: | set -eux @@ -417,8 +419,8 @@ jobs: env: CIBW_BUILD: ${{ matrix.spec }} CIBW_PRERELEASE_PYTHONS: 'True' - CIBW_TEST_REQUIRES: pytest setuptools - CIBW_TEST_COMMAND: pip install pip --upgrade; cd {project}; PYTHONUNBUFFERED=1 pytest + CIBW_TEST_REQUIRES: pytest setuptools pytest-xdist filelock + CIBW_TEST_COMMAND: pip install pip --upgrade; cd {project}; PYTHONUNBUFFERED=1 pytest -n 4 MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }} SDKROOT: ${{ matrix.sdkroot || 'macosx' }} run: | @@ -508,10 +510,8 @@ jobs: env: CIBW_BUILD: ${{ matrix.spec }} CIBW_PRERELEASE_PYTHONS: 'True' - CIBW_TEST_REQUIRES: pytest setuptools - CIBW_TEST_COMMAND: 'python -m pytest {package}/src/c' - # FIXME: /testing takes ~45min on Windows and has some failures... - # CIBW_TEST_COMMAND='python -m pytest {package}/src/c {project}/testing' + CIBW_TEST_REQUIRES: pytest setuptools pytest-xdist filelock + CIBW_TEST_COMMAND: 'python -m pytest {package} -n 4' run: | set -eux diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 8e69b827..647a9049 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -42,9 +42,10 @@ Requirements: * pycparser >= 2.06: https://github.com/eliben/pycparser (automatically tracked by ``pip install cffi``). -* `pytest`_ is needed to run the tests of CFFI itself. +* `pytest`_ and `filelock` are needed to run the tests of CFFI itself. .. _`pytest`: http://pypi.python.org/pypi/pytest +.. _`filelock`: http://pypi.python.org/pypi/filelock Download and Installation: @@ -54,8 +55,7 @@ Download and Installation: ``git clone https://github.com/python-cffi/cffi`` * running the tests: ``pytest c/ testing/`` (if you didn't - install cffi yet, you need first ``python setup_base.py build_ext -f - -i``) + install cffi yet, you need first ``python -m pip install -e .``) .. _`GitHub`: https://github.com/python-cffi/cffi diff --git a/src/cffi/verifier.py b/src/cffi/verifier.py index e392a2b7..cffbe052 100644 --- a/src/cffi/verifier.py +++ b/src/cffi/verifier.py @@ -1,7 +1,7 @@ # # DEPRECATED: implementation for ffi.verify() # -import sys, os, binascii, shutil, io +import sys, os, binascii, shutil, io, threading from . import __version_verifier_modules__ from . import ffiplatform from .error import VerificationError @@ -197,8 +197,20 @@ def _write_source(self, file=None): def _compile_module(self): # compile this C source + # Note: compilation will create artifacts in tmpdir + sourcefilename + # This can exceed the windows MAXPATH quite easily. To make it shorter, + # cd into tmpdir and make the sourcefilename relative to tmdir tmpdir = os.path.dirname(self.sourcefilename) - outputfilename = ffiplatform.compile(tmpdir, self.get_extension()) + olddir = os.getcwd() + os.chdir(tmpdir) + self.sourcefilename_orig = self.sourcefilename + try: + self.sourcefilename = os.path.relpath(self.sourcefilename) + output_rel_filename = ffiplatform.compile(tmpdir, self.get_extension()) + outputfilename = os.path.join(tmpdir, output_rel_filename) + finally: + os.chdir(olddir) + self.sourcefilename = self.sourcefilename_orig try: same = ffiplatform.samefile(outputfilename, self.modulefilename) except OSError: @@ -215,12 +227,13 @@ def _load_library(self): else: return self._vengine.load_library() +local = threading.local() # ____________________________________________________________ -_FORCE_GENERIC_ENGINE = False # for tests +local._FORCE_GENERIC_ENGINE = False # for tests def _locate_engine_class(ffi, force_generic_engine): - if _FORCE_GENERIC_ENGINE: + if local._FORCE_GENERIC_ENGINE: force_generic_engine = True if not force_generic_engine: if '__pypy__' in sys.builtin_module_names: @@ -241,11 +254,11 @@ def _locate_engine_class(ffi, force_generic_engine): # ____________________________________________________________ -_TMPDIR = None +local._TMPDIR = None def _caller_dir_pycache(): - if _TMPDIR: - return _TMPDIR + if local._TMPDIR: + return local._TMPDIR result = os.environ.get('CFFI_TMPDIR') if result: return result @@ -255,8 +268,7 @@ def _caller_dir_pycache(): def set_tmpdir(dirname): """Set the temporary directory to use instead of __pycache__.""" - global _TMPDIR - _TMPDIR = dirname + local._TMPDIR = dirname def cleanup_tmpdir(tmpdir=None, keep_so=False): """Clean up the temporary directory by removing all files in it diff --git a/testing/cffi0/test_ownlib.py b/testing/cffi0/test_ownlib.py index e1a61d67..b414510e 100644 --- a/testing/cffi0/test_ownlib.py +++ b/testing/cffi0/test_ownlib.py @@ -141,10 +141,15 @@ def setup_class(cls): return # try (not too hard) to find the version used to compile this python # no mingw - from distutils.msvc9compiler import get_build_version - version = get_build_version() - toolskey = "VS%0.f0COMNTOOLS" % version - toolsdir = os.environ.get(toolskey, None) + toolsdir = None + try: + # This will always fail on setuptools>73 which removes msvc9compiler + from distutils.msvc9compiler import get_build_version + version = get_build_version() + toolskey = "VS%0.f0COMNTOOLS" % version + toolsdir = os.environ.get(toolskey, None) + except Exception: + pass if toolsdir is None: return productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py index 4942bba6..9ddeac5c 100644 --- a/testing/cffi0/test_verify.py +++ b/testing/cffi0/test_verify.py @@ -1,6 +1,6 @@ import re import pytest -import sys, os, math, weakref +import sys, os, math, weakref, random from cffi import FFI, VerificationError, VerificationMissing, model, FFIError from testing.support import * from testing.support import extra_compile_args, is_musl @@ -18,15 +18,24 @@ if distutils.ccompiler.get_default_compiler() == 'msvc': lib_m = ['msvcrt'] pass # no obvious -Werror equivalent on MSVC + class FFI(FFI): + def verify(self, *args, **kwds): + modulename = kwds.get("modulename", "_cffi_test%d_%d" % ( + random.randint(0,1000000000), random.randint(0,1000000000))) + kwds["modulename"] = modulename + return super(FFI, self).verify(*args, **kwds) + else: class FFI(FFI): def verify(self, *args, **kwds): + modulename = kwds.get("modulename", "_cffi_test%d_%d" % ( + random.randint(0,1000000000), random.randint(0,1000000000))) + kwds["modulename"] = modulename return super(FFI, self).verify( *args, extra_compile_args=extra_compile_args, **kwds) def setup_module(): import cffi.verifier - cffi.verifier.cleanup_tmpdir() # # check that no $ sign is produced in the C file; it used to be the # case that anonymous enums would produce '$enum_$1', which was @@ -52,10 +61,12 @@ def test_module_type(): ffi = FFI() lib = ffi.verify() if hasattr(lib, '_cffi_python_module'): - print('verify got a PYTHON module') + pass + # print('verify got a PYTHON module') if hasattr(lib, '_cffi_generic_module'): - print('verify got a GENERIC module') - expected_generic = (cffi.verifier._FORCE_GENERIC_ENGINE or + pass + # print('verify got a GENERIC module') + expected_generic = (cffi.verifier.local._FORCE_GENERIC_ENGINE or '__pypy__' in sys.builtin_module_names) assert hasattr(lib, '_cffi_python_module') == (not expected_generic) assert hasattr(lib, '_cffi_generic_module') == expected_generic @@ -2164,21 +2175,23 @@ def test_verify_dlopen_flags(): ffi1 = FFI() ffi1.cdef("extern int foo_verify_dlopen_flags;") + modulename = "_cffi_test_verify_dlopen_flags_%d" % random.randint(0, 1000000000) lib1 = ffi1.verify("int foo_verify_dlopen_flags;", - flags=ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY) - lib2 = get_second_lib() + flags=ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY, + modulename=modulename) + lib2 = get_second_lib(modulename) lib1.foo_verify_dlopen_flags = 42 assert lib2.foo_verify_dlopen_flags == 42 lib2.foo_verify_dlopen_flags += 1 assert lib1.foo_verify_dlopen_flags == 43 -def get_second_lib(): - # Hack, using modulename makes the test fail +def get_second_lib(modulename): ffi2 = FFI() ffi2.cdef("extern int foo_verify_dlopen_flags;") lib2 = ffi2.verify("int foo_verify_dlopen_flags;", - flags=ffi2.RTLD_GLOBAL | ffi2.RTLD_LAZY) + flags=ffi2.RTLD_GLOBAL | ffi2.RTLD_LAZY, + modulename=modulename) return lib2 def test_consider_not_implemented_function_type(): @@ -2246,7 +2259,7 @@ def test_implicit_unicode_on_windows(): def test_use_local_dir(): ffi = FFI() - lib = ffi.verify("", modulename="test_use_local_dir") + lib = ffi.verify("", modulename="_cffi_test_use_local_dir") this_dir = os.path.dirname(__file__) pycache_files = os.listdir(os.path.join(this_dir, '__pycache__')) assert any('test_use_local_dir' in s for s in pycache_files) @@ -2585,3 +2598,20 @@ def test_passing_large_list(): arg = list(range(20000000)) lib.passing_large_list(arg) # assert did not segfault + +def test_no_regen(): + from cffi.verifier import Verifier, _caller_dir_pycache + import os + ffi = FFI() + modulename = "_cffi_test_no_regen_%d" % random.randint(0, 1000000000) + ffi.cdef("double sin(double x);") + lib = ffi.verify('#include ', libraries=lib_m, modulename=modulename) + assert lib.sin(1.23) == math.sin(1.23) + # Make sure that recompiling the same code does not rebuild the C file + cfile = os.path.join(ffi.verifier.tmpdir, f"{modulename}.c") + assert os.path.exists(cfile) + os.unlink(cfile) + assert not os.path.exists(cfile) + lib = ffi.verify('#include ', libraries=lib_m, modulename=modulename) + assert lib.sin(1.23) == math.sin(1.23) + assert not os.path.exists(cfile) diff --git a/testing/cffi0/test_verify2.py b/testing/cffi0/test_verify2.py deleted file mode 100644 index 25f1e3f5..00000000 --- a/testing/cffi0/test_verify2.py +++ /dev/null @@ -1,13 +0,0 @@ -import pytest -from .test_verify import * - -# eliminate warning noise from common test modules that are repeatedly re-imported -pytestmark = pytest.mark.filterwarnings("ignore:reimporting:UserWarning") - -# This test file runs normally after test_verify. We only clean up the .c -# sources, to check that it also works when we have only the .so. The -# tests should run much faster than test_verify. - -def setup_module(): - import cffi.verifier - cffi.verifier.cleanup_tmpdir(keep_so=True) diff --git a/testing/cffi0/test_vgen.py b/testing/cffi0/test_vgen.py index 1a7e05db..5f3104bb 100644 --- a/testing/cffi0/test_vgen.py +++ b/testing/cffi0/test_vgen.py @@ -3,10 +3,9 @@ def setup_module(): - cffi.verifier.cleanup_tmpdir() - cffi.verifier._FORCE_GENERIC_ENGINE = True + cffi.verifier.local._FORCE_GENERIC_ENGINE = True # Runs all tests with _FORCE_GENERIC_ENGINE = True, to make sure we # also test vengine_gen.py. def teardown_module(): - cffi.verifier._FORCE_GENERIC_ENGINE = False + cffi.verifier.local._FORCE_GENERIC_ENGINE = False diff --git a/testing/cffi0/test_vgen2.py b/testing/cffi0/test_vgen2.py deleted file mode 100644 index 34147c87..00000000 --- a/testing/cffi0/test_vgen2.py +++ /dev/null @@ -1,13 +0,0 @@ -import cffi.verifier -from .test_vgen import * - -# This test file runs normally after test_vgen. We only clean up the .c -# sources, to check that it also works when we have only the .so. The -# tests should run much faster than test_vgen. - -def setup_module(): - cffi.verifier.cleanup_tmpdir(keep_so=True) - cffi.verifier._FORCE_GENERIC_ENGINE = True - -def teardown_module(): - cffi.verifier._FORCE_GENERIC_ENGINE = False diff --git a/testing/cffi0/test_zdistutils.py b/testing/cffi0/test_zdistutils.py index 08c432c7..75edcda8 100644 --- a/testing/cffi0/test_zdistutils.py +++ b/testing/cffi0/test_zdistutils.py @@ -90,7 +90,7 @@ def test_compile_module_explicit_filename(self): csrc = '/*hi there %s!2*/\n#include \n' % self v = Verifier(ffi, csrc, force_generic_engine=self.generic, libraries=[self.lib_m]) - basename = self.__class__.__name__[:10] + '_test_compile_module' + basename = self.__class__.__name__[:20] + '_test_compile_module' v.modulefilename = filename = str(udir.join(basename + '.so')) v.compile_module() assert filename == v.modulefilename diff --git a/testing/cffi1/test_parse_c_type.py b/testing/cffi1/test_parse_c_type.py index 49a31d54..f20ae352 100644 --- a/testing/cffi1/test_parse_c_type.py +++ b/testing/cffi1/test_parse_c_type.py @@ -2,6 +2,7 @@ import pytest import cffi from cffi import cffi_opcode +from filelock import FileLock from pathlib import Path if '__pypy__' in sys.builtin_module_names: @@ -26,14 +27,21 @@ ffi = cffi.FFI() ffi.cdef(header) -with open(os.path.join(cffi_dir, '..', 'c', 'parse_c_type.c')) as _body: - body = _body.read() -lib = ffi.verify( - body + """ -static const char *get_common_type(const char *search, size_t search_len) { - return NULL; -} -""", include_dirs=[cffi_dir]) + +def build_lib(): + sourcename = os.path.join(cffi_dir, '..', 'c', 'parse_c_type.c') + with FileLock(sourcename + ".lock"): + with open(sourcename) as _body: + body = _body.read() + lib = ffi.verify( + body + """ + static const char *get_common_type(const char *search, size_t search_len) { + return NULL; + } + """, include_dirs=[cffi_dir]) + return lib + +lib = build_lib() class ParseError(Exception): pass diff --git a/testing/embedding/__init__.py b/testing/embedding/__init__.py index e69de29b..39ac8cc4 100644 --- a/testing/embedding/__init__.py +++ b/testing/embedding/__init__.py @@ -0,0 +1,5 @@ +import sys +import pytest + +if sys.platform == "win32": + pytest.skip("XXX fixme", allow_module_level=True)