Merge branch 'main' into hoel/add_functions

.github/workflows/mypy_primer.yml

@@ -31,7 +31,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
+          python-version: "3.13"
       - name: Install dependencies
         run: pip install git+https://github.com/hauntsaninja/mypy_primer.git
       - name: Run mypy_primer

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
         args: [--fix=lf]
       - id: check-case-conflict
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.0 # must match requirements-tests.txt
+    rev: v0.8.5 # must match requirements-tests.txt
     hooks:
       - id: ruff
         name: Run ruff on stubs, tests and scripts

.vscode/settings.default.json

@@ -70,13 +70,14 @@
         "editor.defaultFormatter": "ms-python.black-formatter",
         "editor.codeActionsOnSave": {
             // Let Ruff lint fixes handle imports
-            "source.organizeImports": "never"
+            "source.organizeImports": "never",
+            "source.unusedImports": "never"
         }
     },
     // python.analysis is Pylance (pyright) configurations
     "python.analysis.fixAll": [
-        "source.unusedImports"
         // Explicitly omiting "source.convertImportFormat", some stubs use relative imports
+        // Explicitly omiting "source.unusedImports", Let Ruff lint fixes handle imports
     ],
     "python.analysis.typeshedPaths": [
         "${workspaceFolder}"

lib/ts_utils/metadata.py

@@ -23,8 +23,8 @@
 
 __all__ = [
     "NoSuchStubError",
-    "StubMetadata",
     "PackageDependencies",
+    "StubMetadata",
     "StubtestSettings",
     "get_recursive_requirements",
     "read_dependencies",
@@ -184,7 +184,7 @@ def is_obsolete(self) -> bool:
 
 
 class NoSuchStubError(ValueError):
-    """Raise NoSuchStubError to indicate that a stubs/{distribution} directory doesn't exist"""
+    """Raise NoSuchStubError to indicate that a stubs/{distribution} directory doesn't exist."""
 
 
 @cache
@@ -302,9 +302,10 @@ def read_metadata(distribution: str) -> StubMetadata:
 
 
 def update_metadata(distribution: str, **new_values: object) -> tomlkit.TOMLDocument:
-    """Updates a distribution's METADATA.toml.
+    """Update a distribution's METADATA.toml.
 
-    Return the updated TOML dictionary for use without having to open the file separately."""
+    Return the updated TOML dictionary for use without having to open the file separately.
+    """
     path = metadata_path(distribution)
     try:
         with path.open("rb") as file:

lib/ts_utils/utils.py

@@ -17,7 +17,7 @@
     from termcolor import colored as colored  # pyright: ignore[reportAssignmentType]
 except ImportError:
 
-    def colored(text: str, color: str | None = None, **kwargs: Any) -> str:  # type: ignore[misc]
+    def colored(text: str, color: str | None = None, **kwargs: Any) -> str:  # type: ignore[misc] # noqa: ARG001
         return text
 
 
@@ -92,7 +92,6 @@ def venv_python(venv_dir: Path) -> Path:
 @cache
 def parse_requirements() -> Mapping[str, Requirement]:
     """Return a dictionary of requirements from the requirements file."""
-
     with REQUIREMENTS_PATH.open(encoding="UTF-8") as requirements_file:
         stripped_lines = map(strip_comments, requirements_file)
         stripped_more = [li for li in stripped_lines if not li.startswith("-")]

pyproject.toml

@@ -37,17 +37,39 @@ exclude = ["**/test_cases/**/*.py"]
 # tell ruff not to flag these as e.g. "unused noqa comments"
 external = ["F821", "NQA", "Y"]
 select = [
+    "ARG", # flake8-unused-arguments
     "B", # flake8-bugbear
+    "D", # pydocstyle
+    "EXE", # flake8-executable
     "FA", # flake8-future-annotations
     "I", # isort
+    "N", # pep8-naming
     "PGH", # pygrep-hooks
     "RUF", # Ruff-specific and unused-noqa
+    "TRY", # tryceratops
     "UP", # pyupgrade
     "YTT", # flake8-2020
     # Flake8 base rules
     "E", # pycodestyle Error
     "F", # Pyflakes
     "W", # pycodestyle Warning
+    # Only include flake8-annotations rules that are autofixable. Otherwise leave this to mypy+pyright
+    "ANN2",
+    # Don't include TC rules that create a TYPE_CHECKING block or stringifies annotations
+    "TC004", # Move import `{qualified_name}` out of type-checking block. Import is used for more than type hinting.
+    "TC005", # Found empty type-checking block
+    # "TC008", # TODO: Enable when out of preview
+    "TC010", # Invalid string member in `X | Y`-style union type
+    # Most refurb rules are in preview and can be opinionated,
+    # consider them individually as they come out of preview (last check: 0.8.4)
+    "FURB105", # Unnecessary empty string passed to `print`
+    "FURB129", # Instead of calling `readlines()`, iterate over file object directly
+    "FURB136", # Replace `if` expression with `{min_max}` call
+    "FURB167", # Use of regular expression alias `re.{}`
+    "FURB168", # Prefer `is` operator over `isinstance` to check if an object is `None`
+    "FURB169", # Compare the identities of `{object}` and None instead of their respective types
+    "FURB177", # Prefer `Path.cwd()` over `Path().resolve()` for current-directory lookups
+    "FURB187", # Use of assignment of `reversed` on list `{name}`
     # PYI: only enable rules that have autofixes and that we always want to fix (even manually),
     # avoids duplicate # noqa with flake8-pyi and flake8-noqa flagging `PYI` codes
     # See https://github.com/plinss/flake8-noqa/issues/22
@@ -77,10 +99,6 @@ extend-safe-fixes = [
     "UP036", # Remove unnecessary `sys.version_info` blocks
 ]
 ignore = [
-    # TODO: Ruff 0.8.0 added sorting of __all__ and __slots_. Validate whether we want this in stubs
-    "RUF022",
-    "RUF023",
-
     ###
     # Rules that can conflict with the formatter (Black)
     # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules
@@ -92,11 +110,17 @@ ignore = [
     ###
     # Rules we don't want or don't agree with
     ###
-    # Slower and more verbose https://github.com/astral-sh/ruff/issues/7871
-    "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)`
+    # We're not a library, no need to document everything
+    "D1", # Missing docstring in ...
+    # Doesn't support split "summary line"
+    "D205", # 1 blank line required between summary line and description
     # Used for direct, non-subclass type comparison, for example: `type(val) is str`
     # see https://github.com/astral-sh/ruff/issues/6465
     "E721", # Do not compare types, use `isinstance()`
+    # Mostly from scripts and tests, it's ok to have messages passed directly to exceptions
+    "TRY003", # Avoid specifying long messages outside the exception class
+    # Slower and more verbose https://github.com/astral-sh/ruff/issues/7871
+    "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)`
     ###
     # False-positives, but already checked by type-checkers
     ###
@@ -106,20 +130,33 @@ ignore = [
 
 [tool.ruff.lint.per-file-ignores]
 "*.pyi" = [
-    # Most flake8-bugbear rules don't apply for third-party stubs like typeshed.
-    # B033 could be slightly useful but Ruff doesn't have per-file select
-    "B", # flake8-bugbear
+    # A lot of stubs are incomplete on purpose, and that's configured through pyright
+    # Some ANN204 (special method) are autofixable in stubs, but not all.
+    "ANN2", # Missing return type annotation for ...
+    # Most pep8-naming rules don't apply for third-party stubs like typeshed.
+    # N811 to N814 could apply, but we often use them to disambiguate a name whilst making it look like a more common one
+    "N8",
     # Rules that are out of the control of stub authors:
     "F403", # `from . import *` used; unable to detect undefined names
     # Stubs can sometimes re-export entire modules.
     # Issues with using a star-imported name will be caught by type-checkers.
     "F405", # may be undefined, or defined from star imports
+    # Ruff 0.8.0 added sorting of __all__ and __slots_.
+    # There is no consensus on whether we want to apply this to stubs, so keeping the status quo.
+    # See https://github.com/python/typeshed/pull/13108
+    "RUF022",
+    "RUF023",
 ]
-# See comment on black's force-exclude config above
 "*_pb2.pyi" = [
+    # Leave the docstrings as-is, matching source
+    "D", # pydocstyle
+    # See comment on black's force-exclude config above
     "E501", # Line too long
 ]
 
+[tool.ruff.lint.pydocstyle]
+convention = "pep257" # https://docs.astral.sh/ruff/settings/#lint_pydocstyle_convention
+
 [tool.ruff.lint.isort]
 split-on-trailing-comma = false
 combine-as-imports = true

pyrightconfig.stricter.json

@@ -44,6 +44,7 @@
         "stubs/corus",
         "stubs/dateparser",
         "stubs/defusedxml",
+        "stubs/django-import-export",
         "stubs/docker",
         "stubs/docutils",
         "stubs/Flask-SocketIO",

requirements-tests.txt

@@ -1,6 +1,6 @@
 # Type checkers that we test our stubs against. These should always
 # be pinned to a specific version to make failure reproducible.
-mypy==1.14.0
+mypy==1.14.1
 pyright==1.1.389
 # pytype can be installed on Windows, but requires building wheels, let's not do that on the CI
 pytype==2024.10.11; platform_system != "Windows" and python_version >= "3.10" and python_version < "3.13"
@@ -9,17 +9,17 @@ pytype==2024.10.11; platform_system != "Windows" and python_version >= "3.10" an
 aiohttp==3.10.11
 grpcio-tools>=1.66.2 # For grpc_tools.protoc
 mypy-protobuf==3.6.0
-packaging==24.1
+packaging==24.2
 pathspec>=0.11.1
 pre-commit
 # Required by create_baseline_stubs.py. Must match .pre-commit-config.yaml.
-ruff==0.8.0
+ruff==0.8.5
 stubdefaulter==0.1.0
 termcolor>=2.3
-tomli==2.0.2
+tomli==2.2.1
 tomlkit==0.13.2
 typing_extensions>=4.12.0rc1
-uv==0.4.26
+uv==0.5.14
 
 # Utilities for typeshed infrastructure scripts.
 ts_utils @ file:lib

scripts/create_baseline_stubs.py

@@ -218,7 +218,7 @@ def main() -> None:
     info = get_installed_package_info(project)
     if info is None:
         print(f'Error: "{project}" is not installed', file=sys.stderr)
-        print("", file=sys.stderr)
+        print(file=sys.stderr)
         print(f'Suggestion: Run "python3 -m pip install {project}" and try again', file=sys.stderr)
         sys.exit(1)
     project, version = info

scripts/stubsabot.py

@@ -632,13 +632,13 @@ def latest_commit_is_different_to_last_commit_on_origin(branch: str) -> bool:
         return True
 
 
-class RemoteConflict(Exception):
+class RemoteConflictError(Exception):
     pass
 
 
 def somewhat_safe_force_push(branch: str) -> None:
     if has_non_stubsabot_commits(branch):
-        raise RemoteConflict(f"origin/{branch} has non-stubsabot changes that are not on {branch}!")
+        raise RemoteConflictError(f"origin/{branch} has non-stubsabot changes that are not on {branch}!")
     subprocess.check_call(["git", "push", "origin", branch, "--force"])
 
 
@@ -808,7 +808,7 @@ async def main() -> None:
                     if isinstance(update, Obsolete):  # pyright: ignore[reportUnnecessaryIsInstance]
                         await suggest_typeshed_obsolete(update, session, action_level=args.action_level)
                         continue
-                except RemoteConflict as e:
+                except RemoteConflictError as e:
                     print(colored(f"... but ran into {type(e).__qualname__}: {e}", "red"))
                     continue
                 raise AssertionError

scripts/sync_protobuf/_utils.py

@@ -33,7 +33,7 @@ def extract_archive(archive_path: StrPath, destination: StrPath) -> None:
 def run_protoc(
     proto_paths: Iterable[StrPath], mypy_out: StrPath, proto_globs: Iterable[str], cwd: StrOrBytesPath | None = None
 ) -> str:
-    """TODO: Describe parameters and return"""
+    """TODO: Describe parameters and return."""
     protoc_version = (
         subprocess.run([sys.executable, "-m", "grpc_tools.protoc", "--version"], capture_output=True).stdout.decode().strip()
     )

scripts/sync_protobuf/google_protobuf.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 """
 Generates the protobuf stubs for the given protobuf version using mypy-protobuf.
 Generally, new minor versions are a good time to update the stubs.
@@ -31,7 +32,7 @@
 
 
 def extract_python_version(file_path: Path) -> str:
-    """Extract the Python version from https://github.com/protocolbuffers/protobuf/blob/main/version.json"""
+    """Extract the Python version from https://github.com/protocolbuffers/protobuf/blob/main/version.json ."""
     with open(file_path) as file:
         data: dict[str, Any] = json.load(file)
     # The root key will be the protobuf source code version
@@ -44,7 +45,7 @@ def extract_proto_file_paths(temp_dir: Path) -> list[str]:
     """
     Roughly reproduce the subset of .proto files on the public interface
     as described in py_proto_library calls in
-    https://github.com/protocolbuffers/protobuf/blob/main/python/dist/BUILD.bazel
+    https://github.com/protocolbuffers/protobuf/blob/main/python/dist/BUILD.bazel .
     """
     with open(temp_dir / EXTRACTED_PACKAGE_DIR / "python" / "dist" / "BUILD.bazel") as file:
         matched_lines = filter(None, (re.search(PROTO_FILE_PATTERN, line) for line in file))
@@ -66,14 +67,14 @@ def main() -> None:
     for old_stub in STUBS_FOLDER.rglob("*_pb2.pyi"):
         old_stub.unlink()
 
-    PROTOC_VERSION = run_protoc(
+    protoc_version = run_protoc(
         proto_paths=(f"{EXTRACTED_PACKAGE_DIR}/src",),
         mypy_out=STUBS_FOLDER,
         proto_globs=extract_proto_file_paths(temp_dir),
         cwd=temp_dir,
     )
 
-    PYTHON_PROTOBUF_VERSION = extract_python_version(temp_dir / EXTRACTED_PACKAGE_DIR / "version.json")
+    python_protobuf_version = extract_python_version(temp_dir / EXTRACTED_PACKAGE_DIR / "version.json")
 
     # Cleanup after ourselves, this is a temp dir, but it can still grow fast if run multiple times
     shutil.rmtree(temp_dir)
@@ -82,9 +83,9 @@ def main() -> None:
         "protobuf",
         extra_description=f"""Partially generated using \
 [mypy-protobuf=={MYPY_PROTOBUF_VERSION}](https://github.com/nipunn1313/mypy-protobuf/tree/v{MYPY_PROTOBUF_VERSION}) \
-and {PROTOC_VERSION} on \
+and {protoc_version} on \
 [protobuf v{PACKAGE_VERSION}](https://github.com/protocolbuffers/protobuf/releases/tag/v{PACKAGE_VERSION}) \
-(python `protobuf=={PYTHON_PROTOBUF_VERSION}`).""",
+(python `protobuf=={python_protobuf_version}`).""",
     )
     print("Updated protobuf/METADATA.toml")
 

scripts/sync_protobuf/s2clientprotocol.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 """
 Generates the protobuf stubs for the given s2clientprotocol version using mypy-protobuf.
 Generally, new minor versions are a good time to update the stubs.
@@ -29,7 +30,7 @@
 
 
 def extract_python_version(file_path: Path) -> str:
-    """Extract Python version from s2clientprotocol's build file"""
+    """Extract Python version from s2clientprotocol's build file."""
     match = re.search(VERSION_PATTERN, file_path.read_text())
     assert match
     return match.group(1)
@@ -46,14 +47,14 @@ def main() -> None:
     for old_stub in STUBS_FOLDER.rglob("*_pb2.pyi"):
         old_stub.unlink()
 
-    PROTOC_VERSION = run_protoc(
+    protoc_version = run_protoc(
         proto_paths=(EXTRACTED_PACKAGE_DIR,),
         mypy_out=STUBS_FOLDER,
         proto_globs=(f"{EXTRACTED_PACKAGE_DIR}/s2clientprotocol/*.proto",),
         cwd=temp_dir,
     )
 
-    PYTHON_S2_CLIENT_PROTO_VERSION = extract_python_version(temp_dir / EXTRACTED_PACKAGE_DIR / "s2clientprotocol" / "build.py")
+    python_s2_client_proto_version = extract_python_version(temp_dir / EXTRACTED_PACKAGE_DIR / "s2clientprotocol" / "build.py")
 
     # Cleanup after ourselves, this is a temp dir, but it can still grow fast if run multiple times
     shutil.rmtree(temp_dir)
@@ -62,8 +63,8 @@ def main() -> None:
         "s2clientprotocol",
         extra_description=f"""Partially generated using \
 [mypy-protobuf=={MYPY_PROTOBUF_VERSION}](https://github.com/nipunn1313/mypy-protobuf/tree/v{MYPY_PROTOBUF_VERSION}) \
-and {PROTOC_VERSION} on \
-[s2client-proto {PYTHON_S2_CLIENT_PROTO_VERSION}](https://github.com/Blizzard/s2client-proto/tree/{PACKAGE_VERSION}).""",
+and {protoc_version} on \
+[s2client-proto {python_s2_client_proto_version}](https://github.com/Blizzard/s2client-proto/tree/{PACKAGE_VERSION}).""",
     )
     print("Updated s2clientprotocol/METADATA.toml")