From b6d1655de9fba2c65630111d1dfeb973b3d01296 Mon Sep 17 00:00:00 2001 From: yanghua Date: Sat, 17 Aug 2024 15:18:12 +0800 Subject: [PATCH] Introduce ruff to check code style(#12) --- .pylintrc | 6 -- Makefile | 10 +-- poetry.lock | 148 +++++++----------------------------- pyproject.toml | 72 +++++++++++++++++- tosfs/core.py | 136 +++++++++++++++++---------------- tosfs/exceptions.py | 9 +-- tosfs/tests/conftest.py | 23 +++--- tosfs/tests/test_logging.py | 2 +- tosfs/tests/test_tosfs.py | 19 +++-- tosfs/tests/test_utils.py | 7 +- tosfs/utils.py | 34 ++++----- 11 files changed, 217 insertions(+), 249 deletions(-) delete mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index 5496816..0000000 --- a/.pylintrc +++ /dev/null @@ -1,6 +0,0 @@ -[MESSAGES CONTROL] -# disable these rules: -# R0913: Too many arguments (used when a function or method has too many arguments). -# W0718: Exception raised is not an instance of BaseException. -# W0223: Method 'method' is abstract in class 'class' but is not overridden. -disable=R0913,W0718,W0223 diff --git a/Makefile b/Makefile index 2e2f26a..73a7fb2 100644 --- a/Makefile +++ b/Makefile @@ -34,17 +34,15 @@ install: ## Install the project in dev mode. .PHONY: fmt fmt: ## Format code using black & isort. + poetry run ruff check tosfs/ --fix $(ENV_PREFIX)isort tosfs/ - $(ENV_PREFIX)black -l 79 tosfs/ - $(ENV_PREFIX)black -l 79 tosfs/tests/ + $(ENV_PREFIX)black -l 88 tosfs/ .PHONY: lint lint: ## Run pep8, black, mypy linters. set -e; - $(ENV_PREFIX)pylint tosfs/ - $(ENV_PREFIX)flake8 tosfs/ - $(ENV_PREFIX)black -l 79 --check tosfs/ - $(ENV_PREFIX)black -l 79 --check tosfs/tests/ + $(ENV_PREFIX)black -l 88 --check tosfs/ + poetry run ruff check tosfs/ $(ENV_PREFIX)mypy --ignore-missing-imports tosfs/ .PHONY: test diff --git a/poetry.lock b/poetry.lock index f3336a4..fa70cdd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,19 +1,5 @@ # This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. -[[package]] -name = "astroid" -version = "3.2.4" -description = "An abstract syntax tree for Python with inference support." -optional = false -python-versions = ">=3.8.0" -files = [ - {file = "astroid-3.2.4-py3-none-any.whl", hash = "sha256:413658a61eeca6202a59231abb473f932038fbcbf1666587f66d482083413a25"}, - {file = "astroid-3.2.4.tar.gz", hash = "sha256:0e14202810b30da1b735827f78f5157be2bbd4a7a59b7707ca0bfc2fb4c0063a"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} - [[package]] name = "black" version = "24.4.1" @@ -289,21 +275,6 @@ wrapt = ">=1.10,<2" [package.extras] dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] -[[package]] -name = "dill" -version = "0.3.8" -description = "serialize all of Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, - {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, -] - -[package.extras] -graph = ["objgraph (>=1.7.2)"] -profile = ["gprof2dot (>=2022.7.29)"] - [[package]] name = "exceptiongroup" version = "1.2.2" @@ -318,22 +289,6 @@ files = [ [package.extras] test = ["pytest (>=6)"] -[[package]] -name = "flake8" -version = "7.0.0" -description = "the modular source code checker: pep8 pyflakes and co" -optional = false -python-versions = ">=3.8.1" -files = [ - {file = "flake8-7.0.0-py2.py3-none-any.whl", hash = "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3"}, - {file = "flake8-7.0.0.tar.gz", hash = "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132"}, -] - -[package.dependencies] -mccabe = ">=0.7.0,<0.8.0" -pycodestyle = ">=2.11.0,<2.12.0" -pyflakes = ">=3.2.0,<3.3.0" - [[package]] name = "fsspec" version = "2024.6.1" @@ -409,17 +364,6 @@ files = [ [package.extras] colors = ["colorama (>=0.4.6)"] -[[package]] -name = "mccabe" -version = "0.7.0" -description = "McCabe checker, plugin for flake8" -optional = false -python-versions = ">=3.6" -files = [ - {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, - {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, -] - [[package]] name = "mypy" version = "1.10.0" @@ -531,58 +475,6 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] -[[package]] -name = "pycodestyle" -version = "2.11.1" -description = "Python style guide checker" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"}, - {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"}, -] - -[[package]] -name = "pyflakes" -version = "3.2.0" -description = "passive checker of Python programs" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a"}, - {file = "pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f"}, -] - -[[package]] -name = "pylint" -version = "3.2.6" -description = "python code static checker" -optional = false -python-versions = ">=3.8.0" -files = [ - {file = "pylint-3.2.6-py3-none-any.whl", hash = "sha256:03c8e3baa1d9fb995b12c1dbe00aa6c4bcef210c2a2634374aedeb22fb4a8f8f"}, - {file = "pylint-3.2.6.tar.gz", hash = "sha256:a5d01678349454806cff6d886fb072294f56a58c4761278c97fb557d708e1eb3"}, -] - -[package.dependencies] -astroid = ">=3.2.4,<=3.3.0-dev0" -colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} -dill = [ - {version = ">=0.2", markers = "python_version < \"3.11\""}, - {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, - {version = ">=0.3.6", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, -] -isort = ">=4.2.5,<5.13.0 || >5.13.0,<6" -mccabe = ">=0.6,<0.8" -platformdirs = ">=2.2.0" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -tomlkit = ">=0.10.1" -typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} - -[package.extras] -spelling = ["pyenchant (>=3.2,<4.0)"] -testutils = ["gitpython (>3)"] - [[package]] name = "pytest" version = "8.1.1" @@ -655,6 +547,33 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "ruff" +version = "0.6.0" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.6.0-py3-none-linux_armv6l.whl", hash = "sha256:92dcce923e5df265781e5fc76f9a1edad52201a7aafe56e586b90988d5239013"}, + {file = "ruff-0.6.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:31b90ff9dc79ed476c04e957ba7e2b95c3fceb76148f2079d0d68a908d2cfae7"}, + {file = "ruff-0.6.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6d834a9ec9f8287dd6c3297058b3a265ed6b59233db22593379ee38ebc4b9768"}, + {file = "ruff-0.6.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2089267692696aba342179471831a085043f218706e642564812145df8b8d0d"}, + {file = "ruff-0.6.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:aa62b423ee4bbd8765f2c1dbe8f6aac203e0583993a91453dc0a449d465c84da"}, + {file = "ruff-0.6.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7344e1a964b16b1137ea361d6516ce4ee61a0403fa94252a1913ecc1311adcae"}, + {file = "ruff-0.6.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:487f3a35c3f33bf82be212ce15dc6278ea854e35573a3f809442f73bec8b2760"}, + {file = "ruff-0.6.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:75db409984077a793cf344d499165298a6f65449e905747ac65983b12e3e64b1"}, + {file = "ruff-0.6.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:84908bd603533ecf1db456d8fc2665d1f4335d722e84bc871d3bbd2d1116c272"}, + {file = "ruff-0.6.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f1749a0aef3ec41ed91a0e2127a6ae97d2e2853af16dbd4f3c00d7a3af726c5"}, + {file = "ruff-0.6.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:016fea751e2bcfbbd2f8cb19b97b37b3fd33148e4df45b526e87096f4e17354f"}, + {file = "ruff-0.6.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:6ae80f141b53b2e36e230017e64f5ea2def18fac14334ffceaae1b780d70c4f7"}, + {file = "ruff-0.6.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:eaaaf33ea4b3f63fd264d6a6f4a73fa224bbfda4b438ffea59a5340f4afa2bb5"}, + {file = "ruff-0.6.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:7667ddd1fc688150a7ca4137140867584c63309695a30016880caf20831503a0"}, + {file = "ruff-0.6.0-py3-none-win32.whl", hash = "sha256:ae48365aae60d40865a412356f8c6f2c0be1c928591168111eaf07eaefa6bea3"}, + {file = "ruff-0.6.0-py3-none-win_amd64.whl", hash = "sha256:774032b507c96f0c803c8237ce7d2ef3934df208a09c40fa809c2931f957fe5e"}, + {file = "ruff-0.6.0-py3-none-win_arm64.whl", hash = "sha256:a5366e8c3ae6b2dc32821749b532606c42e609a99b0ae1472cf601da931a048c"}, + {file = "ruff-0.6.0.tar.gz", hash = "sha256:272a81830f68f9bd19d49eaf7fa01a5545c5a2e86f32a9935bb0e4bb9a1db5b8"}, +] + [[package]] name = "six" version = "1.16.0" @@ -677,17 +596,6 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -[[package]] -name = "tomlkit" -version = "0.13.2" -description = "Style preserving TOML library" -optional = false -python-versions = ">=3.8" -files = [ - {file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"}, - {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, -] - [[package]] name = "tos" version = "2.7.1" @@ -815,4 +723,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "475bbf73236f742838299a02ab94db0e945f623b1135d07d2e640db48acc02e3" +content-hash = "4e5dfa8983ef184a7ecc10626b7ada201d7c852379100f28640f56a2d4a03af6" diff --git a/pyproject.toml b/pyproject.toml index 6064bed..f68ada4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,14 +14,82 @@ tos = ">=2.7.0" [tool.poetry.group.dev.dependencies] fsspec = ">=2023.5.0" tos = ">=2.7.0" -pylint = "==3.2.6" -flake8 = "==7.0.0" black = "==24.4.1" isort = "==5.13.2" mypy = "==1.10.0" pytest = "==8.1.1" pytest-cov = "==5.0.0" coverage = "==7.5.0" +ruff = "==0.6.0" + +[tool.pydocstyle] +convention = "numpy" + +[tool.ruff] +target-version = "py39" +line-length = 88 + +[tool.ruff.lint] +select = [ + "A", # Annotations rules + "ASYNC", # Asynchronous programming rules + "B", # Bugbear rules + "C", # Complexity rules + "D", # Docstring rules + "E", # Error rules + "F", # Pyflakes rules + "FURB", # FURB rules + "G", # General rules + "I", # Import rules + "LOG", # Logging rules + "N", # Naming rules + "PERF", # Performance rules + "PIE", # PIE rules + "PLC", # Pylint convention rules + "PLE", # Pylint error rules + "PLR", # Pylint refactor rules + "PLW", # Pylint warning rules + "PT", # Pytest rules + "PYI", # Pyright rules + "Q", # Quotes rules + "RUF", # Ruff-specific rules + "S", # Security rules + "SIM", # Similarity rules + "SLOT", # Slot rules + "T", # Testing rules + "W", # Whitespace rules + "YTT", # YTT rules +] +ignore = [ + "S101", # Use of `assert` detected + "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes + "D203", # no-blank-line-before-class + "D213", # multi-line-summary-second-line + "PLR0913", # Too many arguments in function definition + "SIM105", # Use `contextlib.suppress(IOError)` instead of `try`-`except`-`pass` +] + +[tool.ruff.lint.per-file-ignores] +"tosfs/tests/*" = [ + #D103 Missing docstring in public function + "D103", + #D100 Missing docstring in public module + "D100" +] +"tosfs/__init__.py" = [ + #D104 Missing docstring in public package + "D104", + #D107 Missing docstring in __init__ + "D107" +] + +[tool.mypy] +python_version = "3.9" +ignore_missing_imports = true +disallow_untyped_calls = true +disallow_untyped_defs = true +strict_optional = true +exclude = "tosfs/tests/.*" [build-system] requires = ["poetry-core"] diff --git a/tosfs/core.py b/tosfs/core.py index 1d5b1d8..42949f1 100644 --- a/tosfs/core.py +++ b/tosfs/core.py @@ -12,18 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -The core module of TOSFS. -""" +"""The core module of TOSFS.""" import logging import os -from typing import Optional, Tuple +from typing import List, Optional, Tuple, Union import tos from fsspec import AbstractFileSystem from fsspec.utils import setup_logging as setup_logger from tos.models import CommonPrefixInfo -from tos.models2 import ListedObjectVersion +from tos.models2 import ListedObject, ListedObjectVersion from tosfs.exceptions import TosfsError from tosfs.utils import find_bucket_key @@ -34,10 +32,8 @@ logger = logging.getLogger("tosfs") -def setup_logging(): - """ - Set up the logging configuration for TOSFS. - """ +def setup_logging() -> None: + """Set up the logging configuration for TOSFS.""" setup_logger( logger=logger, level=os.environ.get(ENV_NAME_TOSFS_LOGGING_LEVEL, "INFO"), @@ -52,21 +48,23 @@ def setup_logging(): class TosFileSystem(AbstractFileSystem): - """ - Tos file system. An implementation of AbstractFileSystem which is an + """Tos file system. + + It's an implementation of AbstractFileSystem which is an abstract super-class for pythonic file-systems. """ def __init__( self, - endpoint_url=None, - key="", - secret="", - region=None, - version_aware=False, - credentials_provider=None, - **kwargs, - ): + endpoint_url: Optional[str] = None, + key: str = "", + secret: str = "", + region: Optional[str] = None, + version_aware: bool = False, + credentials_provider: Optional[object] = None, + **kwargs: Union[str, bool, float, None], + ) -> None: + """Initialise the TosFileSystem.""" self.tos_client = tos.TosClientV2( key, secret, @@ -77,9 +75,16 @@ def __init__( self.version_aware = version_aware super().__init__(**kwargs) - def ls(self, path, detail=False, refresh=False, versions=False, **kwargs): - """ - List objects under the given path. + def ls( + self, + path: str, + detail: bool = False, + refresh: bool = False, + versions: bool = False, + **kwargs: Union[str, bool, float, None], + ) -> Union[List[dict], List[str]]: + """List objects under the given path. + :param path: The path to list. :param detail: Whether to return detailed information. :param refresh: Whether to refresh the cache. @@ -108,9 +113,9 @@ def ls(self, path, detail=False, refresh=False, versions=False, **kwargs): return files if detail else sorted([o["name"] for o in files]) - def _lsbuckets(self, refresh=False): - """ - List all buckets in the account. + def _lsbuckets(self, refresh: bool = False) -> List[dict]: + """List all buckets in the account. + :param refresh: Whether to refresh the cache. :return: A list of buckets. """ @@ -125,40 +130,37 @@ def _lsbuckets(self, refresh=False): raise e except Exception as e: logger.error("Tosfs failed with unknown error: %s", e) - raise TosfsError( - f"Tosfs failed with unknown error: {e}" - ) from e - - buckets = [] - for bucket in resp.buckets: - buckets.append( - { - "Key": bucket.name, - "Size": 0, - "StorageClass": "BUCKET", - "size": 0, - "type": "directory", - "name": bucket.name, - } - ) + raise TosfsError(f"Tosfs failed with unknown error: {e}") from e + + buckets = [ + { + "Key": bucket.name, + "Size": 0, + "StorageClass": "BUCKET", + "size": 0, + "type": "directory", + "name": bucket.name, + } + for bucket in resp.buckets + ] self.dircache[""] = buckets return self.dircache[""] def _lsdir( self, - path, - refresh=False, + path: str, + refresh: bool = False, max_items: int = 1000, - delimiter="/", - prefix="", - versions=False, - ): - """ - List objects in a bucket, here we use cache to improve performance. + delimiter: str = "/", + prefix: str = "", + versions: bool = False, + ) -> List[Union[CommonPrefixInfo, ListedObject, ListedObjectVersion]]: + """List objects in a bucket, here we use cache to improve performance. + :param path: The path to list. :param refresh: Whether to refresh the cache. - :param max_items: The maximum number of items to return, default is 1000. # noqa: E501 + :param max_items: The maximum number of items to return, default is 1000. :param delimiter: The delimiter to use for grouping objects. :param prefix: The prefix to use for filtering objects. :param versions: Whether to list object versions. @@ -193,16 +195,16 @@ def _lsdir( def _listdir( self, - bucket, + bucket: str, max_items: int = 1000, - delimiter="/", - prefix="", - versions=False, - ): - """ - List objects in a bucket. + delimiter: str = "/", + prefix: str = "", + versions: bool = False, + ) -> List[Union[CommonPrefixInfo, ListedObject, ListedObjectVersion]]: + """List objects in a bucket. + :param bucket: The bucket name. - :param max_items: The maximum number of items to return, default is 1000. # noqa: E501 + :param max_items: The maximum number of items to return, default is 1000. :param delimiter: The delimiter to use for grouping objects. :param prefix: The prefix to use for filtering objects. :param versions: Whether to list object versions. @@ -231,9 +233,7 @@ def _listdir( ) is_truncated = resp.is_truncated all_results.extend( - resp.versions - + resp.common_prefixes - + resp.delete_markers + resp.versions + resp.common_prefixes + resp.delete_markers ) key_marker, version_id_marker = ( resp.next_key_marker, @@ -270,9 +270,8 @@ def _listdir( logger.error("Tosfs failed with unknown error: %s", e) raise TosfsError(f"Tosfs failed with unknown error: {e}") from e - def _split_path(self, path) -> Tuple[str, str, Optional[str]]: - """ - Normalise tos path string into bucket and key. + def _split_path(self, path: str) -> Tuple[str, str, Optional[str]]: + """Normalise tos path string into bucket and key. Parameters ---------- @@ -284,8 +283,9 @@ def _split_path(self, path) -> Tuple[str, str, Optional[str]]: >>> split_path("tos://mybucket/path/to/file") ['mybucket', 'path/to/file', None] # pylint: disable=line-too-long - >>> split_path("tos://mybucket/path/to/versioned_file?versionId=some_version_id") # noqa: E501 + >>> split_path("tos://mybucket/path/to/versioned_file?versionId=some_version_id") ['mybucket', 'path/to/versioned_file', 'some_version_id'] + """ path = self._strip_protocol(path) path = path.lstrip("/") @@ -301,7 +301,7 @@ def _split_path(self, path) -> Tuple[str, str, Optional[str]]: ) @staticmethod - def _fill_common_prefix_info(common_prefix: CommonPrefixInfo, bucket): + def _fill_common_prefix_info(common_prefix: CommonPrefixInfo, bucket: str) -> dict: return { "name": common_prefix.prefix[:-1], "Key": "/".join([bucket, common_prefix.prefix]), @@ -310,7 +310,9 @@ def _fill_common_prefix_info(common_prefix: CommonPrefixInfo, bucket): } @staticmethod - def _fill_object_info(obj, bucket, versions=False): + def _fill_object_info( + obj: ListedObject, bucket: str, versions: bool = False + ) -> dict: result = { "Key": f"{bucket}/{obj.key}", "size": obj.size, diff --git a/tosfs/exceptions.py b/tosfs/exceptions.py index fe306c4..dd86ddf 100644 --- a/tosfs/exceptions.py +++ b/tosfs/exceptions.py @@ -11,15 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -This module contains exceptions definition for the tosfs package. -""" +"""iT contains exceptions definition for the tosfs package.""" class TosfsError(Exception): - """ - Base class for all tosfs exceptions. - """ + """Base class for all tosfs exceptions.""" def __init__(self, message: str): + """Initialize the base class for all exceptions in the tosfs package.""" super().__init__(message) diff --git a/tosfs/tests/conftest.py b/tosfs/tests/conftest.py index 4ecfac9..488ed31 100644 --- a/tosfs/tests/conftest.py +++ b/tosfs/tests/conftest.py @@ -13,6 +13,7 @@ # limitations under the License. import os +from typing import Generator import pytest from tos import EnvCredentialsProvider @@ -22,34 +23,32 @@ @pytest.fixture(scope="module") -def tosfs_env_prepare(): +def _tosfs_env_prepare() -> None: if "TOS_ACCESS_KEY" not in os.environ: - raise EnvironmentError( - "Can not find TOS_ACCESS_KEY in environment variables." - ) + raise EnvironmentError("Can not find TOS_ACCESS_KEY in environment variables.") if "TOS_SECRET_KEY" not in os.environ: - raise EnvironmentError( - "Can not find TOS_SECRET_KEY in environment variables." - ) + raise EnvironmentError("Can not find TOS_SECRET_KEY in environment variables.") @pytest.fixture(scope="module") -def tosfs(tosfs_env_prepare): +def tosfs(_tosfs_env_prepare: None) -> TosFileSystem: tosfs = TosFileSystem( endpoint_url=os.environ.get("TOS_ENDPOINT"), region=os.environ.get("TOS_REGION"), credentials_provider=EnvCredentialsProvider(), ) - yield tosfs + return tosfs @pytest.fixture(scope="module") -def bucket(): - yield os.environ.get("TOS_BUCKET", "proton-ci") +def bucket() -> str: + return os.environ.get("TOS_BUCKET", "proton-ci") @pytest.fixture(autouse=True) -def temporary_workspace(tosfs, bucket): +def temporary_workspace( + tosfs: TosFileSystem, bucket: str +) -> Generator[str, None, None]: workspace = random_path() # currently, make dir via purely tos python client, # will replace with tosfs.mkdir in the future diff --git a/tosfs/tests/test_logging.py b/tosfs/tests/test_logging.py index 5925cd3..c2b5460 100644 --- a/tosfs/tests/test_logging.py +++ b/tosfs/tests/test_logging.py @@ -18,7 +18,7 @@ from tosfs.core import ENV_NAME_TOSFS_LOGGING_LEVEL, setup_logging -def test_logging_level_debug(): +def test_logging_level_debug() -> None: # Set the environment variable to DEBUG os.environ[ENV_NAME_TOSFS_LOGGING_LEVEL] = "DEBUG" diff --git a/tosfs/tests/test_tosfs.py b/tosfs/tests/test_tosfs.py index f282513..eebefce 100644 --- a/tosfs/tests/test_tosfs.py +++ b/tosfs/tests/test_tosfs.py @@ -17,8 +17,10 @@ import pytest from tos.exceptions import TosServerError +from tosfs.core import TosFileSystem -def test_ls_bucket(tosfs, bucket): + +def test_ls_bucket(tosfs: TosFileSystem, bucket: str) -> None: assert bucket in tosfs.ls("", detail=False) detailed_list = tosfs.ls("", detail=True) assert detailed_list @@ -31,7 +33,7 @@ def test_ls_bucket(tosfs, bucket): tosfs.ls("nonexistent", detail=False) -def test_ls_dir(tosfs, bucket, temporary_workspace): +def test_ls_dir(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> None: assert temporary_workspace in tosfs.ls(bucket, detail=False) detailed_list = tosfs.ls(bucket, detail=True) assert detailed_list @@ -40,18 +42,15 @@ def test_ls_dir(tosfs, bucket, temporary_workspace): assert item["type"] == "directory" break else: - assert ( - False - ), f"Directory {temporary_workspace} not found in {detailed_list}" + raise AssertionError( + f"Directory {temporary_workspace} not found in {detailed_list}" + ) assert tosfs.ls(f"{bucket}/{temporary_workspace}", detail=False) == [] - assert ( - tosfs.ls(f"{bucket}/{temporary_workspace}/nonexistent", detail=False) - == [] - ) + assert tosfs.ls(f"{bucket}/{temporary_workspace}/nonexistent", detail=False) == [] -def test_ls_cache(tosfs, bucket): +def test_ls_cache(tosfs: TosFileSystem, bucket: str) -> None: tosfs.tos_client.list_objects_type2 = MagicMock( return_value=MagicMock( is_truncated=False, diff --git a/tosfs/tests/test_utils.py b/tosfs/tests/test_utils.py index 39a8684..dbbadf3 100644 --- a/tosfs/tests/test_utils.py +++ b/tosfs/tests/test_utils.py @@ -14,11 +14,12 @@ import pytest +from tosfs.core import TosFileSystem from tosfs.utils import find_bucket_key @pytest.mark.parametrize( - "input_str, expected_output", + ("input_str", "expected_output"), [ ("bucket/key", ("bucket", "key")), ("bucket/", ("bucket", "")), @@ -28,5 +29,7 @@ ("", ("", "")), ], ) -def test_find_bucket_key(tosfs, input_str, expected_output): +def test_find_bucket_key( + tosfs: TosFileSystem, input_str: str, expected_output: str +) -> None: assert find_bucket_key(input_str) == expected_output diff --git a/tosfs/utils.py b/tosfs/utils.py index dc8caa5..6b5a6ae 100644 --- a/tosfs/utils.py +++ b/tosfs/utils.py @@ -12,45 +12,45 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -This module contains utility functions for the tosfs package. -""" +"""The module contains utility functions for the tosfs package.""" import random import re import string +from typing import Tuple def random_path(length: int = 5) -> str: - """ - Generate a random path(dir or file) of the given length. + """Generate a random path(dir or file) of the given length. Args: - length (int): The length of the random string. + ---- + length (int): The length of the random string. Returns: - str: The random string. + ------- + str: The random string. + """ - return "".join( - random.choices(string.ascii_letters + string.digits, k=length) - ) + return "".join(random.choices(string.ascii_letters + string.digits, k=length)) -def find_bucket_key(tos_path): - """ - This is a helper function that given an tos path such that the path - is of the form: bucket/key - It will return the bucket and the key represented by the tos path +def find_bucket_key(tos_path: str) -> Tuple[str, str]: + """It's a helper function to find bucket and key pair. + + It receives a tos path such that the path + is of the form: bucket/key. + It will return the bucket and the key represented by the tos path. """ bucket_format_list = [ re.compile( - r"^(?P:tos:[a-z\-0-9]*:[0-9]{12}:accesspoint[:/][^/]+)/?" # noqa: E501 + r"^(?P:tos:[a-z\-0-9]*:[0-9]{12}:accesspoint[:/][^/]+)/?" r"(?P.*)$" ), re.compile( r"^(?P:tos-outposts:[a-z\-0-9]+:[0-9]{12}:outpost[/:]" # pylint: disable=line-too-long - r"[a-zA-Z0-9\-]{1,63}[/:](bucket|accesspoint)[/:][a-zA-Z0-9\-]{1,63})[/:]?(?P.*)$" # noqa: E501 + r"[a-zA-Z0-9\-]{1,63}[/:](bucket|accesspoint)[/:][a-zA-Z0-9\-]{1,63})[/:]?(?P.*)$" ), re.compile( r"^(?P:tos-outposts:[a-z\-0-9]+:[0-9]{12}:outpost[/:]"