Add fsspec compatibility test cases

fsspec · Sep 4, 2024 · 705bd6f · 705bd6f
1 parent c63fa37
commit 705bd6f
Show file tree

Hide file tree

Showing 4 changed files with 103 additions and 118 deletions.
diff --git a/.github/workflows/compatibilitytest.yml b/.github/workflows/compatibilitytest.yml
@@ -0,0 +1,32 @@
+name: CompatibilityTest
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        set -x
+        python -m pip install --upgrade pip
+        pip install --user poetry
+        make install
+    - name: Prepare Env
+      run: |
+        echo "Preparing environment variables"
+        echo "TOS_ACCESS_KEY=${{ secrets.TOS_ACCESS_KEY }}" >> $GITHUB_ENV
+        echo "TOS_SECRET_KEY=${{ secrets.TOS_SECRET_KEY }}" >> $GITHUB_ENV
+        echo "TOS_REGION=${{ vars.TOS_REGION }}" >> $GITHUB_ENV
+        echo "TOS_ENDPOINT=${{ vars.TOS_ENDPOINT }}" >> $GITHUB_ENV
+        echo "TOSFS_LOGGING_LEVEL=${{ vars.TOSFS_LOGGING_LEVEL }}" >> $GITHUB_ENV
+    - name: Run compatibility tests
+      run: make test_compatibility
diff --git a/Makefile b/Makefile
@@ -7,17 +7,18 @@ help:             ## Show the help.
 	@echo "Usage: make <target>"
 	@echo ""
 	@echo "Targets:"
-	@echo "help:             ## Show the help."
-	@echo "show:             ## Show the current environment."
-	@echo "install:          ## Install the project in dev mode."
-	@echo "fmt:              ## Format code using black & isort."
-	@echo "lint:             ## Run pep8, black, mypy linters."
-	@echo "test: lint        ## Run tests and generate coverage report."
-	@echo "watch:            ## Run tests on every change."
-	@echo "clean:            ## Clean unused files."
-	@echo "release:          ## Create a new tag for release."
-	@echo "docs:             ## Build the documentation."
-	@echo "release_wheel:    ## Release wheel for python client."
+	@echo "help:             	## Show the help."
+	@echo "show:             	## Show the current environment."
+	@echo "install:          	## Install the project in dev mode."
+	@echo "fmt:              	## Format code using black & isort."
+	@echo "lint:             	## Run pep8, black, mypy linters."
+	@echo "test: lint        	## Run tests and generate coverage report."
+	@echo "test_compatibility: 	## Run compatibility tests."
+	@echo "watch:            	## Run tests on every change."
+	@echo "clean:            	## Clean unused files."
+	@echo "release:          	## Create a new tag for release."
+	@echo "docs:             	## Build the documentation."
+	@echo "release_wheel:    	## Release wheel for python client."
 
 .PHONY: show
 show:             ## Show the current environment.
@@ -47,7 +48,11 @@ lint:             ## Run pep8, black, mypy linters.
 
 .PHONY: test
 test:             ## Run tests and generate coverage report.
-	$(ENV_PREFIX)pytest -vv -s --cov-config .coveragerc --cov=tosfs -l --tb=short --maxfail=1 ${TEST_DIR}
+	$(ENV_PREFIX)pytest -vv -s --cov-config .coveragerc --cov=tosfs -l --tb=short --maxfail=1 ${TEST_DIR} --ignore=${TEST_DIR}/test_fsspec.py
+
+.PHONY: test_compatibility
+test_compatibility:             ## Run compatibility tests.
+	$(ENV_PREFIX)pytest -vv -s -l --tb=short --maxfail=1 ${TEST_DIR} ${TEST_DIR}/test_fsspec.py
 
 .PHONY: watch
 watch:            ## Run tests on every change.

diff --git a/tosfs/tests/conftest.py b/tosfs/tests/conftest.py
@@ -41,20 +41,20 @@ def tosfs(_tosfs_env_prepare: None) -> TosFileSystem:
     )
     return tosfs
 
+
 @pytest.fixture(scope="module")
 def fsspecfs(_tosfs_env_prepare: None) -> Any:
-    known_implementations["tos"] = {
-        "class": "tosfs.core.TosFileSystem"
-    }
+    known_implementations["tos"] = {"class": "tosfs.core.TosFileSystem"}
 
     fsspecfs, _ = fsspec.core.url_to_fs(
         "tos://",
-        endpoint_url = os.environ.get("TOS_ENDPOINT"),
-        region = os.environ.get("TOS_REGION"),
-        credentials_provider=EnvCredentialsProvider()
+        endpoint_url=os.environ.get("TOS_ENDPOINT"),
+        region=os.environ.get("TOS_REGION"),
+        credentials_provider=EnvCredentialsProvider(),
     )
     return fsspecfs
 
+
 @pytest.fixture(scope="module")
 def bucket() -> str:
     return os.environ.get("TOS_BUCKET", "proton-ci")