Run test cases on hns

.github/workflows/ci.yml

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: CI
+name: CI-FNS
 
 on:
   push:
@@ -36,6 +36,7 @@ jobs:
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
         fsspec-version: ["2023.5.0", "2024.9.0", "2024.10.0"]
+        bucket-name: ["proton-ci"]
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
@@ -60,5 +61,6 @@ jobs:
         echo "TOS_ENDPOINT=${{ vars.TOS_ENDPOINT }}" >> $GITHUB_ENV
         echo "TOSFS_LOGGING_LEVEL=${{ vars.TOSFS_LOGGING_LEVEL }}" >> $GITHUB_ENV
         echo "TOS_SDK_LOGGING_LEVEL=${{ vars.TOS_SDK_LOGGING_LEVEL }}" >> $GITHUB_ENV
+        echo "TOS_BUCKET=${{ matrix.bucket-name }}" >> $GITHUB_ENV
     - name: Run tests
       run: make test

.github/workflows/ci_hns.yml

@@ -0,0 +1,66 @@
+# ByteDance Volcengine EMR, Copyright 2024.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: CI-HNS
+
+on:
+  push:
+    paths-ignore:
+      - '*.md'
+      - 'README.md'
+      - 'pyproject.toml'
+      - 'poetry.lock'
+      - 'Makefile'
+      - 'LICENSE'
+      - '.github/ISSUE_TEMPLATE/**'
+      - '.gitignore'
+      - 'docs/**'
+      - 'examples/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        fsspec-version: ["2023.5.0", "2024.9.0", "2024.10.0"]
+        bucket-name: ["proton-ci-hns"]
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        set -x
+        python -m pip install --upgrade pip
+        pip install --user poetry
+        poetry lock
+        poetry install --with dev
+        poetry add fsspec==${{ matrix.fsspec-version }}
+        poetry show fsspec
+    - name: Prepare Env
+      run: |
+        echo "Preparing environment variables"
+        echo "TOS_ACCESS_KEY=${{ secrets.TOS_ACCESS_KEY }}" >> $GITHUB_ENV
+        echo "TOS_SECRET_KEY=${{ secrets.TOS_SECRET_KEY }}" >> $GITHUB_ENV
+        echo "TOS_REGION=${{ vars.TOS_REGION }}" >> $GITHUB_ENV
+        echo "TOS_ENDPOINT=${{ vars.TOS_ENDPOINT }}" >> $GITHUB_ENV
+        echo "TOSFS_LOGGING_LEVEL=${{ vars.TOSFS_LOGGING_LEVEL }}" >> $GITHUB_ENV
+        echo "TOS_SDK_LOGGING_LEVEL=${{ vars.TOS_SDK_LOGGING_LEVEL }}" >> $GITHUB_ENV
+        echo "TOS_BUCKET=${{ matrix.bucket-name }}" >> $GITHUB_ENV
+    - name: Run tests
+      run: make test

tosfs/consts.py

@@ -17,8 +17,12 @@
 # Tos server response status codes
 TOS_SERVER_STATUS_CODE_NOT_FOUND = 404
 
-MANAGED_COPY_MAX_THRESHOLD = 5 * 2**30  # 5GB
-MANAGED_COPY_MIN_THRESHOLD = 5 * 2**20  # 5MB
+# tos bucket type (hns, fns)
+TOS_BUCKET_TYPE_HNS = "hns"
+TOS_BUCKET_TYPE_FNS = "fns"
+
+MANAGED_COPY_MAX_THRESHOLD = 5 * 2**30
+MANAGED_COPY_MIN_THRESHOLD = 5 * 2**20
 
 RETRY_NUM = 5
 PART_MIN_SIZE = 5 * 2**20