ci(appveyor): run tests via hatch, update coverage upload

The tests run on appveyor like the run locally too. There is no
significant use of appveyor-provided Python installations and virtual
environments anymore. This is all managed via hatch.

The changeset also includes the switch to CodeCov's new CLI uploaders.
This now requires an access token to be set (done via ENV).

.appveyor.yml

@@ -3,8 +3,10 @@
 # to create a "native" platform experience, using as few cross-platform
 # helper tools as possible.
 #
-# On Linux/Mac a virtualenv is used for testing. The effective virtual env
-# is available under ~/VENV.
+# On all platforms `hatch` is used for testing, and managing the test
+# environment. This yields a near-identical environment/behavior across
+# platforms and Python versions. The main difference between running tests
+# on Appveyor and locally should be the service setup (e.g., SSH, HTTPBIN).
 #
 # All workers support remote login. Login details are shown at the top of each
 # CI run log.
@@ -40,10 +42,6 @@
 #     Desktop (a required .txt extension will be added automatically). The session
 #     will run until the file is removed (or 60 min have passed)
 #
-#   - in a terminal execute, for example, `C:\datalad_debug.bat 39` to set up the
-#     environment to debug in a Python 3.8 session (should generally match the
-#     respective CI run configuration).
-
 
 # do not make repository clone cheap: interfers with VCS-based version determination
 shallow_clone: false
@@ -52,6 +50,13 @@ shallow_clone: false
 build: off
 
 environment:
+  # place coverage files to a known location regardless of where a test run
+  # is happening
+  COVERAGE_ROOT: /home/appveyor/DLTMP
+  # we pin hatch's data file to make it easy to cache it
+  HATCH_DATA_DIR: /home/appveyor/hatch-data-dir
+  # same for pip
+  PIP_CACHE: /home/appveyor/.cache/pip
   # Do not use `image` as a matrix dimension, to have fine-grained control over
   # what tests run on which platform
   # The ID variable had no impact, but sorts first in the CI run overview
@@ -62,10 +67,9 @@ environment:
 
     # Ubuntu core tests
     - job_name: test-linux
-      APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
+      APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2204
       PY: 3.9
       INSTALL_GITANNEX: git-annex -m snapshot
-      CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
 
     # same as 'test-linux', but TMPDIR is on a crippled filesystem, causing
     # most, if not all test datasets to be created on that filesystem
@@ -74,22 +78,26 @@ environment:
       PY: 3.9
       # datalad-annex git remote needs something after git-annex_8.20211x
       INSTALL_GITANNEX: git-annex -m snapshot
-      CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
 
     # Windows core tests
     - job_name: test-win
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
       # Python version specification is non-standard on windows
       PY: 39-x64
+      PY: 3.12
       INSTALL_GITANNEX: git-annex -m datalad/packages
+      COVERAGE_ROOT: C:\DLTMP
+      HATCH_DATA_DIR: C:\hatch-data-dir
+      PIP_CACHE: C:\Users\appveyor\AppData\Local\pip\Cache
 
     # MacOS core tests
     - job_name: test-mac
       APPVEYOR_BUILD_WORKER_IMAGE: macos-sonoma
-      PY: 3.9
+      PY: 3.12
       INSTALL_GITANNEX: git-annex
-      # codecov is broken: https://github.com/codecov/feedback/issues/141
-      #CODECOV_BINARY: https://uploader.codecov.io/latest/macos/codecov
+      COVERAGE_ROOT: /Users/appveyor/DLTMP
+      HATCH_DATA_DIR: /Users/appveyor/hatch-data-dir
+      PIP_CACHE: /Users/appveyor/.cache/pip
 
 
 # only run the CI if there are code or tooling changes
@@ -123,7 +131,9 @@ for:
 
     cache:
       # pip cache
-      - /home/appveyor/.cache/pip -> .appveyor.yml
+      - "${PIP_CACHE} -> .appveyor.yml"
+      # hatch-managed python versions
+      - "${HATCH_DATA_DIR}/env/virtual/.pythons -> pyproject.toml"
 
     # init cannot use any components from the repo, because it runs prior to
     # cloning it
@@ -142,16 +152,14 @@ for:
       - "[ \"x$PY\" != x ]"
       # Missing system software
       - tools/appveyor/install-syspkgs $INSTALL_SYSPKGS
-      # If a particular Python version is requested, use env setup (using the
-      # appveyor provided environments/installation). Note, these are broken
-      # on the ubuntu images
-      # https://help.appveyor.com/discussions/problems/28217-appveyor-ubunu-image-with-python3-lzma-module
-      # Otherwise create a virtualenv using the default Python 3, to enable uniform
-      # use of python/pip executables below
-      - "[ \"x$PY\" != x ] && . ${HOME}/venv${PY}/bin/activate || virtualenv -p 3 ${HOME}/dlvenv && . ${HOME}/dlvenv/bin/activate; ln -s \"$VIRTUAL_ENV\" \"${HOME}/VENV\""
+      # activate Python env solely to get `python` to become available consistently
+      # hatch will manage the actual testing environment
+      - '. ${HOME}/venv${PY}/bin/activate'
       - tools/appveyor/install-git-annex ${INSTALL_GITANNEX}
       # enable the git-annex provisioned by the installer
       - "[ -f ${HOME}/dlinstaller_env.sh ] && . ${HOME}/dlinstaller_env.sh || true"
+    before_test:
+      - 'hatch run tests.py${PY}:datalad wtf'
 
     test_script:
       # store original TMPDIR setting to limit modification to test execution
@@ -172,13 +180,12 @@ for:
           export TMPDIR=/crippledfs
         fi
       - echo TMPDIR=$TMPDIR
-      - hatch test --cover --doctest-modules
+      - 'hatch run tests.py${PY}:run-cov --doctest-modules --durations 10'
 
     after_test:
-      - |
-        [ -n "$CODECOV_BINARY"] \
-        && (python -m coverage xml && curl -Os "$CODECOV_BINARY" && chmod +x codecov && ./codecov) \
-        || true
+      - 'hatch run tests.py${PY}:cov-combine'
+      - 'hatch run tests.py${PY}:coverage xml'
+      - 'codecovcli --auto-load-params-from AppVeyor upload-process -n "appveyor-$APPVEYOR_JOB_NAME" --disable-search -f coverage.xml'
 
     on_finish:
       # conditionally block the exit of a CI run for direct debugging
@@ -193,7 +200,9 @@ for:
         - job_name: test-win
     cache:
       # pip cache
-      - C:\Users\appveyor\AppData\Local\pip\Cache -> .appveyor.yml
+      - "%PIP_CACHE% -> .appveyor.yml"
+      # hatch-managed python versions
+      - "%HATCH_DATA_DIR%\\env\\virtual\\.pythons -> pyproject.toml"
 
     # init cannot use any components from the repo, because it runs prior to
     # cloning it
@@ -228,13 +237,16 @@ for:
       # deploy git-annex, if desired
       - cmd: IF DEFINED INSTALL_GITANNEX datalad-installer --sudo ok %INSTALL_GITANNEX%
 
+    before_test:
+      - cmd: 'hatch run tests.py%PY%:datalad wtf'
+
     test_script:
-      - hatch test --cover
+      - cmd: 'hatch run tests.py%PY%:run-cov --doctest-modules --durations 10'
 
     after_test:
-      - cmd: python -m coverage xml
-      - cmd: curl -fsSL -o codecov.exe "https://uploader.codecov.io/latest/windows/codecov.exe"
-      - cmd: .\codecov.exe -f "coverage.xml"
+      - cmd: 'hatch run tests.py%PY%:cov-combine'
+      - cmd: 'hatch run tests.py%PY%:coverage xml'
+      - cmd: 'codecovcli --auto-load-params-from AppVeyor upload-process -n "appveyor-%APPVEYOR_JOB_NAME%" --disable-search -f coverage.xml'
 
     on_finish:
       # conditionally block the exit of a CI run for direct debugging
@@ -245,8 +257,7 @@ for:
 # ALL TEST RUNS
 #
 build_script:
-  # install the pieces that we need in this CI run
-  - python -m pip install hatch coverage
+  - python -m pip install hatch codecov-cli
 
 after_build:
   # Identity setup