ci: basic, cross-platform appveyor setup

.appveyor.yml

@@ -0,0 +1,254 @@
+# This CI setup provides a largely homogeneous configuration across all
+# major platforms (Windows, MacOS, and Linux). The aim of this test setup is
+# to create a "native" platform experience, using as few cross-platform
+# helper tools as possible.
+#
+# On Linux/Mac a virtualenv is used for testing. The effective virtual env
+# is available under ~/VENV.
+#
+# All workers support remote login. Login details are shown at the top of each
+# CI run log.
+#
+# - Linux/Mac workers (via SSH):
+#
+#   - A permitted SSH key must be defined in an APPVEYOR_SSH_KEY environment
+#     variable (via the appveyor project settings)
+#
+#   - SSH login info is given in the form of: '[email protected] -p 22xxx'
+#
+#   - Login with:
+#
+#     ssh -o StrictHostKeyChecking=no <LOGIN>
+#
+#   - to prevent the CI run from exiting, `touch` a file named `BLOCK` in the
+#     user HOME directory (current directory directly after login). The session
+#     will run until the file is removed (or 60 min have passed)
+#
+# - Windows workers (via RDP):
+#
+#   - An RDP password should be defined in an APPVEYOR_RDP_PASSWORD environment
+#     variable (via the appveyor project settings), or a random password is used
+#     every time
+#
+#   - RDP login info is given in the form of IP:PORT
+#
+#   - Login with:
+#
+#     xfreerdp /cert:ignore /dynamic-resolution /u:appveyor /p:<PASSWORD> /v:<LOGIN>
+#
+#   - to prevent the CI run from exiting, create a textfile named `BLOCK` on the
+#     Desktop (a required .txt extension will be added automatically). The session
+#     will run until the file is removed (or 60 min have passed)
+#
+#   - in a terminal execute, for example, `C:\datalad_debug.bat 39` to set up the
+#     environment to debug in a Python 3.8 session (should generally match the
+#     respective CI run configuration).
+
+
+# do not make repository clone cheap: interfers with VCS-based version determination
+shallow_clone: false
+
+# turn of support for MS project build support (not needed)
+build: off
+
+environment:
+  # Do not use `image` as a matrix dimension, to have fine-grained control over
+  # what tests run on which platform
+  # The ID variable had no impact, but sorts first in the CI run overview
+  # an intelligible name can help to locate a specific test run
+  matrix:
+    # List a CI run for each platform first, to have immediate access when there
+    # is a need for debugging
+
+    # Ubuntu core tests
+    - job_name: test-linux
+      APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
+      PY: 3.9
+      INSTALL_GITANNEX: git-annex -m snapshot
+      CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
+
+    # same as 'test-linux', but TMPDIR is on a crippled filesystem, causing
+    # most, if not all test datasets to be created on that filesystem
+    - job_name: test-linux-crippled
+      APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2204
+      PY: 3.9
+      # datalad-annex git remote needs something after git-annex_8.20211x
+      INSTALL_GITANNEX: git-annex -m snapshot
+      CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
+
+    # Windows core tests
+    - job_name: test-win
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+      # Python version specification is non-standard on windows
+      PY: 39-x64
+      INSTALL_GITANNEX: git-annex -m datalad/packages
+
+    # MacOS core tests
+    - job_name: test-mac
+      APPVEYOR_BUILD_WORKER_IMAGE: macos-sonoma
+      PY: 3.9
+      INSTALL_GITANNEX: git-annex
+      # codecov is broken: https://github.com/codecov/feedback/issues/141
+      #CODECOV_BINARY: https://uploader.codecov.io/latest/macos/codecov
+
+
+# only run the CI if there are code or tooling changes
+only_commits:
+  files:
+    - datalad_core/
+    - tools/
+    - pyproject.toml
+    - .appveyor.yml
+
+
+## tests need specific hostnames to be available
+## note, this is insufficient on MacOS, and needs to be reflected
+## in the SSH config too
+#hosts:
+#  datalad-test-sshd: 127.0.0.1
+#  # same, but for datalad-core implementations
+#  datalad-test: 127.0.0.1
+
+
+# job-specific configurations
+for:
+  #
+  # POSIX TEST RUNS
+  #
+  - matrix:
+      only:
+        - job_name: test-linux
+        - job_name: test-linux-crippled
+        - job_name: test-mac
+
+    cache:
+      # pip cache
+      - /home/appveyor/.cache/pip -> .appveyor.yml
+
+    # init cannot use any components from the repo, because it runs prior to
+    # cloning it
+    init:
+      # enable external SSH access to CI worker
+      # needs APPVEYOR_SSH_KEY defined in project settings (or environment)
+      - curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -
+      # Scratch space
+      # we place the "unix" one into the user's HOME to avoid git-annex issues on MacOSX
+      # gh-5291
+      - mkdir ~/DLTMP && export TMPDIR=~/DLTMP
+
+    install:
+      # verify that a PY variable is declared that identifies the desired Python version
+      # for this run
+      - "[ \"x$PY\" != x ]"
+      # Missing system software
+      - tools/appveyor/install-syspkgs $INSTALL_SYSPKGS
+      # If a particular Python version is requested, use env setup (using the
+      # appveyor provided environments/installation). Note, these are broken
+      # on the ubuntu images
+      # https://help.appveyor.com/discussions/problems/28217-appveyor-ubunu-image-with-python3-lzma-module
+      # Otherwise create a virtualenv using the default Python 3, to enable uniform
+      # use of python/pip executables below
+      - "[ \"x$PY\" != x ] && . ${HOME}/venv${PY}/bin/activate || virtualenv -p 3 ${HOME}/dlvenv && . ${HOME}/dlvenv/bin/activate; ln -s \"$VIRTUAL_ENV\" \"${HOME}/VENV\""
+      - tools/appveyor/install-git-annex ${INSTALL_GITANNEX}
+      # enable the git-annex provisioned by the installer
+      - "[ -f ${HOME}/dlinstaller_env.sh ] && . ${HOME}/dlinstaller_env.sh || true"
+
+    test_script:
+      # store original TMPDIR setting to limit modification to test execution
+      - export PREV_TMPDIR=$TMPDIR
+      # make TMPDIR a "crippled filesystem" to test wrong assumptions of POSIX-ness
+      # on POSIX OSes. The test fixtures will create all test datasets under TMPDIR
+      - |
+        set -e
+        if [ "$APPVEYOR_JOB_NAME" = "test-linux-crippled" ]; then
+          # 100 MB VFAT FS in a box
+          sudo dd if=/dev/zero of=/crippledfs.img count=100 bs=1M
+          sudo mkfs.vfat /crippledfs.img
+          sudo mkdir /crippledfs
+          sudo mount -o "uid=$(id -u),gid=$(id -g)" /crippledfs.img /crippledfs
+          echo "== mount >>"
+          mount | grep crippled
+          echo "<< mount =="
+          export TMPDIR=/crippledfs
+        fi
+      - echo TMPDIR=$TMPDIR
+      - hatch test --cover --doctest-modules
+
+    after_test:
+      - |
+        [ -n "$CODECOV_BINARY"] \
+        && (python -m coverage xml && curl -Os "$CODECOV_BINARY" && chmod +x codecov && ./codecov) \
+        || true
+
+    on_finish:
+      # conditionally block the exit of a CI run for direct debugging
+      - while [ -f ~/BLOCK ]; do sleep 5; done
+
+
+  #
+  # WINDOWS TEST RUNS
+  #
+  - matrix:
+      only:
+        - job_name: test-win
+    cache:
+      # pip cache
+      - C:\Users\appveyor\AppData\Local\pip\Cache -> .appveyor.yml
+
+    # init cannot use any components from the repo, because it runs prior to
+    # cloning it
+    init:
+      # remove windows 260-char limit on path names
+      - ps: Set-Itemproperty -path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name LongPathsEnabled -value 1
+      # enable developer mode on windows
+      # this should enable mklink without admin privileges, but it doesn't seem to work
+      #- ps: tools\ci\appveyor_enable_windevmode.ps1
+      # enable RDP access on windows (RDP password is in appveyor project config)
+      # this is relatively expensive (1-2min), but very convenient to jump into any build at any time
+      - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
+      # Scratch space
+      - cmd: md C:\DLTMP
+      # and use that scratch space to get short paths in test repos
+      # (avoiding length-limits as much as possible)
+      - cmd: "set TMP=C:\\DLTMP"
+      - cmd: "set TEMP=C:\\DLTMP"
+
+    install:
+      # place a debug setup helper at a convenient location
+      - cmd: copy tools\appveyor\env_setup.bat C:\\datalad_debug.bat
+      - cmd: "set PATH=C:\\Python%PY%;C:\\Python%PY%\\Scripts;%PATH%"
+      # deploy the datalad installer, override version via DATALAD_INSTALLER_VERSION
+      - cmd:
+          IF DEFINED DATALAD_INSTALLER_VERSION (
+          python -m pip install "datalad-installer%DATALAD_INSTALLER_VERSION%"
+          ) ELSE (
+          python -m pip install datalad-installer
+          )
+      # Install git-annex on windows, otherwise INSTALL_SYSPKGS can be used
+      # deploy git-annex, if desired
+      - cmd: IF DEFINED INSTALL_GITANNEX datalad-installer --sudo ok %INSTALL_GITANNEX%
+
+    test_script:
+      - hatch test --cover
+
+    after_test:
+      - cmd: python -m coverage xml
+      - cmd: curl -fsSL -o codecov.exe "https://uploader.codecov.io/latest/windows/codecov.exe"
+      - cmd: .\codecov.exe -f "coverage.xml"
+
+    on_finish:
+      # conditionally block the exit of a CI run for direct debugging
+      - ps: while ((Test-Path "C:\Users\\appveyor\\Desktop\\BLOCK.txt")) { Start-Sleep 5 }
+
+
+#
+# ALL TEST RUNS
+#
+build_script:
+  # install the pieces that we need in this CI run
+  - python -m pip install hatch coverage
+
+after_build:
+  # Identity setup
+  - git config --global user.email "[email protected]"
+  - git config --global user.name "Appveyor Almighty"

datalad_core/tests/test_dummy.py

@@ -0,0 +1,6 @@
+import datalad_core  # noqa: F401
+
+
+def test_dummy():
+    # nothing but a placeholder
+    pass

tools/appveyor/env_setup.bat

@@ -0,0 +1,4 @@
+set PY=%1-x64
+set TMP=C:\DLTMP
+set TEMP=C:\DLTMP
+set PATH=C:\Python%PY%;C:\Python%PY%\Scripts;%PATH%

tools/appveyor/install-git-annex

@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+# Install git-annex. Any environment setup to source would be
+# written to ${HOME}/dlinstaller_env.sh
+#
+set -e -u
+
+# no install requested -> exit
+[ -z "$1" ] && exit 0 || true
+
+# assumes a virtualenv or equivalent python env
+# get the installer for this
+python -m pip install datalad-installer${DATALAD_INSTALLER_VERSION:-}
+datalad-installer -E ${HOME}/dlinstaller_env.sh --sudo ok $*

tools/appveyor/install-syspkgs

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+# no install requested -> exit
+[ -z "$1" ] && exit 0 || true
+
+if (which apt-get > /dev/null ); then
+	sudo apt-get update -qq -y --allow-releaseinfo-change
+	sudo apt-get install -q --no-install-recommends -y eatmydata
+	sudo eatmydata apt-get install -q --no-install-recommends -y $*
+else
+	brew install -q $*
+fi