diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 0000000..635c978 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,254 @@ +# This CI setup provides a largely homogeneous configuration across all +# major platforms (Windows, MacOS, and Linux). The aim of this test setup is +# to create a "native" platform experience, using as few cross-platform +# helper tools as possible. +# +# On Linux/Mac a virtualenv is used for testing. The effective virtual env +# is available under ~/VENV. +# +# All workers support remote login. Login details are shown at the top of each +# CI run log. +# +# - Linux/Mac workers (via SSH): +# +# - A permitted SSH key must be defined in an APPVEYOR_SSH_KEY environment +# variable (via the appveyor project settings) +# +# - SSH login info is given in the form of: 'appveyor@67.225.164.xx -p 22xxx' +# +# - Login with: +# +# ssh -o StrictHostKeyChecking=no +# +# - to prevent the CI run from exiting, `touch` a file named `BLOCK` in the +# user HOME directory (current directory directly after login). The session +# will run until the file is removed (or 60 min have passed) +# +# - Windows workers (via RDP): +# +# - An RDP password should be defined in an APPVEYOR_RDP_PASSWORD environment +# variable (via the appveyor project settings), or a random password is used +# every time +# +# - RDP login info is given in the form of IP:PORT +# +# - Login with: +# +# xfreerdp /cert:ignore /dynamic-resolution /u:appveyor /p: /v: +# +# - to prevent the CI run from exiting, create a textfile named `BLOCK` on the +# Desktop (a required .txt extension will be added automatically). The session +# will run until the file is removed (or 60 min have passed) +# +# - in a terminal execute, for example, `C:\datalad_debug.bat 39` to set up the +# environment to debug in a Python 3.8 session (should generally match the +# respective CI run configuration). + + +# do not make repository clone cheap: interfers with VCS-based version determination +shallow_clone: false + +# turn of support for MS project build support (not needed) +build: off + +environment: + # Do not use `image` as a matrix dimension, to have fine-grained control over + # what tests run on which platform + # The ID variable had no impact, but sorts first in the CI run overview + # an intelligible name can help to locate a specific test run + matrix: + # List a CI run for each platform first, to have immediate access when there + # is a need for debugging + + # Ubuntu core tests + - job_name: test-linux + APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004 + PY: 3.9 + INSTALL_GITANNEX: git-annex -m snapshot + CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov + + # same as 'test-linux', but TMPDIR is on a crippled filesystem, causing + # most, if not all test datasets to be created on that filesystem + - job_name: test-linux-crippled + APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2204 + PY: 3.9 + # datalad-annex git remote needs something after git-annex_8.20211x + INSTALL_GITANNEX: git-annex -m snapshot + CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov + + # Windows core tests + - job_name: test-win + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022 + # Python version specification is non-standard on windows + PY: 39-x64 + INSTALL_GITANNEX: git-annex -m datalad/packages + + # MacOS core tests + - job_name: test-mac + APPVEYOR_BUILD_WORKER_IMAGE: macos-sonoma + PY: 3.9 + INSTALL_GITANNEX: git-annex + # codecov is broken: https://github.com/codecov/feedback/issues/141 + #CODECOV_BINARY: https://uploader.codecov.io/latest/macos/codecov + + +# only run the CI if there are code or tooling changes +only_commits: + files: + - datalad_core/ + - tools/ + - pyproject.toml + - .appveyor.yml + + +## tests need specific hostnames to be available +## note, this is insufficient on MacOS, and needs to be reflected +## in the SSH config too +#hosts: +# datalad-test-sshd: 127.0.0.1 +# # same, but for datalad-core implementations +# datalad-test: 127.0.0.1 + + +# job-specific configurations +for: + # + # POSIX TEST RUNS + # + - matrix: + only: + - job_name: test-linux + - job_name: test-linux-crippled + - job_name: test-mac + + cache: + # pip cache + - /home/appveyor/.cache/pip -> .appveyor.yml + + # init cannot use any components from the repo, because it runs prior to + # cloning it + init: + # enable external SSH access to CI worker + # needs APPVEYOR_SSH_KEY defined in project settings (or environment) + - curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - + # Scratch space + # we place the "unix" one into the user's HOME to avoid git-annex issues on MacOSX + # gh-5291 + - mkdir ~/DLTMP && export TMPDIR=~/DLTMP + + install: + # verify that a PY variable is declared that identifies the desired Python version + # for this run + - "[ \"x$PY\" != x ]" + # Missing system software + - tools/appveyor/install-syspkgs $INSTALL_SYSPKGS + # If a particular Python version is requested, use env setup (using the + # appveyor provided environments/installation). Note, these are broken + # on the ubuntu images + # https://help.appveyor.com/discussions/problems/28217-appveyor-ubunu-image-with-python3-lzma-module + # Otherwise create a virtualenv using the default Python 3, to enable uniform + # use of python/pip executables below + - "[ \"x$PY\" != x ] && . ${HOME}/venv${PY}/bin/activate || virtualenv -p 3 ${HOME}/dlvenv && . ${HOME}/dlvenv/bin/activate; ln -s \"$VIRTUAL_ENV\" \"${HOME}/VENV\"" + - tools/appveyor/install-git-annex ${INSTALL_GITANNEX} + # enable the git-annex provisioned by the installer + - "[ -f ${HOME}/dlinstaller_env.sh ] && . ${HOME}/dlinstaller_env.sh || true" + + test_script: + # store original TMPDIR setting to limit modification to test execution + - export PREV_TMPDIR=$TMPDIR + # make TMPDIR a "crippled filesystem" to test wrong assumptions of POSIX-ness + # on POSIX OSes. The test fixtures will create all test datasets under TMPDIR + - | + set -e + if [ "$APPVEYOR_JOB_NAME" = "test-linux-crippled" ]; then + # 100 MB VFAT FS in a box + sudo dd if=/dev/zero of=/crippledfs.img count=100 bs=1M + sudo mkfs.vfat /crippledfs.img + sudo mkdir /crippledfs + sudo mount -o "uid=$(id -u),gid=$(id -g)" /crippledfs.img /crippledfs + echo "== mount >>" + mount | grep crippled + echo "<< mount ==" + export TMPDIR=/crippledfs + fi + - echo TMPDIR=$TMPDIR + - hatch test --cover --doctest-modules + + after_test: + - | + [ -n "$CODECOV_BINARY"] \ + && (python -m coverage xml && curl -Os "$CODECOV_BINARY" && chmod +x codecov && ./codecov) \ + || true + + on_finish: + # conditionally block the exit of a CI run for direct debugging + - while [ -f ~/BLOCK ]; do sleep 5; done + + + # + # WINDOWS TEST RUNS + # + - matrix: + only: + - job_name: test-win + cache: + # pip cache + - C:\Users\appveyor\AppData\Local\pip\Cache -> .appveyor.yml + + # init cannot use any components from the repo, because it runs prior to + # cloning it + init: + # remove windows 260-char limit on path names + - ps: Set-Itemproperty -path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name LongPathsEnabled -value 1 + # enable developer mode on windows + # this should enable mklink without admin privileges, but it doesn't seem to work + #- ps: tools\ci\appveyor_enable_windevmode.ps1 + # enable RDP access on windows (RDP password is in appveyor project config) + # this is relatively expensive (1-2min), but very convenient to jump into any build at any time + - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + # Scratch space + - cmd: md C:\DLTMP + # and use that scratch space to get short paths in test repos + # (avoiding length-limits as much as possible) + - cmd: "set TMP=C:\\DLTMP" + - cmd: "set TEMP=C:\\DLTMP" + + install: + # place a debug setup helper at a convenient location + - cmd: copy tools\appveyor\env_setup.bat C:\\datalad_debug.bat + - cmd: "set PATH=C:\\Python%PY%;C:\\Python%PY%\\Scripts;%PATH%" + # deploy the datalad installer, override version via DATALAD_INSTALLER_VERSION + - cmd: + IF DEFINED DATALAD_INSTALLER_VERSION ( + python -m pip install "datalad-installer%DATALAD_INSTALLER_VERSION%" + ) ELSE ( + python -m pip install datalad-installer + ) + # Install git-annex on windows, otherwise INSTALL_SYSPKGS can be used + # deploy git-annex, if desired + - cmd: IF DEFINED INSTALL_GITANNEX datalad-installer --sudo ok %INSTALL_GITANNEX% + + test_script: + - hatch test --cover + + after_test: + - cmd: python -m coverage xml + - cmd: curl -fsSL -o codecov.exe "https://uploader.codecov.io/latest/windows/codecov.exe" + - cmd: .\codecov.exe -f "coverage.xml" + + on_finish: + # conditionally block the exit of a CI run for direct debugging + - ps: while ((Test-Path "C:\Users\\appveyor\\Desktop\\BLOCK.txt")) { Start-Sleep 5 } + + +# +# ALL TEST RUNS +# +build_script: + # install the pieces that we need in this CI run + - python -m pip install hatch coverage + +after_build: + # Identity setup + - git config --global user.email "test@appveyor.land" + - git config --global user.name "Appveyor Almighty" diff --git a/datalad_core/tests/test_dummy.py b/datalad_core/tests/test_dummy.py new file mode 100644 index 0000000..55c0cb5 --- /dev/null +++ b/datalad_core/tests/test_dummy.py @@ -0,0 +1,6 @@ +import datalad_core # noqa: F401 + + +def test_dummy(): + # nothing but a placeholder + pass diff --git a/tools/appveyor/env_setup.bat b/tools/appveyor/env_setup.bat new file mode 100644 index 0000000..d464901 --- /dev/null +++ b/tools/appveyor/env_setup.bat @@ -0,0 +1,4 @@ +set PY=%1-x64 +set TMP=C:\DLTMP +set TEMP=C:\DLTMP +set PATH=C:\Python%PY%;C:\Python%PY%\Scripts;%PATH% diff --git a/tools/appveyor/install-git-annex b/tools/appveyor/install-git-annex new file mode 100755 index 0000000..6840c39 --- /dev/null +++ b/tools/appveyor/install-git-annex @@ -0,0 +1,14 @@ +#!/bin/bash +# +# Install git-annex. Any environment setup to source would be +# written to ${HOME}/dlinstaller_env.sh +# +set -e -u + +# no install requested -> exit +[ -z "$1" ] && exit 0 || true + +# assumes a virtualenv or equivalent python env +# get the installer for this +python -m pip install datalad-installer${DATALAD_INSTALLER_VERSION:-} +datalad-installer -E ${HOME}/dlinstaller_env.sh --sudo ok $* diff --git a/tools/appveyor/install-syspkgs b/tools/appveyor/install-syspkgs new file mode 100755 index 0000000..bfb84a9 --- /dev/null +++ b/tools/appveyor/install-syspkgs @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +# no install requested -> exit +[ -z "$1" ] && exit 0 || true + +if (which apt-get > /dev/null ); then + sudo apt-get update -qq -y --allow-releaseinfo-change + sudo apt-get install -q --no-install-recommends -y eatmydata + sudo eatmydata apt-get install -q --no-install-recommends -y $* +else + brew install -q $* +fi