feat: devcontainer development environment

.devcontainer/Dockerfile.dev

@@ -0,0 +1,79 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.231.6/containers/python-3/.devcontainer/base.Dockerfile
+
+# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
+ARG VARIANT="3.10-bullseye"
+FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT} as development_build
+
+ARG YOUR_ENV
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONFAULTHANDLER=1 \
+    PYTHONHASHSEED=random \
+    PYTHONDONTWRITEBYTECODE=1 \
+    # pip
+    # PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    # PIP_DEFAULT_TIMEOUT=100 \
+    PIP_ROOT_USER_ACTION=ignore \
+    # Poetry
+    # https://python-poetry.org/docs/configuration/#using-environment-variables
+    POETRY_VERSION=1.7.1 \
+    # make poetry install to this location
+    POETRY_HOME="/usr/local" \
+    # do not ask any interactive question
+    POETRY_NO_INTERACTION=1 \
+    # cahce dir for poetry
+    POETRY_CACHE_DIR='/var/cache/pypoetry' \
+    # never create virtual environment automatically, only use env prepared by us
+    POETRY_VIRTUALENVS_CREATE=false
+
+
+# System deps (we don't use exact versions because it is hard to update them,
+# pin when needed):
+# hadolint ignore=DL3008
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    default-jdk \
+    htop \
+    fzf \
+    jq \
+    bat \
+    # Installing `poetry` package manager:
+    # https://github.com/python-poetry/poetry
+    && curl -sSL 'https://install.python-poetry.org' | python - \
+    # Enable tab completion for bash
+    && poetry completions bash >> /home/vscode/.bash_completion \
+    # Enable tab completion for Zsh
+    && mkdir -p /home/vscode/.zfunc/ \
+    && poetry completions zsh > /home/vscode/.zfunc/_poetry \
+    && echo "fpath+=~/.zfunc\nautoload -Uz compinit && compinit" >> /home/vscode/.zshrc \
+    && poetry --version \
+    && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
+    && apt-get clean -y && rm -rf /var/lib/apt/lists/*
+
+# Add fzf to .zshrc
+RUN echo "source /usr/share/doc/fzf/examples/key-bindings.zsh" >> /home/vscode/.zshrc && \
+    echo "source /usr/share/doc/fzf/examples/completion.zsh" >> /home/vscode/.zshrc
+
+ARG USERNAME=vscode
+# Used to persist bash history as per https://code.visualstudio.com/remote/advancedcontainers/persist-bash-history
+# hadolint ignore=SC2086
+RUN SNIPPET="export PROMPT_COMMAND='history -a' && export HISTFILE=/commandhistory/.zsh_history" \
+    && mkdir /commandhistory \
+    && touch /commandhistory/.zsh_history \
+    && chown -R $USERNAME /commandhistory \
+    && echo "$SNIPPET" >> "/home/$USERNAME/.zshrc"
+
+# working directory
+WORKDIR /workspaces
+
+# Copy only requirements, to cache them in docker layer
+COPY ./poetry.lock ./pyproject.toml ./README.md ./
+COPY src/gentropy src/gentropy
+COPY src/utils src/utils
+
+# Install runtime deps:
+# hadolint ignore=SC2046
+RUN --mount=type=cache,target="$POETRY_CACHE_DIR" \
+    poetry install --no-interaction --no-ansi
+
+EXPOSE 8080

.devcontainer/devcontainer.json

@@ -0,0 +1,78 @@
+{
+  "name": "OTGdev",
+  "build": {
+    "dockerfile": "Dockerfile.dev",
+    "context": "..",
+    "args": {
+      "VARIANT": "3.10-bullseye",
+      "POETRY_VERSION": "1.7.1"
+    }
+  },
+  "features": {
+    "ghcr.io/devcontainers/features/github-cli:1": {},
+    "ghcr.io/dhoeric/features/google-cloud-cli:1": {},
+    "ghcr.io/mikaello/devcontainer-features/modern-shell-utils:1": {},
+    "ghcr.io/devcontainers-contrib/features/pre-commit:2": {},
+    "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}
+  },
+  "mounts": [
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind",
+    "source=devcontainer-bashhistory,target=/commandhistory,type=volume",
+    "source=${localEnv:HOME}/.gitconfig,target=/home/vscode/.gitconfig,type=bind,consistency=cached",
+    "source=${localEnv:HOME}/.config/gcloud,target=/home/vscode/.config/gcloud,type=bind,consistency=cached"
+  ],
+  "runArgs": ["-e", "GIT_EDITOR=code --wait"],
+  "postCreateCommand": "bash .devcontainer/postCreateCommand.sh",
+  "customizations": {
+    "vscode": {
+      // Please keep this file in sync with settings in /.vscode/extensions.json
+      "extensions": [
+        "charliermarsh.ruff",
+        "ms-python.mypy-type-checker",
+        "ms-python.python",
+        "esbenp.prettier-vscode",
+        "mhutchie.git-graph",
+        "eamodio.gitlens",
+        "ms-azuretools.vscode-docker",
+        "github.vscode-github-actions",
+        "timonwong.shellcheck",
+        "GitHub.copilot",
+        "vivaxy.vscode-conventional-commits",
+        "ms-vscode-remote.remote-ssh",
+        "ms-toolsai.jupyter",
+        "ms-vscode.makefile-tools",
+        "GitHub.vscode-pull-request-github",
+        "fnando.linter"
+      ],
+      // Please keep this file in sync with settings in /.vscode/settings.json
+      "settings": {
+        "editor.defaultFormatter": "esbenp.prettier-vscode",
+        "terminal.integrated.fontFamily": "Fira Code",
+        "terminal.integrated.defaultProfile.linux": "zsh",
+        "editor.formatOnPaste": false,
+        "editor.formatOnSave": true,
+        "editor.formatOnType": true,
+        "[python]": {
+          "editor.defaultFormatter": "charliermarsh.ruff"
+        },
+        "python.terminal.launchArgs": ["-m", "IPython", "--no-autoindent"],
+        "python.defaultInterpreterPath": "/usr/local/bin/python",
+        "python.testing.pytestEnabled": true,
+        "python.testing.pytestArgs": [
+          "src/",
+          "tests/",
+          "--doctest-modules",
+          "--no-cov"
+        ]
+      }
+    }
+  },
+  "portsAttributes": {
+    "4040": {
+      "label": "SparkUI",
+      "onAutoForward": "notify"
+    }
+  },
+
+  "forwardPorts": [4040]
+}

.devcontainer/postCreateCommand.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# # Install Pre-commit
+pre-commit install --install-hooks

.gitignore

@@ -11,3 +11,4 @@ src/airflow/logs/*
 !src/airflow/logs/.gitkeep
 site/
 .env
+node_modules

.vscode/extensions.json

@@ -4,7 +4,6 @@
     "ms-python.mypy-type-checker",
     "ms-python.python",
     "esbenp.prettier-vscode",
-    "redhat.vscode-yaml",
     "fnando.linter"
   ],
   "unwantedRecommendations": ["ms-python.flake8"]

.vscode/settings.json

@@ -1,29 +1,28 @@
 {
+  "editor.defaultFormatter": "esbenp.prettier-vscode",
+  "terminal.integrated.fontFamily": "Fira Code",
+  "terminal.integrated.defaultProfile.linux": "zsh",
+  "editor.formatOnPaste": false,
+  "editor.formatOnSave": true,
+  "editor.formatOnType": true,
   "[python]": {
     "editor.defaultFormatter": "charliermarsh.ruff",
-    "editor.formatOnPaste": false,
-    "editor.formatOnSave": true,
     "editor.codeActionsOnSave": {
       "source.fixAll": "never",
       "source.organizeImports": "explicit"
     }
   },
   "python.terminal.launchArgs": ["-m", "IPython", "--no-autoindent"],
+  "python.testing.pytestEnabled": true,
+  "python.testing.pytestArgs": [
+    "src/",
+    "tests/",
+    "--doctest-modules",
+    "--cov=src/"
+  ],
   "[jsonc]": {
-    "editor.tabSize": 2,
-    "editor.insertSpaces": true,
-    "editor.detectIndentation": false,
-    "editor.formatOnSave": true,
     "editor.codeActionsOnSave": {
       "source.organizeImports": "explicit"
     }
-  },
-  "json.format.keepLines": true,
-  "autoDocstring.docstringFormat": "google",
-  "python.testing.pytestArgs": [".", "--doctest-modules", "--cov=src/"],
-  "python.testing.unittestEnabled": false,
-  "python.testing.pytestEnabled": true,
-  "mypy-type-checker.severity": {
-    "error": "Information"
   }
 }

README.md

@@ -9,6 +9,7 @@
 <a href="https://opentargets.github.io/gentropy/"><img src="https://github.com/opentargets/gentropy/actions/workflows/release.yaml/badge.svg" alt="image" /></a>
 <a href="https://codecov.io/gh/opentargets/gentropy"><img src="https://codecov.io/gh/opentargets/gentropy/branch/main/graph/badge.svg?token=5ixzgu8KFP" alt="codecov" /></a>
 <a href="https://opensource.org/licenses/Apache-2.0"><img src="https://img.shields.io/badge/License-Apache_2.0-blue.svg" alt="License" /></a>
+<a href="https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/opentargets/gentropy"><img src="https://img.shields.io/badge/Dev%20Containers-Open-blue?logo=visualstudiocode" alt="Dev Containers" /></a>
 <a href="https://doi.org/10.5281/zenodo.10527086"><img src="https://zenodo.org/badge/DOI/10.5281/zenodo.10527086.svg" alt="DOI" /></a>
 </p>
 

docs/development/a_environment.md

@@ -0,0 +1,46 @@
+---
+Title: Environment
+---
+
+# Environment
+
+## Devcontainer (recommended)
+
+Developing in a devcontainer ensures a reproducible development environment with minimal setup. To use the devcontainer, you need to have Docker installed on your system and use Visual Studio Code as your IDE.
+
+For a quick start you can either:
+
+1. [Open existing folder in container](https://code.visualstudio.com/docs/devcontainers/containers#_quick-start-open-an-existing-folder-in-a-container).
+
+1. [Clone repository in container volume](https://code.visualstudio.com/docs/devcontainers/containers#_quick-start-open-a-git-repository-or-github-pr-in-an-isolated-container-volume) [![Open in Dev Containers](https://img.shields.io/static/v1?label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/opentargets/gentropy)
+
+If you already have VS Code and Docker installed, you can click the badge above or [here](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/opentargets/gentropy) to get started. Clicking these links will cause VS Code to automatically install the Dev Containers extension if needed, clone the source code into a container volume, and spin up a dev container for use.
+
+More information on working with devcontainers can be found [here](https://code.visualstudio.com/docs/devcontainers/containers).
+
+!!! note "I/O performance"
+
+    The Dev Containers extension uses "bind mounts" to source code in your local filesystem by default. While this is the simplest option, on macOS and Windows, you may encounter slower disk performance. There are [few things you can do](https://code.visualstudio.com/remote/advancedcontainers/improve-performance) to resolve these type of issues including cloning repository in container volume.
+
+## Codespaces
+
+A devcontainer can also be triggered within Github using [Codespaces](https://github.com/features/codespaces). This option requires no local setup as the environment is managed by Github.
+
+## Local environment
+
+To setup a full local environment of the package please follow the next steps.
+
+Requirements:
+
+- java
+- make
+- [Google Cloud SDK](https://cloud.google.com/sdk/docs/install).
+
+Run `make setup-dev` to install/update the necessary packages and activate the development environment.
+
+!!! info "Google Cloud configuration"
+
+    To complete the Google Cloud configuration, you need to:
+
+    - Log in to your work Google Account: run `gcloud auth login` and follow instructions.
+    - Obtain Google application credentials: run `gcloud auth application-default login` and follow instructions.

docs/development/b_contributing.md

@@ -0,0 +1,34 @@
+---
+Title: Contributing
+---
+
+# Contributing guidelines
+
+## Contributing checklist
+
+When making changes, and especially when implementing a new module or feature, it's essential to ensure that all relevant sections of the code base are modified.
+
+- [ ] Run `make check`. This will run the linter and formatter to ensure that the code is compliant with the project conventions.
+- [ ] Develop unit tests for your code and run `make test`. This will run all unit tests in the repository, including the examples appended in the docstrings of some methods.
+- [ ] Update the configuration if necessary.
+- [ ] Update the documentation and check it with `make build-documentation`. This will start a local server to browse it (URL will be printed, usually `http://127.0.0.1:8000/`)
+
+For more details on each of these steps, see the sections below.
+
+## Documentation
+
+- If during development you had a question which wasn't covered in the documentation, and someone explained it to you, add it to the documentation. The same applies if you encountered any instructions in the documentation which were obsolete or incorrect.
+- Documentation autogeneration expressions start with `:::`. They will automatically generate sections of the documentation based on class and method docstrings. Be sure to update them for:
+  - Dataset definitions in `docs/python_api/datasource/STEP` (example: `docs/python_api/datasource/finngen/study_index.md`)
+  - Step definition in `docs/python_api/step/STEP.md` (example: `docs/python_api/step/finngen.md`)
+
+## Classes
+
+- Dataset class in `src/gentropy/datasource/STEP` (example: `src/gentropy/datasource/finngen/study_index.py` → `FinnGenStudyIndex`)
+- Step main running class in `src/gentropy/STEP.py` (example: `src/gentropy/finngen.py`)
+
+## Tests
+
+- Test study fixture in `tests/conftest.py` (example: `mock_study_index_finngen` in that module)
+- Test sample data in `tests/data_samples` (example: `tests/data_samples/finngen_studies_sample.json`)
+- Test definition in `tests/` (example: `tests/dataset/test_study_index.py` → `test_study_index_finngen_creation`)

docs/development/airflow.md → docs/development/c_airflow.md

@@ -1,3 +1,7 @@
+---
+Title: Airflow
+---
+
 # Airflow configuration
 
 This section describes how to set up a local Airflow server which will orchestrate running workflows in Google Cloud Platform. This is useful for testing and debugging, but for production use, it is recommended to run Airflow on a dedicated server.
@@ -117,6 +121,31 @@ More information on running Airflow with Docker Compose can be found in the [off
 
 1. **Additional pip packages**. They can be added to the `requirements.txt` file.
 
+## Example: A gentropy running DAG using GCP
+
+All pipelines in this repository are intended to be run in Google Dataproc. Running them locally is not currently supported.
+
+In order to run the code:
+
+1. Manually edit your local `src/airflow/dags/*` file and comment out the steps you do not want to run.
+
+2. Manually edit your local `pyproject.toml` file and modify the version of the code.
+
+   - This must be different from the version used by any other people working on the repository to avoid any deployment conflicts, so it's a good idea to use your name, for example: `1.2.3+jdoe`.
+   - You can also add a brief branch description, for example: `1.2.3+jdoe.myfeature`.
+   - Note that the version must comply with [PEP440 conventions](https://peps.python.org/pep-0440/#normalization), otherwise Poetry will not allow it to be deployed.
+   - Do not use underscores or hyphens in your version name. When building the WHL file, they will be automatically converted to dots, which means the file name will no longer match the version and the build will fail. Use dots instead.
+
+3. Manually edit your local `src/airflow/dags/common_airflow.py` and set `OTG_VERSION` to the same version as you did in the previous step.
+
+4. Run `make build`.
+
+   - This will create a bundle containing the neccessary code, configuration and dependencies to run the ETL pipeline, and then upload this bundle to Google Cloud.
+   - A version specific subpath is used, so uploading the code will not affect any branches but your own.
+   - If there was already a code bundle uploaded with the same version number, it will be replaced.
+
+5. Open Airflow UI and run the DAG.
+
 ## Troubleshooting
 
 Note that when you a a new workflow under `dags/`, Airflow will not pick that up immediately. By default the filesystem is only scanned for new DAGs every 300s. However, once the DAG is added, updates are applied nearly instantaneously.