Feature/covalent

.github/workflows/cloud-ci.yml

@@ -0,0 +1,109 @@
+name: tests
+
+on:
+  # Runs for pull requests
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  could-tests:
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - arch: cuda
+            exclude: "no-cuda"
+            run_on: ec2
+          # - arch: rocm
+          #   exclude : "no-rocm"
+
+    runs-on: ubuntu-latest
+    environment: cloud-ci
+
+    # Cancel previous jobs if a new version was pushed
+    concurrency:
+      group: "${{ github.ref }}-${{ matrix.arch }}-${{ matrix.run_on }}"
+      cancel-in-progress: true
+
+    defaults:
+      run:
+        shell: bash -el {0}
+
+    env:
+      MILABENCH_CONFIG: "config/test.yaml"
+      MILABENCH_SYSTEM: "config/examples/${{ matrix.run_on }}-system.yaml"
+      MILABENCH_BASE: "output"
+      MILABENCH_ARGS: ""
+      MILABENCH_GPU_ARCH: "${{ matrix.arch }}"
+      MILABENCH_DASH: "no"
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.REPORTS_PAT }}
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: dependencies
+        run: |
+          python -m pip install -U pip
+          python -m pip install -U poetry
+          poetry lock --no-update
+          poetry install
+
+      - name: setup cloud credentials
+        run: |
+          mkdir -p ~/.aws
+          mkdir -p ~/.ssh/covalent
+          echo "${{ secrets.COVALENT_EC2_EXECUTOR_KEYPAIR }}" >~/.ssh/covalent/covalent-ec2-executor-keypair.pem
+          echo "[default]" >~/.aws/credentials
+          echo "aws_access_key_id=${{ secrets.AWS_ACCESS_KEY_ID }}" >>~/.aws/credentials
+          echo "aws_secret_access_key=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >>~/.aws/credentials
+          chmod -R a-rwx,u+rwX ~/.aws ~/.ssh
+
+      - name: setup cloud
+        run: |
+          _system=$(
+          poetry run milabench cloud \
+            --setup \
+            --run-on ${{ matrix.run_on }}
+          )
+          { read _hash ; }< <(
+          echo -n "$_system" | while read l
+          do
+              if [[ "$l" == "# hash::>"* ]]
+              then
+                  echo -n "${l#*::>}"
+              fi
+          done
+          echo
+          )
+          if [[ -z "${_hash}" ]]
+          then
+            >&2 echo "Failed to fetch system config hash"
+            exit 1
+          fi
+          echo -n "$_system" >$MILABENCH_SYSTEM.$_hash
+          echo "MILABENCH_SYSTEM=$MILABENCH_SYSTEM.$_hash" >>$GITHUB_ENV
+
+      - name: install benchmarks
+        run: |
+          poetry run milabench install
+
+      - name: prepare benchmarks
+        run: |
+          poetry run milabench prepare
+
+      - name: run benchmarks
+        run: |
+          poetry run milabench run
+
+      - name: Summary
+        run: |
+          git remote set-url origin "https://${{ vars.REPORTS_USERNAME }}:${{ secrets.REPORTS_PAT }}@$(git remote get-url origin | cut -d'/' -f3-)"
+          git config --global user.email "[email protected]"
+          git config --global user.name "GitHub CI"
+          poetry run milabench report --push

benchmarks/_template/requirements.cpu.txt

@@ -0,0 +1,46 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile --output-file=benchmarks/_template/requirements.cpu.txt benchmarks/_template/requirements.in
+#
+antlr4-python3-runtime==4.9.3
+    # via omegaconf
+asttokens==2.4.1
+    # via giving
+codefind==0.1.3
+    # via ptera
+executing==1.2.0
+    # via varname
+giving==0.4.2
+    # via
+    #   ptera
+    #   voir
+markdown-it-py==3.0.0
+    # via rich
+mdurl==0.1.2
+    # via markdown-it-py
+omegaconf==2.3.0
+    # via voir
+ovld==0.3.2
+    # via voir
+ptera==1.4.1
+    # via voir
+pygments==2.17.2
+    # via rich
+pynvml==11.5.0
+    # via voir
+pyyaml==6.0.1
+    # via omegaconf
+reactivex==4.0.4
+    # via giving
+rich==13.7.0
+    # via voir
+six==1.16.0
+    # via asttokens
+typing-extensions==4.10.0
+    # via reactivex
+varname==0.10.0
+    # via giving
+voir==0.2.12
+    # via -r benchmarks/_template/requirements.in

config/examples/ec2-system.yaml

@@ -0,0 +1,19 @@
+system:
+  # Nodes list
+  nodes:
+      # Alias used to reference the node
+    - name: manager
+      # Use 1.1.1.1 as an ip placeholder
+      ip: 1.1.1.1
+      # Use this node as the master node or not
+      main: true
+      # User to use in remote milabench operations
+      user: user
+
+  # Cloud instances profiles
+  cloud_profiles:
+    ec2:
+      username: ubuntu
+      instance_type: t2.micro
+      volume_size: 8
+      region: us-east-2

config/test.yaml

@@ -0,0 +1,24 @@
+_defaults:
+  max_duration: 600
+  voir:
+    options:
+      stop: 60
+      interval: "1s"
+
+test:
+  inherits: _defaults
+  group: test_remote
+  install_group: test_remote
+  definition: ../benchmarks/_template
+  plan:
+    method: njobs
+    n: 1
+
+testing:
+  inherits: _defaults
+  definition: ../benchmarks/_template
+  group: test_remote_2
+  install_group: test_remote_2
+  plan:
+    method: njobs
+    n: 1

milabench/__init__.py

@@ -0,0 +1,5 @@
+import pathlib
+
+ROOT_FOLDER = pathlib.Path(__file__).resolve().parent.parent
+CONFIG_FOLDER = ROOT_FOLDER / "config"
+BENCHMARK_FOLDER = ROOT_FOLDER / "benchmarks"

milabench/cli/__init__.py

@@ -3,6 +3,7 @@
 
 from coleo import run_cli
 
+from .cloud import cli_cloud
 from .compare import cli_compare
 from .dev import cli_dev
 from .install import cli_install
@@ -37,6 +38,10 @@ def pin():
         """Pin the benchmarks' dependencies."""
         cli_pin()
 
+    def cloud():
+        """Setup cloud instances."""
+        cli_cloud()
+
     def dev():
         """Create a shell in a benchmark's environment for development."""
         cli_dev()

milabench/cli/badges/__main__.py

@@ -0,0 +1,45 @@
+import pathlib
+import subprocess
+import sys
+
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv[1:]
+
+    try:
+        import pybadges as _
+    except ImportError:
+        module = pathlib.Path(__file__).resolve().parent
+        cache_dir = pathlib.Path(f"/tmp/milabench/{module.name}_venv")
+        python3 = str(cache_dir / "bin/python3")
+        check_module = "import pybadges"
+        try:
+            subprocess.run([python3, "-c", check_module], check=True)
+        except (FileNotFoundError, subprocess.CalledProcessError):
+            cache_dir.mkdir(parents=True, exist_ok=True)
+            subprocess.run([sys.executable, "-m", "virtualenv", str(cache_dir)], check=True)
+            subprocess.run([python3, "-m", "pip", "install", "-U", "pip"], check=True)
+            subprocess.run([
+                python3,
+                "-m",
+                "pip",
+                "install",
+                "-r",
+                str(module / "requirements.txt")
+            ], check=True)
+            subprocess.run([python3, "-c", check_module], check=True)
+        return subprocess.call(
+            [python3, __file__, *argv],
+        )
+
+    return subprocess.run([
+        sys.executable,
+        "-m",
+        "pybadges",
+        *argv
+    ], check=True).returncode
+
+
+if __name__ == "__main__":
+    sys.exit(main())

milabench/cli/badges/requirements.txt

@@ -0,0 +1 @@
+pybadges