From 0a8c1b6f899b33bbae25ddc4fdd08ef5077a8462 Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Wed, 7 Feb 2024 10:06:19 +0000 Subject: [PATCH 01/13] install poppunk from conda in gha, and use python 3.10 --- .github/workflows/pytest.yml | 34 +++++----------------------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 54177b35..3fba8ddc 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -34,15 +34,15 @@ jobs: run: | ./scripts/download_db --small storage - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v2 with: - python-version: "3.9" + python-version: "3.10" - name: Setup Miniconda uses: conda-incubator/setup-miniconda@v2.1.1 with: - python-version: 3.9 + python-version: 3.10 activate-environment: beebop_py channels: bioconda, conda-forge @@ -64,32 +64,8 @@ jobs: CACHE_NUMBER: 0 id: cache-conda -# - name: Install poppunk -# run: conda install poppunk -# currently the latest poppunk release is missing some functions required for beebop. For now installing from source: - - - name: Get poppunk source code - uses: actions/checkout@v3 - with: - repository: bacpop/PopPUNK - ref: fix-json-serialisation - path: poppunk - - - name: install poppunk & dependencies - working-directory: ./poppunk - run: | - source $CONDA/etc/profile.d/conda.sh - conda activate beebop_py - conda install graph-tool - conda install mandrake - conda install rapidnj - sudo apt-get update - sudo apt-get install libeigen3-dev libegl1 - sudo apt-get install libopenblas-dev - sudo apt-get install -y '^libxcb.*-dev' libx11-xcb-dev libglu1-mesa-dev libxrender-dev libxi-dev libxkbcommon-dev libxkbcommon-x11-dev - pip install joblib==1.1.0 - conda install -c bioconda pp-sketchlib=2.0.0 - pip3 install git+https://github.com/bacpop/PopPUNK + - name: Install poppunk + run: conda install poppunk - name: Install Poetry uses: snok/install-poetry@v1 From 42d4e6e2efcb6aadd1afdfeee0455ab72c23af5c Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Wed, 7 Feb 2024 10:12:33 +0000 Subject: [PATCH 02/13] manual cache update --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 3fba8ddc..18c623a1 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -61,7 +61,7 @@ jobs: steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER}} env: # Increment this value to manually reset cache. - CACHE_NUMBER: 0 + CACHE_NUMBER: 1 id: cache-conda - name: Install poppunk From 8c0c3e99949e7d7875c46cea30044bb6348a3cab Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Wed, 7 Feb 2024 10:18:05 +0000 Subject: [PATCH 03/13] separate Python 3.10 install --- .github/workflows/pytest.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 18c623a1..2abd152a 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -42,7 +42,7 @@ jobs: - name: Setup Miniconda uses: conda-incubator/setup-miniconda@v2.1.1 with: - python-version: 3.10 + python-version: 3.9 activate-environment: beebop_py channels: bioconda, conda-forge @@ -61,9 +61,12 @@ jobs: steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER}} env: # Increment this value to manually reset cache. - CACHE_NUMBER: 1 + CACHE_NUMBER: 0 id: cache-conda + - name: Install Python 3.10 + run: conda install python=3.10 + - name: Install poppunk run: conda install poppunk From 708631905414a87e1fe23af4467fef114e09cb15 Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Wed, 7 Feb 2024 10:24:28 +0000 Subject: [PATCH 04/13] skip cache restore for now --- .github/workflows/pytest.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 2abd152a..3dc661b7 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -51,18 +51,18 @@ jobs: run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT shell: bash - - name: Restore Conda env from cache - uses: actions/cache@v2 - with: - path: /usr/share/miniconda/envs/beebop_py - key: - # Refresh cache each day - conda-${{ runner.os }}--${{ runner.arch }}--${{ - steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER}} - env: - # Increment this value to manually reset cache. - CACHE_NUMBER: 0 - id: cache-conda + #- name: Restore Conda env from cache + # uses: actions/cache@v2 + # with: + # path: /usr/share/miniconda/envs/beebop_py + # key: + # # Refresh cache each day + # conda-${{ runner.os }}--${{ runner.arch }}--${{ + # steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER}} + # env: + # # Increment this value to manually reset cache. + # CACHE_NUMBER: 0 + # id: cache-conda - name: Install Python 3.10 run: conda install python=3.10 From c6c36691f31a08ede690e4a1d326b7ff95247f03 Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Wed, 7 Feb 2024 10:36:14 +0000 Subject: [PATCH 05/13] install gl --- .github/workflows/pytest.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 3dc661b7..65b7f1a1 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -64,8 +64,10 @@ jobs: # CACHE_NUMBER: 0 # id: cache-conda - - name: Install Python 3.10 - run: conda install python=3.10 + - name: Install poppunk dependencies + run: | + conda install python=3.10 + apt-get update && apt-get install libgl1 - name: Install poppunk run: conda install poppunk From 4a58d463b5f1abef7f0f078189ed9a54ab141448 Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Wed, 7 Feb 2024 10:40:04 +0000 Subject: [PATCH 06/13] sudo.. --- .github/workflows/pytest.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 65b7f1a1..48749c20 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -67,7 +67,8 @@ jobs: - name: Install poppunk dependencies run: | conda install python=3.10 - apt-get update && apt-get install libgl1 + sudo apt-get update + sudo apt-get install libgl1 - name: Install poppunk run: conda install poppunk From a0de1fac35df1f051bd75814b8124a5cde559600 Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Wed, 7 Feb 2024 16:27:31 +0000 Subject: [PATCH 07/13] Python 3.9 bas --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 48749c20..ca45e12e 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -37,7 +37,7 @@ jobs: - name: Set up Python 3.10 uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.9" - name: Setup Miniconda uses: conda-incubator/setup-miniconda@v2.1.1 From a6b42f26f66e6ff3bef19b71e238369279332da5 Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Wed, 7 Feb 2024 16:37:43 +0000 Subject: [PATCH 08/13] install libglib2 --- .github/workflows/pytest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index ca45e12e..14913f04 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -34,7 +34,7 @@ jobs: run: | ./scripts/download_db --small storage - - name: Set up Python 3.10 + - name: Set up Python 3.9 uses: actions/setup-python@v2 with: python-version: "3.9" @@ -68,7 +68,7 @@ jobs: run: | conda install python=3.10 sudo apt-get update - sudo apt-get install libgl1 + sudo apt-get install libglib2.0-0 - name: Install poppunk run: conda install poppunk From 655322d09d83d38bb1bda9768d90efe15d9f9d62 Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Wed, 7 Feb 2024 16:46:01 +0000 Subject: [PATCH 09/13] install libsm6 --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 14913f04..96273077 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -68,7 +68,7 @@ jobs: run: | conda install python=3.10 sudo apt-get update - sudo apt-get install libglib2.0-0 + sudo apt-get install ffmpeg libsm6 libxext6 -y - name: Install poppunk run: conda install poppunk From e412d090fd85bd8f3d7560862a4b0a28fae5040c Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Wed, 7 Feb 2024 16:53:04 +0000 Subject: [PATCH 10/13] install linux packages first --- .github/workflows/pytest.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 96273077..fa56c5da 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -33,6 +33,11 @@ jobs: working-directory: ./main run: | ./scripts/download_db --small storage + + - name: Install Linux packages + run: | + sudo apt-get update + sudo apt-get install ffmpeg libsm6 libxext6 -y - name: Set up Python 3.9 uses: actions/setup-python@v2 @@ -67,8 +72,6 @@ jobs: - name: Install poppunk dependencies run: | conda install python=3.10 - sudo apt-get update - sudo apt-get install ffmpeg libsm6 libxext6 -y - name: Install poppunk run: conda install poppunk From 8b60fbfb83ae2facc6caf236d52cf09dda7ee22d Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Wed, 7 Feb 2024 16:58:25 +0000 Subject: [PATCH 11/13] try without setup-python --- .github/workflows/pytest.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index fa56c5da..3c4758dc 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -39,15 +39,15 @@ jobs: sudo apt-get update sudo apt-get install ffmpeg libsm6 libxext6 -y - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: "3.9" + #- name: Set up Python 3.9 + # uses: actions/setup-python@v2 + # with: + # python-version: "3.9" - name: Setup Miniconda uses: conda-incubator/setup-miniconda@v2.1.1 with: - python-version: 3.9 + python-version: 3.10 activate-environment: beebop_py channels: bioconda, conda-forge From eef82821ff1073d95580221af5f5e5a1c9485b41 Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Wed, 7 Feb 2024 17:05:10 +0000 Subject: [PATCH 12/13] python version string --- .github/workflows/pytest.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 3c4758dc..26ef0c12 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -39,15 +39,15 @@ jobs: sudo apt-get update sudo apt-get install ffmpeg libsm6 libxext6 -y - #- name: Set up Python 3.9 - # uses: actions/setup-python@v2 - # with: - # python-version: "3.9" + - name: Set up Python 3.10 + uses: actions/setup-python@v2 + with: + python-version: "3.10" - name: Setup Miniconda uses: conda-incubator/setup-miniconda@v2.1.1 with: - python-version: 3.10 + python-version: "3.10" activate-environment: beebop_py channels: bioconda, conda-forge From 4f77070faa2207b7d7052ac16d6989ee16554716 Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Wed, 7 Feb 2024 17:21:00 +0000 Subject: [PATCH 13/13] mimic dockerfile installs --- .github/workflows/pytest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 26ef0c12..fcb2f107 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -36,8 +36,8 @@ jobs: - name: Install Linux packages run: | - sudo apt-get update - sudo apt-get install ffmpeg libsm6 libxext6 -y + sudo apt-get update -q + sudo apt-get install -q -y --no-install-recommends bzip2 ca-certificates git libglib2.0-0 libsm6 libxext6 libxrender1 mercurial openssh-client procps subversion wget - name: Set up Python 3.10 uses: actions/setup-python@v2