From 122238569e254cecea344b0d99139b3c99680caf Mon Sep 17 00:00:00 2001 From: Sara JC Gosline Date: Mon, 8 Jul 2024 16:27:49 -0700 Subject: [PATCH 1/7] updated docker build to save time --- build/broad_sanger/exp_requirements.r | 9 +++++---- build/broad_sanger/omics_requirements.r | 2 +- build/docker/Dockerfile.broad_sanger_exp | 3 --- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/build/broad_sanger/exp_requirements.r b/build/broad_sanger/exp_requirements.r index 2e4c2275..9cc16e17 100755 --- a/build/broad_sanger/exp_requirements.r +++ b/build/broad_sanger/exp_requirements.r @@ -1,8 +1,9 @@ -install.packages("readr") -install.packages("stringr") -install.packages("dplyr") +#install.packages("readr") +#install.packages("stringr") +#install.packages("dplyr") install.packages('reticulate') -install.packages('tidyr') + #install.packages('tidyr') +install.packages('tidyverse') install.packages("BiocManager") BiocManager::install("PharmacoGx",update=TRUE,ask=FALSE) diff --git a/build/broad_sanger/omics_requirements.r b/build/broad_sanger/omics_requirements.r index 98a4be32..785dc09a 100755 --- a/build/broad_sanger/omics_requirements.r +++ b/build/broad_sanger/omics_requirements.r @@ -5,5 +5,5 @@ install.packages("curl") install.packages("stringr") install.packages("dplyr") install.packages("XML") -install.packages('reticulate') +#install.packages('reticulate') install.packages('tidyr') diff --git a/build/docker/Dockerfile.broad_sanger_exp b/build/docker/Dockerfile.broad_sanger_exp index 7a8207f9..1b1b1600 100755 --- a/build/docker/Dockerfile.broad_sanger_exp +++ b/build/docker/Dockerfile.broad_sanger_exp @@ -1,14 +1,11 @@ FROM r-base:4.2.1 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update --fix-missing -#RUN apt-get install -y --fix-missing --allow-unauthenticated build-essential libpq-dev python3.10 python3-pip python3-setuptools python3-dev python3-venv libcurl4-openssl-dev libxml2-dev libglpk-dev - RUN apt-get install -y --fix-missing --allow-unauthenticated build-essential python3-pip python3-setuptools python3-dev python3-venv libcurl4-openssl-dev libglpk-dev libxml2-dev libpq-dev RUN python3 -m venv /opt/venv RUN /opt/venv/bin/pip3 install --upgrade pip - # Set MPLCONFIGDIR to a writable directory ENV MPLCONFIGDIR=/tmp/matplotlib RUN mkdir -p /tmp/matplotlib From 2a70040c12a4b1c639706c71af9aa9261611ba1a Mon Sep 17 00:00:00 2001 From: Sara JC Gosline Date: Mon, 8 Jul 2024 16:37:16 -0700 Subject: [PATCH 2/7] updated build files --- .github/workflows/build.yml | 1612 ++++------------------------------- 1 file changed, 171 insertions(+), 1441 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 51b9b3a2..682ddfe6 100755 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,1451 +1,181 @@ -name: CI +name: Build docker images on: push: branches: - - builder_branch_JJ - - docs_update_4_5_24 - - doc_update_4_23_24 + - '**' # Allows you to run this workflow manually from the Actions tab workflow_dispatch: -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: write - pages: write - id-token: write jobs: -# Placeholder - -# depmap-docker-build: -# runs-on: ubuntu-latest -# environment: build -# steps: -# - name: Checkout Repo -# uses: actions/checkout@v2 -# - name: Log in to DockerHub -# uses: docker/login-action@v3 -# with: -# username: ${{ secrets.DOCKER_USERNAME }} -# password: ${{ secrets.DOCKER_TOKEN_SECRET }} -# - name: Build and push depmap images -# run: | -# docker build -t jjacobson95/depmap -f build/docker/Dockerfile.depmap . -# docker push jjacobson95/depmap - -# cptac-docker-build: -# runs-on: ubuntu-latest -# environment: build -# steps: -# - name: Checkout Repo -# uses: actions/checkout@v2 -# - name: Log in to DockerHub -# uses: docker/login-action@v3 -# with: -# username: ${{ secrets.DOCKER_USERNAME }} -# password: ${{ secrets.DOCKER_TOKEN_SECRET }} -# - name: Build and push cptac images -# run: | -# docker build -t jjacobson95/cptac -f build/docker/Dockerfile.cptac . -# docker push jjacobson95/cptac - -# hcmi-docker-build: -# runs-on: ubuntu-latest -# environment: build -# steps: -# - name: Checkout Repo -# uses: actions/checkout@v2 -# - name: Log in to DockerHub -# uses: docker/login-action@v3 -# with: -# username: ${{ secrets.DOCKER_USERNAME }} -# password: ${{ secrets.DOCKER_TOKEN_SECRET }} -# - name: Build and push hcmi images -# run: | -# docker build -t jjacobson95/hcmi -f build/docker/Dockerfile.hcmi . -# docker push jjacobson95/hcmi - -# beataml-docker-build: -# runs-on: ubuntu-latest -# environment: build -# steps: -# - name: Checkout Repo -# uses: actions/checkout@v2 -# - name: Log in to DockerHub -# uses: docker/login-action@v3 -# with: -# username: ${{ secrets.DOCKER_USERNAME }} -# password: ${{ secrets.DOCKER_TOKEN_SECRET }} -# - name: Build and push beataml image -# run: | -# docker build -t jjacobson95/beataml -f build/docker/Dockerfile.beataml . -# docker push jjacobson95/beataml - -# #placeholder -# mpnst-docker-build: -# runs-on: ubuntu-latest -# environment: build -# steps: -# - uses: actions/checkout@v2 -# - name: Log in to DockerHub -# uses: docker/login-action@v3 -# with: -# username: ${{ secrets.DOCKER_USERNAME }} -# password: ${{ secrets.DOCKER_TOKEN_SECRET }} - -# build-depmap-samples: -# if: github.actor != 'github-actions[bot]' -# needs: [depmap-docker-build] -# runs-on: ubuntu-20.04 -# environment: build -# steps: -# #Add upload artifact from depmap Samples -# - name: Checkout DepMap -# uses: actions/checkout@v2 -# # - name: Copy DepMap sample from build location. -# # run: | -# # cp build/depmap/requirements.txt . -# # cp build/depmap/requirements.r . -# # cp build/depmap/*.R . -# # cp build/depmap/*.py . -# # cp build/utils/* . -# - name: List files cwd -# run: ls -lah -# - name: cat reqs -# run: cat requirements.txt -# - name: Pull depmap image -# run: docker pull jjacobson95/depmap -# - name: Run depmap samples Container -# run: | -# docker run -v $PWD:/tmp/ jjacobson95/depmap Rscript /app/00-buildGeneFile.R -# docker run -v $PWD:/tmp/ jjacobson95/depmap Rscript /app/01-depmapSamples.R -# - name: Copy file from depmap Samples Container -# run: | -# mkdir depmap-samples genes-file -# mv depmap_samples.csv depmap-samples -# mv genes.csv genes-file -# - name: Upload artifacts for depmap -# uses: actions/upload-artifact@v2 -# with: -# name: depmap-samples -# path: depmap-samples -# - name: Upload artifacts for genes -# uses: actions/upload-artifact@v2 -# with: -# name: genes-file -# path: genes-file - -# build-drugs-beataml: -# if: github.actor != 'github-actions[bot]' -# runs-on: ubuntu-20.04 -# environment: build -# steps: -# - name: Checkout Repo -# uses: actions/checkout@v2 -# - name: Copy pubchem util and drug script to base dir -# run: | -# cp build/utils/pubchem_retrieval.py . -# cp build/beatAML/beataml_drug_retrieval.py . -# - name: install reqs -# run: | -# pip install -r requirements.txt -# - name: Append PYTHONPATH -# run: echo "PYTHONPATH=$PWD" >> $GITHUB_ENV -# - name: Run beataml Drug retrieval -# run: python beataml_drug_retrieval.py -# - name: Run beataml Drug retrieval second time to catch stragglers -# run: | -# mkdir additive-drugs-file-1 -# mv ignore_chems.txt additive-drugs-file-1 -# python beataml_drug_retrieval.py -# - name: Copy drugs file to artifact -# run: | -# mv drugs.tsv additive-drugs-file-1 -# - name: Upload artifacts for genes -# uses: actions/upload-artifact@v2 -# with: -# name: additive-drugs-file-1 -# path: additive-drugs-file-1 - -# #placeholder -# # build-drugs-mpnst: -# # if: github.actor != 'github-actions[bot]' -# # runs-on: ubuntu-20.04 -# # environment: build -# # steps: -# # - name: Checkout Repo -# # uses: actions/checkout@v2 -# # - name: Download artifacts from previous drug gen -# # uses: actions/download-artifact@v2 -# # with: -# # name: additive-drugs-file -# # - name: List files cwd -# # run: ls -lah -# # - name: Run mpnst Drug retrieval -# # run: python /build/mpnst/retrieve_drugs_mpnst.py -# # - name: Copy drugs file to artifact -# # run: | -# # mkdir additive-drugs-file -# # mv drugs.tsv ignore_chems.txt additive-drugs-file -# # - name: Upload artifacts for genes -# # uses: actions/upload-artifact@v2 -# # with: -# # name: additive-drugs-file -# # path: additive-drugs-file - -# build-drugs-depmap-CTRPv2-GDSC-gCSI-PRISM-CCLE-FIMM: -# if: github.actor != 'github-actions[bot]' -# runs-on: ubuntu-20.04 -# environment: build -# needs: [build-drugs-beataml,depmap-docker-build] -# steps: -# - name: Checkout Repo -# uses: actions/checkout@v2 -# - name: Download artifacts from previous drug gen -# uses: actions/download-artifact@v2 -# with: -# name: additive-drugs-file-1 -# - name: List files cwd -# run: ls -lah -# # - name: Install R -# # uses: r-lib/actions/setup-r@v2 -# # - name: Install R and Python Packages -# # run: | -# # Rscript build/depmap/requirements.r -# # pip install -r build/depmap/requirements.txt -# - name: Pull depmap image -# run: docker pull jjacobson95/depmap -# - name: Run depmap CTRPv2 Drug retrieval -# run: | -# docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R CTRPv2 -# - name: List files cwd -# run: ls -lah -# - name: Run depmap GDSC Drug retrieval -# run: | -# docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R GDSC -# - name: Run depmap gCSI Drug retrieval -# run: | -# docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R gCSI -# - name: Run depmap PRISM Drug retrieval -# run: | -# docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R PRISM -# - name: Run depmap CCLE Drug retrieval -# run: | -# docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R CCLE -# - name: Run depmap FIMM Drug retrieval -# run: | -# docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R FIMM -# - name: List files cwd -# run: ls -lah -# - name: Copy drugs file to artifact -# run: | -# mkdir additive-drugs-file-2 -# mv drugs.tsv ignore_chems.txt additive-drugs-file-2 -# - name: Upload artifacts for genes -# uses: actions/upload-artifact@v2 -# with: -# name: additive-drugs-file-2 -# path: additive-drugs-file-2 - -# build-drugs-depmap-NCI60-pt1: -# if: github.actor != 'github-actions[bot]' -# runs-on: ubuntu-20.04 -# environment: build -# needs: [build-drugs-depmap-CTRPv2-GDSC-gCSI-PRISM-CCLE-FIMM] -# steps: -# - name: Checkout Repo -# uses: actions/checkout@v2 -# - name: Download artifacts from previous drug gen -# uses: actions/download-artifact@v2 -# with: -# name: additive-drugs-file-2 -# - name: List files cwd -# run: ls -lah -# # - name: Install R -# # uses: r-lib/actions/setup-r@v2 -# # - name: Install R and Python Packages -# # run: | -# # Rscript build/depmap/requirements.r -# # pip install -r build/depmap/requirements.txt -# - name: Pull depmap image -# run: docker pull jjacobson95/depmap -# - name: Run depmap NCI60 Drug retrieval -# run: | -# docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60 -# # - name: Run depmap NCI60 Drug retrieval -# # run: Rscript build/depmap/CreateDrugFileCL.R NCI60 -# - name: Copy drugs file to artifact -# run: | -# mkdir additive-drugs-file-3 -# mv drugs.tsv ignore_chems.txt additive-drugs-file-3 -# - name: Upload artifacts for genes -# uses: actions/upload-artifact@v2 -# with: -# name: additive-drugs-file-3 -# path: additive-drugs-file-3 - - -# build-drugs-depmap-NCI60-pt2: -# if: github.actor != 'github-actions[bot]' -# runs-on: ubuntu-20.04 -# environment: build -# needs: [build-drugs-depmap-NCI60-pt1] -# steps: -# - name: Checkout Repo -# uses: actions/checkout@v2 -# - name: Download artifacts from previous drug gen -# uses: actions/download-artifact@v2 -# with: -# name: additive-drugs-file-3 -# - name: List files cwd -# run: ls -lah -# - name: Pull depmap image -# run: docker pull jjacobson95/depmap -# - name: Run depmap NCI60 Drug retrieval -# run: | -# docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60 -# - name: Copy drugs file to artifact -# run: | -# mkdir additive-drugs-file-4 -# mv drugs.tsv ignore_chems.txt additive-drugs-file-4 -# - name: Upload artifacts for genes -# uses: actions/upload-artifact@v2 -# with: -# name: additive-drugs-file-4 -# path: additive-drugs-file-4 - - -# build-drugs-depmap-NCI60-pt3: -# if: github.actor != 'github-actions[bot]' -# runs-on: ubuntu-20.04 -# environment: build -# needs: [build-drugs-depmap-NCI60-pt2] -# steps: -# - name: Checkout Repo -# uses: actions/checkout@v2 -# - name: Download artifacts from previous drug gen -# uses: actions/download-artifact@v2 -# with: -# name: additive-drugs-file-4 -# - name: List files cwd -# run: ls -lah -# - name: Pull depmap image -# run: docker pull jjacobson95/depmap -# - name: Run depmap NCI60 Drug retrieval -# run: | -# docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60 -# - name: Copy drugs file to artifact -# run: | -# mkdir additive-drugs-file-5 -# mv drugs.tsv ignore_chems.txt additive-drugs-file-5 -# - name: Upload artifacts for genes -# uses: actions/upload-artifact@v2 -# with: -# name: additive-drugs-file-5 -# path: additive-drugs-file-5 - - -# build-drugs-depmap-NCI60-pt4: -# if: github.actor != 'github-actions[bot]' -# runs-on: ubuntu-20.04 -# environment: build -# needs: [build-drugs-depmap-NCI60-pt3] -# steps: -# - name: Checkout Repo -# uses: actions/checkout@v2 -# - name: Download artifacts from previous drug gen -# uses: actions/download-artifact@v2 -# with: -# name: additive-drugs-file-5 -# - name: List files cwd -# run: ls -lah -# - name: Pull depmap image -# run: docker pull jjacobson95/depmap -# - name: Run depmap NCI60 Drug retrieval -# run: | -# docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60 -# - name: Copy drugs file to artifact -# run: | -# mkdir additive-drugs-file-6 -# mv drugs.tsv ignore_chems.txt additive-drugs-file-6 -# - name: Upload artifacts for genes -# uses: actions/upload-artifact@v2 -# with: -# name: additive-drugs-file-6 -# path: additive-drugs-file-6 - -# # build-drugs-depmap-NCI60-pt5: -# # if: github.actor != 'github-actions[bot]' -# # runs-on: ubuntu-20.04 -# # environment: build -# # needs: [build-drugs-depmap-NCI60-pt4] -# # steps: -# # - name: Checkout Repo -# # uses: actions/checkout@v2 -# # - name: Download artifacts from previous drug gen -# # uses: actions/download-artifact@v2 -# # with: -# # name: additive-drugs-file-6 -# # - name: List files cwd -# # run: ls -lah -# # - name: Pull depmap image -# # run: docker pull jjacobson95/depmap -# # - name: Run depmap NCI60 Drug retrieval -# # run: | -# # docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60 -# # - name: Copy drugs file to artifact -# # run: | -# # mkdir additive-drugs-file-7 -# # mv drugs.tsv ignore_chems.txt additive-drugs-file-7 -# # - name: Upload artifacts for genes -# # uses: actions/upload-artifact@v2 -# # with: -# # name: additive-drugs-file-7 -# # path: additive-drugs-file-7 - -# # build-drugs-depmap-NCI60-pt6: -# # if: github.actor != 'github-actions[bot]' -# # runs-on: ubuntu-20.04 -# # environment: build -# # needs: [build-drugs-depmap-NCI60-pt5] -# # steps: -# # - name: Checkout Repo -# # uses: actions/checkout@v2 -# # - name: Download artifacts from previous drug gen -# # uses: actions/download-artifact@v2 -# # with: -# # name: additive-drugs-file-7 -# # - name: List files cwd -# # run: ls -lah -# # - name: Pull depmap image -# # run: docker pull jjacobson95/depmap -# # - name: Run depmap NCI60 Drug retrieval -# # run: | -# # docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60 -# # - name: Copy drugs file to artifact -# # run: | -# # mkdir additive-drugs-file-8 -# # mv drugs.tsv ignore_chems.txt additive-drugs-file-8 -# # - name: Upload artifacts for genes -# # uses: actions/upload-artifact@v2 -# # with: -# # name: additive-drugs-file-8 -# # path: additive-drugs-file-8 - -# # build-drugs-depmap-NCI60-pt7: -# # if: github.actor != 'github-actions[bot]' -# # runs-on: ubuntu-20.04 -# # environment: build -# # needs: [build-drugs-depmap-NCI60-pt6] -# # steps: -# # - name: Checkout Repo -# # uses: actions/checkout@v2 -# # - name: Download artifacts from previous drug gen -# # uses: actions/download-artifact@v2 -# # with: -# # name: additive-drugs-file-8 -# # - name: List files cwd -# # run: ls -lah -# # - name: Pull depmap image -# # run: docker pull jjacobson95/depmap -# # - name: Run depmap NCI60 Drug retrieval -# # run: | -# # docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60 -# # - name: Copy drugs file to artifact -# # run: | -# # mkdir additive-drugs-file-9 -# # mv drugs.tsv ignore_chems.txt additive-drugs-file-9 -# # - name: Upload artifacts for genes -# # uses: actions/upload-artifact@v2 -# # with: -# # name: additive-drugs-file-9 -# # path: additive-drugs-file-9 - -# # build-drugs-depmap-NCI60-pt8: -# # if: github.actor != 'github-actions[bot]' -# # runs-on: ubuntu-20.04 -# # environment: build -# # needs: [build-drugs-depmap-NCI60-pt7] -# # steps: -# # - name: Checkout Repo -# # uses: actions/checkout@v2 -# # - name: Download artifacts from previous drug gen -# # uses: actions/download-artifact@v2 -# # with: -# # name: additive-drugs-file-9 -# # - name: List files cwd -# # run: ls -lah -# # - name: Pull depmap image -# # run: docker pull jjacobson95/depmap -# # - name: Run depmap NCI60 Drug retrieval -# # run: | -# # docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60 -# # - name: Copy drugs file to artifact -# # run: | -# # mkdir additive-drugs-file-10 -# # mv drugs.tsv ignore_chems.txt additive-drugs-file-10 -# # - name: Upload artifacts for genes -# # uses: actions/upload-artifact@v2 -# # with: -# # name: additive-drugs-file-10 -# # path: additive-drugs-file-10 - -# # build-drugs-depmap-NCI60-pt9: -# # if: github.actor != 'github-actions[bot]' -# # runs-on: ubuntu-20.04 -# # environment: build -# # needs: [build-drugs-depmap-NCI60-pt8] -# # steps: -# # - name: Checkout Repo -# # uses: actions/checkout@v2 -# # - name: Download artifacts from previous drug gen -# # uses: actions/download-artifact@v2 -# # with: -# # name: additive-drugs-file-10 -# # - name: List files cwd -# # run: ls -lah -# # - name: Pull depmap image -# # run: docker pull jjacobson95/depmap -# # - name: Run depmap NCI60 Drug retrieval -# # run: | -# # docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60 -# # - name: Copy drugs file to artifact -# # run: | -# # mkdir additive-drugs-file-11 -# # mv drugs.tsv ignore_chems.txt additive-drugs-file-11 -# # - name: Upload artifacts for genes -# # uses: actions/upload-artifact@v2 -# # with: -# # name: additive-drugs-file-11 -# # path: additive-drugs-file-11 - -# # build-drugs-depmap-NCI60-pt10: -# # if: github.actor != 'github-actions[bot]' -# # runs-on: ubuntu-20.04 -# # environment: build -# # needs: [build-drugs-depmap-NCI60-pt9] -# # steps: -# # - name: Checkout Repo -# # uses: actions/checkout@v2 -# # - name: Download artifacts from previous drug gen -# # uses: actions/download-artifact@v2 -# # with: -# # name: additive-drugs-file-11 -# # - name: List files cwd -# # run: ls -lah -# # - name: Pull depmap image -# # run: docker pull jjacobson95/depmap -# # - name: Run depmap NCI60 Drug retrieval -# # run: | -# # docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60 -# # - name: Copy drugs file to artifact -# # run: | -# # mkdir additive-drugs-file-12 -# # mv drugs.tsv ignore_chems.txt additive-drugs-file-12 -# # - name: Upload artifacts for genes -# # uses: actions/upload-artifact@v2 -# # with: -# # name: additive-drugs-file-12 -# # path: additive-drugs-file-12 - -# # build-drugs-depmap-NCI60-pt11: -# # if: github.actor != 'github-actions[bot]' -# # runs-on: ubuntu-20.04 -# # environment: build -# # needs: [build-drugs-depmap-NCI60-pt10] -# # steps: -# # - name: Checkout Repo -# # uses: actions/checkout@v2 -# # - name: Download artifacts from previous drug gen -# # uses: actions/download-artifact@v2 -# # with: -# # name: additive-drugs-file-12 -# # - name: List files cwd -# # run: ls -lah -# # - name: Pull depmap image -# # run: docker pull jjacobson95/depmap -# # - name: Run depmap NCI60 Drug retrieval -# # run: | -# # docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R NCI60 -# # - name: Copy drugs file to artifact -# # run: | -# # mkdir additive-drugs-file-13 -# # mv drugs.tsv ignore_chems.txt additive-drugs-file-13 -# # - name: Upload artifacts for genes -# # uses: actions/upload-artifact@v2 -# # with: -# # name: additive-drugs-file-13 -# # path: additive-drugs-file-13 - -# build-drugs-depmap-capture-stragglers: -# if: github.actor != 'github-actions[bot]' -# runs-on: ubuntu-20.04 -# environment: build -# needs: [build-drugs-depmap-NCI60-pt4] -# steps: -# - name: Checkout Repo -# uses: actions/checkout@v2 -# - name: Download artifacts from previous drug gen -# uses: actions/download-artifact@v2 -# with: -# name: additive-drugs-file-6 -# - name: List files cwd -# run: ls -lah -# - name: List files cwd -# run: rm ignore_chems.txt -# - name: Pull depmap image -# run: docker pull jjacobson95/depmap -# - name: Run depmap straggler Drug retrieval -# run: | -# docker run -v $PWD:/tmp jjacobson95/depmap Rscript /app/CreateDrugFileCL.R CTRPv2,GDSC,gCSI,PRISM,CCLE,FIMM,NCI60 -# - name: Copy drugs file to artifact -# run: | -# mkdir additive-drugs-file-7 -# mv drugs.tsv ignore_chems.txt additive-drugs-file-7 -# - name: Upload artifacts for genes -# uses: actions/upload-artifact@v2 -# with: -# name: additive-drugs-file-7 -# path: additive-drugs-file-7 - -# build-cptac-samples: -# if: github.actor != 'github-actions[bot]' -# runs-on: ubuntu-20.04 -# environment: build -# needs: [build-depmap-samples,cptac-docker-build] -# steps: -# #Add upload artifact from CPTAC Samples -# - name: Checkout CPTAC -# uses: actions/checkout@v2 -# - name: Download artifacts from depmap -# uses: actions/download-artifact@v2 -# with: -# name: depmap-samples -# - name: Download artifacts from DepMap (genes) -# uses: actions/download-artifact@v2 -# with: -# name: genes-file -# # - name: Copy DepMap sample from build location. -# # run: | -# # cp build/cptac/requirements.txt . -# # cp build/cptac/*.py . -# # # copy build/genes.csv . -# - name: Pull cptac image -# run: docker pull jjacobson95/cptac -# - name: Run CPTAC samples Container -# run: docker run -v $PWD:/tmp jjacobson95/cptac --geneFile=/tmp/genes.csv --prevSampleFile=/tmp/depmap_samples.csv -# - name: Copy file from CPTAC Samples Container -# run: | -# mkdir cptac-samples -# mv cptac_samples.csv cptac-samples -# - name: Upload artifacts for CPTAC -# uses: actions/upload-artifact@v2 -# with: -# name: cptac-samples -# path: cptac-samples - -# build-hcmi-samples: -# if: github.actor != 'github-actions[bot]' -# runs-on: ubuntu-20.04 -# environment: build -# needs: [build-cptac-samples,hcmi-docker-build] -# steps: -# - name: Checkout HCMI -# uses: actions/checkout@v2 -# - name: Download artifacts from CPTAC -# uses: actions/download-artifact@v2 -# with: -# name: cptac-samples -# # - name: Build HCMI Image -# # run: docker build -t hcmi-samples-builder -f build/docker/Dockerfile.hcmi . -# - name: Pull hcmi image -# run: docker pull jjacobson95/hcmi -# - name: Run HCMI samples Container -# run: docker run -v $PWD:/tmp --name hcmi-samples-container jjacobson95/hcmi python 01-createHCMISamplesFile.py -# - name: Copy file from HCMI Samples Container -# run: | -# mkdir hcmi-samples -# for file in hcmi_samples.csv; do -# docker cp hcmi-samples-container:/usr/src/app/$file hcmi-samples/$file -# done -# - name: Get HCMI container logs -# run: docker logs hcmi-samples-container -# - name: Upload artifacts for HCMI -# uses: actions/upload-artifact@v2 -# with: -# name: hcmi-samples -# path: hcmi-samples - -# build-beataml-samples: -# if: github.actor != 'github-actions[bot]' -# runs-on: ubuntu-20.04 -# environment: build -# needs: [build-hcmi-samples,beataml-docker-build] -# steps: -# #Add upload artifact from CPTAC Samples -# - name: Checkout BeatAML -# uses: actions/checkout@v2 -# - name: Download artifacts from HCMI -# uses: actions/download-artifact@v2 -# with: -# name: hcmi-samples -# - name: Pull beataml image -# run: docker pull jjacobson95/beataml -# - name: Run BeatAML Container with samples option -# run: docker run -v $PWD:/tmp --name beataml-samples-container jjacobson95/beataml python GetBeatAML.py --token ${{ secrets.SYNAPSE_TOKEN_SECRET }} --samples -# - name: Copy file from BeatAML Samples Container -# run: | -# mkdir beataml-samples -# for file in beataml_samples.csv; do -# docker cp beataml-samples-container:/usr/src/app/$file beataml-samples/$file -# done -# - name: Get BeatAML container logs -# run: docker logs beataml-samples-container -# - name: Upload artifacts for BeatAML -# uses: actions/upload-artifact@v2 -# with: -# name: beataml-samples -# path: beataml-samples - -# # Placeholder -# build-mpnst-samples: -# if: github.actor != 'github-actions[bot]' -# runs-on: ubuntu-20.04 -# environment: build -# needs: [build-beataml-samples,mpnst-docker-build] -# steps: -# #Add upload artifact from CPTAC Samples -# - name: Checkout MPNST -# uses: actions/checkout@v2 - -# build-hcmi: -# if: github.actor != 'github-actions[bot]' -# needs: [build-hcmi-samples] -# runs-on: ubuntu-20.04 -# environment: build -# steps: -# - name: Checkout HCMI -# uses: actions/checkout@v2 -# - name: Download artifacts from HCMI -# uses: actions/download-artifact@v2 -# with: -# name: hcmi-samples -# - name: List files cwd -# run: ls -lah -# # - name: Build HCMI Image -# # run: docker build -t hcmi-builder -f build/docker/Dockerfile.hcmi . -# - name: Pull hcmi image -# run: docker pull jjacobson95/hcmi -# - name: Run HCMI Container -# run: | -# docker run -v $PWD:/tmp --name hcmi-container jjacobson95/hcmi /bin/bash -c "python 02-getHCMIData.py -m full_manifest.txt -t transcriptomics -o hcmi_transcriptomics.csv && python 02-getHCMIData.py -m full_manifest.txt -M full_manifest_files -t copy_number -o hcmi_copy_number.csv && python 02-getHCMIData.py -m full_manifest.txt -M full_manifest_files -t mutations -o hcmi_mutations.csv" -# - name: Copy files from HCMI Container -# run: | -# mkdir hcmi-files -# for file in hcmi_transcriptomics.csv hcmi_copy_number.csv hcmi_mutations.csv; do -# docker cp hcmi-container:/usr/src/app/$file hcmi-files/$file -# done -# cp hcmi_samples.csv hcmi-files -# - name: Get HCMI container logs -# run: docker logs hcmi-container -# - name: Upload artifacts for HCMI -# uses: actions/upload-artifact@v2 -# with: -# name: hcmi-files -# path: hcmi-files - -# check-hcmi-schema: -# if: github.actor != 'github-actions[bot]' -# needs: [build-hcmi] -# runs-on: ubuntu-20.04 -# environment: build -# steps: -# - name: Checkout code HCMI -# uses: actions/checkout@v2 -# - name: Download artifacts from HCMI Samples -# uses: actions/download-artifact@v2 -# with: -# name: hcmi-samples -# - name: Download artifacts from HCMI -# uses: actions/download-artifact@v2 -# with: -# name: hcmi-files -# - name: Download linkml -# run: | -# pip install linkml -# pip install linkml-validator -# - name: check dir -# run: ls -# - name: Run linkml schema checker -# run: bash schema/check_hcmi_linkml.sh - -# build-beataml: -# if: github.actor != 'github-actions[bot]' -# needs: [build-beataml-samples,build-drugs-depmap-capture-stragglers] -# runs-on: ubuntu-20.04 -# environment: build -# steps: -# - name: Checkout code BeatAML -# uses: actions/checkout@v2 -# - name: Download artifacts from Beataml Samples -# uses: actions/download-artifact@v2 -# with: -# name: beataml-samples -# - name: Download artifacts from Drugs -# uses: actions/download-artifact@v2 -# with: -# name: additive-drugs-file-7 -# - name: List files cwd -# run: ls -lah -# - name: Pull beataml image -# run: docker pull jjacobson95/beataml -# - name: Run BeatAML Container -# run: docker run -v $PWD:/tmp --name beataml-container jjacobson95/beataml python GetBeatAML.py --token ${{ secrets.SYNAPSE_TOKEN_SECRET }} -# - name: Copy files from BeatAML Container -# run: | -# mkdir beataml-files -# for file in beataml_transcriptomics.csv beataml_proteomics.csv beataml_mutations.csv beataml_drugs.tsv beataml_experiments.csv; do -# docker cp beataml-container:/usr/src/app/$file beataml-files/$file -# done -# cp beataml_samples.csv beataml-files -# - name: Get BeatAML container logs -# run: docker logs beataml-container -# - name: Upload artifact for BeatAML -# uses: actions/upload-artifact@v2 -# with: -# name: beataml-files -# path: beataml-files - - -# check-beataml-schema: -# if: github.actor != 'github-actions[bot]' -# needs: [build-beataml] -# runs-on: ubuntu-20.04 -# environment: build -# steps: -# - name: Checkout code BeatAML -# uses: actions/checkout@v2 -# - name: Download artifacts from Beataml Samples -# uses: actions/download-artifact@v2 -# with: -# name: beataml-samples -# - name: Download artifacts from Beataml -# uses: actions/download-artifact@v2 -# with: -# name: beataml-files -# - name: Download linkml -# run: | -# pip install linkml -# pip install linkml-validator -# - name: check dir -# run: ls -# - name: Run linkml schema checker -# run: bash schema/check_beataml_linkml.sh - -# # build-depmap: -# # if: github.actor != 'github-actions[bot]' -# # needs: [build-depmap-samples] -# # runs-on: ubuntu-20.04 -# # environment: build -# # steps: -# # - name: Checkout code DepMap -# # uses: actions/checkout@v2 -# # - name: List files cwd before -# # run: ls -lah -# # - name: Install coderdata -# # run: pip install coderdata -# # - name: Download depmap data from coder data -# # run: coderdata download --prefix depmap -# # - name: List files cwd after -# # run: ls -lah -# # - name: Copy files from DepMap Container -# # run: | -# # mkdir depmap-files -# # for file in *depmap*sv*; do -# # cp $file depmap-files/$file -# # done -# # - name: Upload artifact for DepMap -# # uses: actions/upload-artifact@v2 -# # with: -# # name: depmap-files -# # path: depmap-files - -# build-depmap: -# if: github.actor != 'github-actions[bot]' -# needs: [build-depmap-samples,build-drugs-depmap-capture-stragglers] -# runs-on: ubuntu-20.04 -# environment: build -# steps: -# - name: Checkout DepMap -# uses: actions/checkout@v2 -# # - name: Copy DepMap sample from build location. -# # run: | -# # cp build/depmap/requirements.txt . -# # cp build/depmap/requirements.r . -# # cp build/depmap/*.R . -# # cp build/depmap/*.py . -# # cp build/utils/* . -# - name: Download artifacts from DepMap (samples) -# uses: actions/download-artifact@v2 -# with: -# name: depmap-samples -# - name: Download artifacts from DepMap (genes) -# uses: actions/download-artifact@v2 -# with: -# name: genes-file -# - name: Download artifacts from Drugs -# uses: actions/download-artifact@v2 -# with: -# name: additive-drugs-file-7 -# - name: gzip drugs file -# run: gzip drugs.tsv -# - name: List files cwd -# run: ls -lah -# - name: cat reqs -# run: cat requirements.txt -# - name: Pull depmap image -# run: docker pull jjacobson95/depmap -# # - name: docker save space and stats -# # run: | -# # docker image prune -f -# # docker container prune -f -# # docker volume prune -f -# # docker network prune -f -# # docker system df -# - name: Run depmap 02-pullDepMap.R Container -# run: | -# docker run -v $PWD:/tmp/ jjacobson95/depmap Rscript /app/02-pullDepMap.R /tmp/genes.csv /tmp/depmap_samples.csv -# - name: Run depmap 02b-pullSanger.R Container -# run: | -# docker run -v $PWD:/tmp/ jjacobson95/depmap Rscript /app/02b-pullSanger.R /tmp/genes.csv /tmp/depmap_samples.csv -# # - name: Run depmap 03-createDrugFile.RContainer -# # run: | -# # docker run -v $PWD:/tmp depmap Rscript 03-createDrugFile.R CTRPv2,GDSC,gCSI,PRISM,CCLE,FIMM,NCI60 -# - name: Run depmap 04-drug_dosage_and_curves.py Container -# run: | -# docker run -v $PWD:/tmp/ jjacobson95/depmap /opt/venv/bin/python /app/04-drug_dosage_and_curves.py --drugfile=/tmp/drugs.tsv.gz --curSampleFile=/tmp/depmap_samples.csv -# - name: Run depmap 01b-pullDrugs_LINCS.py Container -# run: | -# docker run -v $PWD:/tmp jjacobson95/depmap /opt/venv/bin/python /app/01b-pullDrugs_LINCS.py --drugFile /tmp/drugs.tsv.gz -# - name: Run depmap 04-pullDrugs_LINCS.py Container -# run: | -# docker run -v $PWD:/tmp jjacobson95/depmap /opt/venv/bin/python /app/04-pullDrugs_LINCS.py --drugFile /tmp/drugs.tsv.gz -# - name: Run depmap 05-LINCS_perturbations.py Container -# run: | -# docker run -v $PWD:/tmp/ jjacobson95/depmap Rscript /app/05-LINCS_perturbations.R /tmp/genes.csv /tmp/drugs.tsv.gz /tmp/depmap_samples.csv -# - name: List files cwd -# run: ls -lah -# - name: Copy files from DepMap Container -# run: | -# mkdir depmap-files -# cp depmap_* depmap-files -# - name: Upload artifacts for DepMap -# uses: actions/upload-artifact@v2 -# with: -# name: depmap-files -# path: depmap-files - -# check-depmap-schema: -# if: github.actor != 'github-actions[bot]' -# needs: [build-depmap] -# runs-on: ubuntu-20.04 -# environment: build -# steps: -# - name: Checkout code DepMap -# uses: actions/checkout@v2 -# - name: Download artifacts from DepMap Samples -# uses: actions/download-artifact@v2 -# with: -# name: depmap-samples -# - name: Download artifacts from DepMap -# uses: actions/download-artifact@v2 -# with: -# name: depmap-files -# - name: Download linkml -# run: | -# pip install linkml -# pip install linkml-validator -# - name: check dir -# run: ls -# - name: Run linkml schema checker -# run: bash schema/check_depmap_linkml.sh - - -# build-cptac: -# if: github.actor != 'github-actions[bot]' -# needs: [build-cptac-samples] -# runs-on: ubuntu-20.04 -# environment: build -# steps: -# - name: Checkout CPTAC -# uses: actions/checkout@v2 -# - name: Download artifacts from CPTAC -# uses: actions/download-artifact@v2 -# with: -# name: cptac-samples -# - name: Download artifacts from DepMap (genes) -# uses: actions/download-artifact@v2 -# with: -# name: genes-file -# - name: Copy files from build location. -# run: | -# cp build/cptac/*.py . -# cp build/cptac/requirements.txt . -# # copy build/genes.csv . -# - name: List files cwd -# run: ls -lah -# - name: Pull cptac image -# run: docker pull jjacobson95/cptac -# - name: Run CPTAC Container -# run: | -# docker run -v $PWD:/tmp jjacobson95/cptac --geneFile=/tmp/genes.csv --curSampleFile=/tmp/cptac_samples.csv -# - name: Copy files from CPTAC Container -# run: | -# mkdir cptac-files -# cp cptac_* cptac-files -# - name: Upload artifacts for CPTAC -# uses: actions/upload-artifact@v2 -# with: -# name: cptac-files -# path: cptac-files - -# check-cptac-schema: -# if: github.actor != 'github-actions[bot]' -# needs: [build-cptac] -# runs-on: ubuntu-20.04 -# environment: build -# steps: -# - name: Checkout code CPTAC -# uses: actions/checkout@v2 -# - name: Download artifacts from CPTAC Samples -# uses: actions/download-artifact@v2 -# with: -# name: cptac-samples -# - name: Download artifacts from CPTAC -# uses: actions/download-artifact@v2 -# with: -# name: cptac-files -# - name: Download linkml and gunzip files -# run: | -# pip install linkml -# pip install linkml-validator -# find . -type f -name '*.gz' -exec gunzip {} + -# - name: check dir -# run: ls -# - name: Run linkml schema checker -# run: bash schema/check_cptac_linkml.sh - -# #placeholder -# build-mpnst: -# if: github.actor != 'github-actions[bot]' -# needs: [build-mpnst-samples] -# runs-on: ubuntu-20.04 -# environment: build -# steps: -# - name: Checkout code MPNST -# uses: actions/checkout@v2 - -# # Placeholder -# check-mpnst-schema: -# if: github.actor != 'github-actions[bot]' -# needs: [build-mpnst] -# runs-on: ubuntu-20.04 -# environment: build -# steps: -# - name: Checkout code MPNST -# uses: actions/checkout@v2 -# # - name: Download artifacts from MPNST Samples -# # uses: actions/download-artifact@v2 -# # with: -# # name: mpnst-samples -# # - name: Download artifacts from MPNST -# # uses: actions/download-artifact@v2 -# # with: -# # name: mpnst-files -# # - name: Download linkml -# # run: | -# # pip install linkml -# # pip install linkml-validator -# # - name: check dir -# # run: ls -# # - name: Run linkml schema checker -# # run: bash schema/check_mpnst_linkml.sh - -# data-visualization: -# if: github.actor != 'github-actions[bot]' -# needs: [check-hcmi-schema, check-beataml-schema, check-depmap-schema, check-cptac-schema, check-mpnst-schema] -# environment: build -# runs-on: ubuntu-20.04 -# steps: -# - name: Set up R -# uses: r-lib/actions/setup-r@v2 -# - name: Install R packages -# run: | -# R -e "install.packages('RColorBrewer', dependencies=TRUE, repos='http://cran.rstudio.com/')" -# R -e "install.packages('circlize', dependencies=TRUE, repos='http://cran.rstudio.com/')" -# R -e "install.packages('dplyr', dependencies=TRUE, repos='http://cran.rstudio.com/')" -# R -e "install.packages('readr', dependencies=TRUE, repos='http://cran.rstudio.com/')" -# R -e "install.packages('tools', dependencies=TRUE, repos='http://cran.rstudio.com/')" -# R -e "install.packages('ggplot2', dependencies=TRUE, repos='http://cran.rstudio.com/')" -# R -e "install.packages('reshape2', dependencies=TRUE, repos='http://cran.rstudio.com/')" -# R -e "install.packages('ComplexUpset', dependencies=TRUE, repos='http://cran.rstudio.com/')" -# - name: checkout repo content -# uses: actions/checkout@v2 -# - name: Download artifacts from HCMI -# uses: actions/download-artifact@v2 -# with: -# name: hcmi-files -# - name: Download artifacts from BeatAML -# uses: actions/download-artifact@v2 -# with: -# name: beataml-files -# - name: Download artifacts from DepMap -# uses: actions/download-artifact@v2 -# with: -# name: depmap-files -# # - name: Download artifacts from MPNST -# # uses: actions/download-artifact@v2 -# # with: -# # name: mpnst-files -# - name: Download artifacts from cptac -# uses: actions/download-artifact@v2 -# with: -# name: cptac-files -# # - name: Install coderdata and skip build steps -# # run: pip install coderdata -# # - name: Download cptac data from coder data -# # run: coderdata download --prefix cptac -# # - name: Download depmap data from coder data -# # run: coderdata download --prefix depmap -# # - name: Download all data from coder data -# # run: coderdata download -# - name: Create all-files directory -# run: mkdir all-files -# - name: List files cwd -# run: ls -lah -# - name: List files dataSummary dir -# run: ls -lah dataSummary -# - name: Gzip all non-samples csv files in current dir. -# run: | -# find . -type f -name '*.csv' ! -name '*samples*' -exec gzip -k "{}" + -# - name: Run Visualization01 -# if: success() || failure() -# run: Rscript dataSummary/visualization01.R -# - name: Run Visualization02 -# if: success() || failure() -# run: Rscript dataSummary/visualization02.R -# - name: setup data-vis dir -# if: success() || failure() -# run: | -# mkdir -p data-vis -# rsync -av --remove-source-files --include='*.png' --include='*.pdf' --include='*table.csv' --exclude='*' . data-vis/ -# - name: Upload artifacts -# if: success() || failure() -# uses: actions/upload-artifact@v2 -# with: -# name: data-vis -# path: data-vis - -# data-visualization-save-to-docs: -# if: github.actor != 'github-actions[bot]' -# needs: [data-visualization] -# environment: build -# runs-on: ubuntu-20.04 -# steps: -# - name: Checkout repo content -# uses: actions/checkout@v2 -# - name: Setup Git config and pull -# run: | -# git config user.name "jjacobson95" -# git config user.email "jeremy.jacobson3402@gmail.com" -# git pull -# - name: Download artifacts from data-vis -# uses: actions/download-artifact@v2 -# with: -# name: data-vis -# - name: List files cwd -# run: ls -lah -# - name: List files .. -# run: ls -lah .. -# - name: List files all -# run: ls -lah * -# - name: Move files to docs directory -# run: | -# rsync -av --remove-source-files --include='*.png' --include='*.pdf' --exclude='*' . docs/assets/stats -# rsync -av --remove-source-files --include='*table.csv' --exclude='*' . docs/_data -# # rsync -av --remove-source-files --include='*.png' --include='*.pdf' --include='*table.csv' --exclude='*' . docs/ -# - name: List files in docs after moving -# run: ls -lah docs -# - name: List files in local after moving -# run: ls -lah -# - name: Git add and push new changes. -# run: | -# git add -f docs/* -# git status -# if git diff --staged --quiet; then -# echo "No changes to commit" -# else -# git commit -m "Add data visualization artifacts to docs" -# git push -# fi -# gen-stats-and-push: -# if: github.actor != 'github-actions[bot]' -# needs: [build-hcmi, build-beataml, build-depmap, build-cptac, build-mpnst] -# environment: build -# runs-on: ubuntu-20.04 -# steps: -# - name: Checkout repo content -# uses: actions/checkout@v2 -# - name: Setup Git config and pull -# run: | -# git config user.name "jjacobson95" -# git config user.email "jeremy.jacobson3402@gmail.com" -# git pull -# - name: Download artifacts from HCMI -# uses: actions/download-artifact@v2 -# with: -# name: hcmi-files -# - name: Download artifacts from BeatAML -# uses: actions/download-artifact@v2 -# with: -# name: beataml-files -# - name: Download artifacts from DepMap -# uses: actions/download-artifact@v2 -# with: -# name: depmap-files -# # If this pipeline is supported in the future, Gen stats has been updated to include mpnst too. -# # - name: Download artifacts from MPNST -# # uses: actions/download-artifact@v2 -# # with: -# # name: mpnst-files -# - name: Download artifacts from cptac -# uses: actions/download-artifact@v2 -# with: -# name: cptac-files -# - name: Install coderdata -# run: | -# pip install coderdata -# pip install pyyaml -# - name: Run stats generation script -# run: | -# python scripts/gen_stats.py -# - name: Git add and push new changes if they exist. -# run: | -# git pull -# mv stats.yml docs/_data/stats.yml -# git add -f docs/_data/stats.yml -# git status -# if git diff --staged --quiet; then -# echo "No changes to commit" -# else -# git commit -m "Add stats.yml to docs/_data" -# git push -# fi - -# upload-to-figshare: -# if: github.actor != 'github-actions[bot]' -# needs: [check-hcmi-schema, check-beataml-schema, check-depmap-schema, check-cptac-schema, check-mpnst-schema] -# # needs: [build-hcmi, build-beataml, build-depmap, build-cptac, build-mpnst] -# environment: build -# runs-on: ubuntu-20.04 -# steps: -# - name: checkout repo content -# uses: actions/checkout@v2 -# - name: Download artifacts from HCMI -# uses: actions/download-artifact@v2 -# with: -# name: hcmi-files -# - name: Download artifacts from BeatAML -# uses: actions/download-artifact@v2 -# with: -# name: beataml-files -# - name: Download artifacts from CPTAC -# uses: actions/download-artifact@v2 -# with: -# name: cptac-files -# - name: Download artifacts from DepMap -# uses: actions/download-artifact@v2 -# with: -# name: depmap-files -# # - name: Download artifacts from MPNST -# # uses: actions/download-artifact@v2 -# # with: -# # name: mpnst-files -# - name: Create all-files directory -# run: mkdir all-files -# - name: List files cwd -# run: ls -lah -# - name: Move files to all-files -# run: | -# rsync -av --remove-source-files --include='*.csv' --include='*.tsv' --include='*.gz' --exclude='*' . all-files/ -# - name: gzip all files in all-files -# run: gzip all-files/* -# - name: gunzip all samples files in all-files -# run: gunzip all-files/*samples* -# - name: List files in all-files -# run: ls -lah all-files/ -# # - name: Upload to Figshare -# # # This can be updated to trigger with tag releases. -# # # run: python scripts/push_to_figshare.py --directory all-files --title "CODERData${{ github.event.release.tag_name }}" --token ${{ secrets.FIGSHARE_TOKEN_SECRET }} -p False -# # #This is a short term alternative. Links to branch. Not as pretty but okay for now. - -# # #To do: Extract version number from setup.py and run there. This can occur after merge with package dev branch. -# # run: python scripts/push_to_figshare.py --directory all-files --title "CODERData${{ github.ref }}" --token ${{ secrets.FIGSHARE_TOKEN_SECRET }} --project_id 189342 --publish -# - name: Extract version from setup.py -# id: version -# run: echo "PACKAGE_VERSION=$(grep 'version=' setup.py | sed -n "s/.*version='\(.*\)'.*/\1/p")" >> $GITHUB_ENV -# - name: Upload to Figshare -# run: python scripts/push_to_figshare.py --directory all-files --title "CODERData${{ env.PACKAGE_VERSION }}" --token ${{ secrets.FIGSHARE_TOKEN_SECRET }} --project_id 189342 --publish -# - name: List files in all-files -# run: ls -lah all-files/ -# - name: Setup Git config -# run: | -# git config user.name "jjacobson95" -# git config user.email "jeremy.jacobson3402@gmail.com" -# - name: Commit and push changes -# run: | -# git pull -# mv figshare_latest.yml docs/_data -# git add -f docs/* -# git status -# git commit -m "Add figshare_latest.yml to docs/_data" -# git push - - - # update-package-download-function: - # if: github.actor != 'github-actions[bot]' - # needs: [upload-to-figshare] - # environment: build - # runs-on: ubuntu-20.04 - # steps: - # - name: checkout repo content - # uses: actions/checkout@v2 - # - name: Update figshare path in downloader.py - # run: | - # python scripts/update_download_function.py -y docs/_data/figshare_latest.yml -d coderdata/download/downloader.py - # cat coderdata/download/downloader.py - # - name: Setup Git config - # run: | - # git config user.name "jjacobson95" - # git config user.email "jeremy.jacobson3402@gmail.com" - # - name: Commit and push changes - # run: | - # git pull - # git add coderdata/download/downloader.py - # git commit -m "update downloader.py" - # git push - - # run-tests: - # if: github.actor != 'github-actions[bot]' - # needs: [update-package-download-function] - # runs-on: ubuntu-latest - # strategy: - # fail-fast: false - # matrix: - # test-file: [ - # 'test_download_and_load_beataml.py', - # 'test_download_and_load_depmap.py', - # 'test_download_beataml_cli.py', - # 'test_download_depmap_cli.py', - # 'test_download_and_load_cptac.py', - # 'test_download_and_load_hcmi.py', - # 'test_download_cptac_cli.py', - # 'test_download_hcmi_cli.py', - # 'test_reformat_datasets.py', - # 'test_reload_one.py', - # 'test_reload_all.py', - # 'test_join_datasets.py', - # 'test_info.py' - # ] - # outputs: - # result: ${{ steps.test-outcome.outcome }} - # steps: - # - uses: actions/checkout@v2 - # with: - # fetch-depth: 0 - # - name: Set up Python - # uses: actions/setup-python@v2 - # with: - # python-version: '3.10' - # - name: Install dependencies - # run: | - # python -m pip install --upgrade pip - # pip install -r requirements.txt - # pip install pytest-xdist - # - name: Install as package - # run: | - # pip install . - # - name: Run test - # id: test-outcome - # run: | - # pytest -n 4 tests/${{ matrix.test-file }} - - # build-and-publish: - # if: github.actor != 'github-actions[bot]' - # needs: [run-tests] - # runs-on: ubuntu-latest - # environment: build - # steps: - # - uses: actions/checkout@v2 - # with: - # fetch-depth: 0 - - # - name: Fetch tags from main branch - # run: | - # git fetch origin 'refs/tags/*:refs/tags/*' - - # - name: Set up Python - # uses: actions/setup-python@v2 - # with: - # python-version: '3.10' - - # - name: Install dependencies - # run: | - # python -m pip install --upgrade pip - # pip install setuptools wheel twine packaging - - # - name: Update version - # run: | - # # Extract the current version from setup.py - # CURRENT_VERSION=$(grep -oP "version='\K[0-9]+\.[0-9]+\.[0-9]+" setup.py) - # echo "Current version: $CURRENT_VERSION" - - # # Fetch tags and get the latest tag, removing the 'v' prefix - # git fetch --tags - # LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1` | sed 's/^v//') - # echo "Latest tag: $LATEST_TAG" - - # # Use Python to compare versions - # HIGHER_VERSION=$(python -c "from packaging.version import parse as parse_version; \ - # current = parse_version('$CURRENT_VERSION'); \ - # latest = parse_version('$LATEST_TAG'); \ - # print(current if current > latest else latest)") - # echo "Higher version: $HIGHER_VERSION" - - # # If current version is higher, increment patch version in setup.py - # if [ "$CURRENT_VERSION" == "$HIGHER_VERSION" ]; then - # IFS='.' read -r -a VERSION_PARTS <<< "$CURRENT_VERSION" - # PATCH=$((VERSION_PARTS[2] + 1)) - # NEW_VERSION="${VERSION_PARTS[0]}.${VERSION_PARTS[1]}.$PATCH" - # echo "Incremented version: $NEW_VERSION" - # sed -i "s/version='\([0-9]\+\.[0-9]\+\.\)[0-9]\+'/version='\1$PATCH'/" setup.py - - # # Commit and push the updated setup.py - # git config --local user.email "action@github.com" - # git config --local user.name "GitHub Action" - # git pull - # git add setup.py - # git commit -m "Increment version number to $NEW_VERSION [skip ci]" - # git push - # else - # echo "No version increment needed" - # fi - # - name: Build local and publish to PyPI - # env: - # TWINE_USERNAME: __token__ - # TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN_SECRET }} - # run: | - # python setup.py sdist bdist_wheel - # twine upload dist/* --verbose + build-cellline-omics: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Build and push broad sanger omics + uses: docker/build-push-action@v3 + with: + file: ./build/docker/Dockerfile.broad_sanger_omics + tags: sgosline/broad-sanger-omics:latest + push: true + platforms: linux/amd64,linux/arm64 + build-cellline-exp: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Build and push broad sanger experiment + uses: docker/build-push-action@v3 + with: + file: ./build/docker/Dockerfile.broad_sanger_exp + tags: sgosline/broad-sanger-exp:latest + push: true + platforms: linux/amd64,linux/arm64 + build-genes: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Build and push genes + uses: docker/build-push-action@v3 + with: + file: ./build/docker/Dockerfile.genes + tags: sgosline/genes:latest + push: true + platforms: linux/amd64,linux/arm64 + build-hcmi: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Build and push hcmi + uses: docker/build-push-action@v3 + with: + file: ./build/docker/Dockerfile.hcmi + tags: sgosline/hcmi:latest + push: true + platforms: linux/amd64,linux/arm64 + build-cptac: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Build and push cptac + uses: docker/build-push-action@v3 + with: + file: ./build/docker/Dockerfile.cptac + tags: sgosline/cptac:latest + push: true + platforms: linux/amd64,linux/arm64 + build-mpnst: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Build and push mpnst + uses: docker/build-push-action@v3 + with: + file: ./build/docker/Dockerfile.mpnst + tags: sgosline/mpnst:latest + push: true + platforms: linux/amd64,linux/arm64 + build-beataml: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Build and push beataml + uses: docker/build-push-action@v3 + with: + file: ./build/docker/Dockerfile.beataml + tags: sgosline/beataml:latest + push: true + platforms: linux/amd64,linux/arm64 + build-upload: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Build and push upload + uses: docker/build-push-action@v3 + with: + file: ./build/docker/Dockerfile.upload + tags: sgosline/upload:latest + push: true + platforms: linux/amd64,linux/arm64 + From 3beffa09077606d9d217773ad3400b647104c02f Mon Sep 17 00:00:00 2001 From: Sara JC Gosline Date: Mon, 8 Jul 2024 16:42:14 -0700 Subject: [PATCH 3/7] dockerfiles required update --- build/docker/Dockerfile.beataml | 2 ++ build/docker/Dockerfile.broad_sanger_omics | 3 +-- build/docker/Dockerfile.genes | 8 +++----- build/docker/Dockerfile.hcmi | 7 +------ build/docker/Dockerfile.mpnst | 6 +++--- 5 files changed, 10 insertions(+), 16 deletions(-) diff --git a/build/docker/Dockerfile.beataml b/build/docker/Dockerfile.beataml index 033a21f3..936a2008 100644 --- a/build/docker/Dockerfile.beataml +++ b/build/docker/Dockerfile.beataml @@ -5,6 +5,8 @@ WORKDIR /usr/src/app COPY build/beatAML/GetBeatAML.py . COPY build/utils/fit_curve.py . +COPY build/utils/build_drug_desc.py . +COPY build/utils/tpmFromCounts.py . COPY build/beatAML/*sh ./ COPY build/beatAML/requirements.txt . diff --git a/build/docker/Dockerfile.broad_sanger_omics b/build/docker/Dockerfile.broad_sanger_omics index 9ec2324f..1a47bb8a 100755 --- a/build/docker/Dockerfile.broad_sanger_omics +++ b/build/docker/Dockerfile.broad_sanger_omics @@ -1,7 +1,6 @@ -FROM r-base:4.2.1 +FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update --fix-missing -#RUN apt-get install -y --fix-missing --allow-unauthenticated build-essential libpq-dev python3.10 python3-pip python3-setuptools python3-dev python3-venv libcurl4-openssl-dev libxml2-dev libglpk-dev RUN apt-get install -y --fix-missing --allow-unauthenticated build-essential python3-pip python3-setuptools python3-dev python3-venv libcurl4-openssl-dev libglpk-dev libxml2-dev libpq-dev diff --git a/build/docker/Dockerfile.genes b/build/docker/Dockerfile.genes index 289578e7..61e8fbdd 100755 --- a/build/docker/Dockerfile.genes +++ b/build/docker/Dockerfile.genes @@ -1,10 +1,8 @@ -#FROM bioconductor/bioconductor_docker -FROM r-base:4.2.1 +FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update --fix-missing -RUN apt-get install -y --fix-missing --allow-unauthenticated build-essential libpq-dev libcurl4-openssl-dev -RUN apt-get install -y --fix-missing libxml2-dev -#libxml2 libglpk-dev +RUN apt-get install -y --fix-missing --allow-unauthenticated build-essential python3-pip python3-setuptools python3-dev python3-venv libcurl4-openssl-dev libglpk-dev libxml2-dev libpq-dev + WORKDIR /app diff --git a/build/docker/Dockerfile.hcmi b/build/docker/Dockerfile.hcmi index 61de5a97..e7b9256f 100644 --- a/build/docker/Dockerfile.hcmi +++ b/build/docker/Dockerfile.hcmi @@ -5,14 +5,9 @@ WORKDIR /usr/src/app COPY build/hcmi/01-createHCMISamplesFile.py . COPY build/hcmi/02-getHCMIData.py . COPY build/hcmi/full_manifest.txt . +COPY build/hcmi/requirements.txt . COPY build/hcmi/*sh ./ COPY build/hcmi/hcmi_cancer_types.csv ./ - -COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -# CMD python 01-createHCMISamplesFile.py && \ -# python 02-getHCMIData.py -m full_manifest.txt -t transcriptomics -o hcmi_transcriptomics.csv && \ -# python 02-getHCMIData.py -m full_manifest.txt -M full_manifest_files -t copy_number -o hcmi_copy_number.csv && \ -# python 02-getHCMIData.py -m full_manifest.txt -M full_manifest_files -t mutations -o hcmi_mutations.csv \ No newline at end of file diff --git a/build/docker/Dockerfile.mpnst b/build/docker/Dockerfile.mpnst index 8862b2b0..0204144b 100755 --- a/build/docker/Dockerfile.mpnst +++ b/build/docker/Dockerfile.mpnst @@ -1,4 +1,4 @@ -FROM r-base:4.2.1 +FROM r-base:4.3.2 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update --allow-insecure-repositories #RUN apt-get install -y --allow-unauthenticated build-essential --fix-missing libpq-dev python3-pip python3-setuptools python3-dev python3-venv libcurl4-openssl-dev libxml2-dev @@ -7,14 +7,14 @@ RUN apt-get install -y --allow-unauthenticated build-essential --fix-missing pyt RUN python3 -m venv /opt/venv RUN /opt/venv/bin/pip3 install --upgrade pip + ENV PYTHONPATH "${PYTHONPATH}:/app" WORKDIR /app ADD build/mpnst/requirements.txt . ADD build/mpnst/requirements.r . -ADD build/mpnst/*.R ./ +ADD build/mpnst/* ./ ADD build/utils/* ./ -ADD build/mpnst/*sh ./ # installing python libraries RUN /opt/venv/bin/pip3 install -r requirements.txt From eedab3f1fe7cb4c7bc91e8b3b0553741671e4191 Mon Sep 17 00:00:00 2001 From: Sara JC Gosline Date: Mon, 8 Jul 2024 16:42:47 -0700 Subject: [PATCH 4/7] updated --- build/docker/Dockerfile.broad_sanger_exp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build/docker/Dockerfile.broad_sanger_exp b/build/docker/Dockerfile.broad_sanger_exp index 1b1b1600..baf1fdf6 100755 --- a/build/docker/Dockerfile.broad_sanger_exp +++ b/build/docker/Dockerfile.broad_sanger_exp @@ -1,11 +1,14 @@ -FROM r-base:4.2.1 +FROM r-base:4.4.1 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update --fix-missing +#RUN apt-get install -y --fix-missing --allow-unauthenticated build-essential libpq-dev python3.10 python3-pip python3-setuptools python3-dev python3-venv libcurl4-openssl-dev libxml2-dev libglpk-dev + RUN apt-get install -y --fix-missing --allow-unauthenticated build-essential python3-pip python3-setuptools python3-dev python3-venv libcurl4-openssl-dev libglpk-dev libxml2-dev libpq-dev RUN python3 -m venv /opt/venv RUN /opt/venv/bin/pip3 install --upgrade pip + # Set MPLCONFIGDIR to a writable directory ENV MPLCONFIGDIR=/tmp/matplotlib RUN mkdir -p /tmp/matplotlib From ed64db0fd03b493a0b8e5b15e95dfbb2632fc312 Mon Sep 17 00:00:00 2001 From: Sara JC Gosline Date: Mon, 8 Jul 2024 16:46:41 -0700 Subject: [PATCH 5/7] added file from main --- build/utils/tpmFromCounts.py | 52 ++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 build/utils/tpmFromCounts.py diff --git a/build/utils/tpmFromCounts.py b/build/utils/tpmFromCounts.py new file mode 100644 index 00000000..bafbbc5f --- /dev/null +++ b/build/utils/tpmFromCounts.py @@ -0,0 +1,52 @@ +''' +python script to read in counts matrix and gene lengths to calculate tpm + +''' + +import argparse +import pandas as pd + +def main(): + parser = argparse.ArgumentParser("Quick script to get TPM from counts matrix") + parser.add_argument('--counts') + + + args = parser.parse_args() + print('Creating TPM from '+args.counts) + counts = pd.read_csv(args.counts,sep='\t') + counts.index=counts.stable_id + + ##get list of patients + pats = set(counts.columns)-set(['stable_id','display_label','description','biotype']) + + ##transcript info from grc37 + gtf = pd.read_csv("https://ftp.ensembl.org/pub/grch37/current/gtf/homo_sapiens/Homo_sapiens.GRCh37.87.gtf.gz",sep='\t',comment='#') + gtf.index = [a.split(';')[0].split(' ')[1].strip('"') for a in gtf[gtf.columns[8]]] + ##first select only exons + gtf = gtf[gtf.gene=='exon'] + ##compute length and convert to kilobases + length = abs(gtf[gtf.columns[4]]-gtf[gtf.columns[3]])/1000 ##get difference between start and end, then divide by kb + + #gtf = gtf[gtf.gene=='exon'].groupby(level=0).sum() ##sum over all the exons for a particular gene. yes this doesn't really tell you about which exon + length = length.groupby(level=0).sum() ##sum over all exon lengths for each gene + + + ##set counts matrix + X = counts[list(pats)] + tg = [g for g in X. index if g in length.index] + + X = X.loc[tg].transpose() + length = pd.Series(length)[X.columns] + # length =length.loc[tg] + + ## df = pd.DataFrame(lengths=lengths,Genes=gnames) + C = X.values + L = length.values + N = X.sum(axis=1).values.reshape(-1,1) + rpk = C/L + per_million_scaling_factor = (rpk.sum(axis=1)/1e6).reshape(-1,1) + tpm = pd.DataFrame( rpk/per_million_scaling_factor, index=X.index, columns=X.columns).transpose() + tpm.to_csv('tpm_'+args.counts,sep='\t') + +if __name__=='__main__': + main() From 7ab3a53749bde33bd31714e826a1193f206db2e4 Mon Sep 17 00:00:00 2001 From: Sara JC Gosline Date: Tue, 9 Jul 2024 08:58:05 -0700 Subject: [PATCH 6/7] removed arm build --- .github/workflows/build.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 682ddfe6..2d2f58bd 100755 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,7 +29,7 @@ jobs: file: ./build/docker/Dockerfile.broad_sanger_omics tags: sgosline/broad-sanger-omics:latest push: true - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 build-cellline-exp: runs-on: ubuntu-latest steps: @@ -50,7 +50,7 @@ jobs: file: ./build/docker/Dockerfile.broad_sanger_exp tags: sgosline/broad-sanger-exp:latest push: true - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 build-genes: runs-on: ubuntu-latest steps: @@ -71,7 +71,7 @@ jobs: file: ./build/docker/Dockerfile.genes tags: sgosline/genes:latest push: true - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 build-hcmi: runs-on: ubuntu-latest steps: @@ -92,7 +92,7 @@ jobs: file: ./build/docker/Dockerfile.hcmi tags: sgosline/hcmi:latest push: true - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 build-cptac: runs-on: ubuntu-latest steps: @@ -113,7 +113,7 @@ jobs: file: ./build/docker/Dockerfile.cptac tags: sgosline/cptac:latest push: true - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 build-mpnst: runs-on: ubuntu-latest steps: @@ -134,7 +134,7 @@ jobs: file: ./build/docker/Dockerfile.mpnst tags: sgosline/mpnst:latest push: true - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 build-beataml: runs-on: ubuntu-latest steps: @@ -155,7 +155,7 @@ jobs: file: ./build/docker/Dockerfile.beataml tags: sgosline/beataml:latest push: true - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 build-upload: runs-on: ubuntu-latest steps: @@ -176,6 +176,6 @@ jobs: file: ./build/docker/Dockerfile.upload tags: sgosline/upload:latest push: true - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 From ed7d7dae9cf4b229a210f50d8b503f565cf4d1a1 Mon Sep 17 00:00:00 2001 From: Sara JC Gosline Date: Tue, 9 Jul 2024 09:01:19 -0700 Subject: [PATCH 7/7] forgot this file --- build/utils/build_drug_desc.py | 97 ++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 build/utils/build_drug_desc.py diff --git a/build/utils/build_drug_desc.py b/build/utils/build_drug_desc.py new file mode 100644 index 00000000..7e25212e --- /dev/null +++ b/build/utils/build_drug_desc.py @@ -0,0 +1,97 @@ +''' +build drug descriptor table from drug table + + +''' + + +import argparse +from rdkit import Chem +from rdkit.Chem import AllChem +#from rdkit.Chem import rdFingerprintGenerator +import pandas as pd +import numpy as np +from mordred import Calculator, descriptors +import multiprocessing + + +def smiles_to_fingerprint(smiles): + ''' + takes smiles nad create morgan fingerprint + ''' + fdict = [] + ##get morgan fingerprint + print('Computing morgan fingerprints for '+str(len(smiles))+' SMILES') + # morgan_fp_gen = rdFingerprintGenerator.GetMorganGenerator(radius=2, fpSize=1024, useCountSimulation=False) + for s in smiles: + # print(s) + mol = Chem.MolFromSmiles(s) + try: + #this has been depracated despite being in Alex's original script + fingerprint = AllChem.GetMorganFingerprintAsBitVect(mol, radius=2, nBits=1024) # update these parameters + # fingerprint = morgan_fp_gen.GetFingerprint(mol) + # vec2 = np.array(fp2) + except: + print('Cannot compute fingerprint for '+s) + continue + fingerprint_array = np.array(fingerprint) + fstr = ''.join([str(a) for a in fingerprint_array]) + fdict.append({'smile':s,'descriptor_value':fstr,'structural_descriptor':'morgan fingerprint'}) + + return pd.DataFrame(fdict)#fingerprint_array + + +def smiles_to_mordred(smiles,nproc=2): + ''' + get descriptors - which ones? + ''' + print('Computing mordred descriptors for '+str(len(smiles))+' SMILES') + + + mols = [Chem.MolFromSmiles(s) for s in smiles] + smols = [] + ssmil = [] + for i in range(len(mols)): + m = mols[i] + if m is not None: + smols.append(m) + ssmil.append(smiles[i]) + + calc = Calculator(descriptors, ignore_3D=True) + dd = calc.pandas(mols=smols, nproc=nproc, quiet=False, ipynb=False ) + values = dd.columns + dd['smile'] = ssmil + ##reformat here + longtab = pd.melt(dd,id_vars='smile',value_vars=values) + longtab = longtab.rename({'variable':'structural_descriptor','value':'descriptor_value'},axis=1) + return longtab + +def main(): + parser = argparse.ArgumentParser('Build drug descriptor table') + parser.add_argument('--drugtable',dest='drugtable') + parser.add_argument('--desctable',dest='outtable') + + args = parser.parse_args() + + cores = multiprocessing.cpu_count() + ncors = cores-1 + print("Running with "+str(ncors)+' out of '+str(cores)+' processors') + print('Adding drug table for '+args.drugtable) + tab = pd.read_csv(args.drugtable,sep='\t') + + cansmiles = [a for a in set(tab.canSMILES) if str(a)!='nan'] + # isosmiles = list(set(tab.isoSMILES)) + morgs = smiles_to_fingerprint(cansmiles) + + ids = pd.DataFrame(tab[['improve_drug_id','canSMILES']]).drop_duplicates() + id_morg = ids.rename({"canSMILES":'smile'},axis=1).merge(morgs)[['improve_drug_id','structural_descriptor','descriptor_value']] + + mords = smiles_to_mordred(cansmiles,nproc=ncors) + + id_mord = ids.rename({'canSMILES':'smile'},axis=1).merge(mords)[['improve_drug_id','structural_descriptor','descriptor_value']] + + full = pd.concat([id_morg,id_mord],axis=0) + full.to_csv(args.outtable,sep='\t',index=False,compression='gzip') + +if __name__=='__main__': + main()