From a1f2060502c343a47c8f4d012261b2e1c833c37c Mon Sep 17 00:00:00 2001 From: Jin wook Lee Date: Tue, 2 Nov 2021 21:02:42 -0700 Subject: [PATCH 1/7] bug fix for dx (remove array of optionals, optional array) --- chip.wdl | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/chip.wdl b/chip.wdl index c5e8e2a..f8419d3 100644 --- a/chip.wdl +++ b/chip.wdl @@ -117,9 +117,9 @@ workflow chip { Array[File] bams = [] Array[File] nodup_bams = [] Array[File] tas = [] - Array[File?] peaks = [] - Array[File?] peaks_pr1 = [] - Array[File?] peaks_pr2 = [] + Array[File] peaks = [] + Array[File] peaks_pr1 = [] + Array[File] peaks_pr2 = [] File? peak_ppr1 File? peak_ppr2 File? peak_pooled @@ -1703,7 +1703,7 @@ workflow chip { # we have all tas and ctl_tas (optional for histone chipseq) ready, let's call peaks scatter(i in range(num_rep)) { Boolean has_input_of_call_peak = defined(ta_[i]) - Boolean has_output_of_call_peak = i Date: Tue, 2 Nov 2021 21:03:14 -0700 Subject: [PATCH 2/7] add example for terra --- ...CSR000DYI_subsampled_chr19_only.terra.json | 15 ++++++++++++ ...CSR936XTK_subsampled_chr19_only.terra.json | 23 +++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 example_input_json/terra/ENCSR000DYI_subsampled_chr19_only.terra.json create mode 100644 example_input_json/terra/ENCSR936XTK_subsampled_chr19_only.terra.json diff --git a/example_input_json/terra/ENCSR000DYI_subsampled_chr19_only.terra.json b/example_input_json/terra/ENCSR000DYI_subsampled_chr19_only.terra.json new file mode 100644 index 0000000..df353d2 --- /dev/null +++ b/example_input_json/terra/ENCSR000DYI_subsampled_chr19_only.terra.json @@ -0,0 +1,15 @@ +{ + "chip.pipeline_type" : "tf", + "chip.genome_tsv" : "gs://encode-pipeline-genome-data/genome_tsv/v3/hg38_chr19_chrM.terra.tsv", + "chip.fastqs_rep1_R1" : ["gs://encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq_subsampled/rep1.subsampled.25.fastq.gz" + ], + "chip.fastqs_rep2_R1" : ["gs://encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq_subsampled/rep2.subsampled.20.fastq.gz" + ], + "chip.ctl_fastqs_rep1_R1" : ["gs://encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq_subsampled/ctl1.subsampled.25.fastq.gz" + ], + "chip.ctl_fastqs_rep2_R1" : ["gs://encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI/fastq_subsampled/ctl2.subsampled.25.fastq.gz" + ], + "chip.paired_end" : false, + "chip.title" : "ENCSR000DYI (subsampled 1/25, chr19_chrM only)", + "chip.description" : "CEBPB ChIP-seq on human A549 produced by the Snyder lab" +} diff --git a/example_input_json/terra/ENCSR936XTK_subsampled_chr19_only.terra.json b/example_input_json/terra/ENCSR936XTK_subsampled_chr19_only.terra.json new file mode 100644 index 0000000..7aec54d --- /dev/null +++ b/example_input_json/terra/ENCSR936XTK_subsampled_chr19_only.terra.json @@ -0,0 +1,23 @@ +{ + "chip.pipeline_type" : "tf", + "chip.genome_tsv" : "gs://encode-pipeline-genome-data/genome_tsv/v3/hg38_chr19_chrM.terra.tsv", + "chip.fastqs_rep1_R1" : ["gs://encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq_subsampled/rep1-R1.subsampled.50.fastq.gz" + ], + "chip.fastqs_rep1_R2" : ["gs://encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq_subsampled/rep1-R2.subsampled.50.fastq.gz" + ], + "chip.fastqs_rep2_R1" : ["gs://encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq_subsampled/rep2-R1.subsampled.50.fastq.gz" + ], + "chip.fastqs_rep2_R2" : ["gs://encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq_subsampled/rep2-R2.subsampled.50.fastq.gz" + ], + "chip.ctl_fastqs_rep1_R1" : ["gs://encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq_subsampled/ctl1-R1.subsampled.80.fastq.gz" + ], + "chip.ctl_fastqs_rep1_R2" : ["gs://encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq_subsampled/ctl1-R2.subsampled.80.fastq.gz" + ], + "chip.ctl_fastqs_rep2_R1" : ["gs://encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq_subsampled/ctl2-R1.subsampled.80.fastq.gz" + ], + "chip.ctl_fastqs_rep2_R2" : ["gs://encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR936XTK/fastq_subsampled/ctl2-R2.subsampled.80.fastq.gz" + ], + "chip.paired_end" : true, + "chip.title" : "ENCSR936XTK (subsampled 1/50, chr19 and chrM Only)", + "chip.description" : "ZNF143 ChIP-seq on human GM12878" +} From ccb939c61b599f6e23b175d8d85c4b58cb52d059 Mon Sep 17 00:00:00 2001 From: Jin wook Lee Date: Wed, 3 Nov 2021 16:54:42 -0700 Subject: [PATCH 3/7] fix java issue --- .circleci/config.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b09758f..fb7c207 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,12 +31,20 @@ commands: steps: - run: command: | - sudo apt-get update && sudo apt-get install software-properties-common git wget curl default-jre -y + sudo apt-get update && sudo apt-get install software-properties-common git wget curl -y + + # install java 11 + sudo add-apt-repository ppa:openjdk-r/ppa -y + sudo apt-get update && sudo apt-get install openjdk-11-jdk -y + # automatically set 11 as default java + sudo update-java-alternatives -a + sudo add-apt-repository ppa:deadsnakes/ppa -y sudo apt-get update && sudo apt-get install python3.6 -y sudo wget --no-check-certificate https://bootstrap.pypa.io/get-pip.py sudo python3.6 get-pip.py sudo ln -s /usr/bin/python3.6 /usr/local/bin/python3 + pip3 install --upgrade pip pip3 install caper google-cloud-storage From fcd9890bf50daf9bcec28dc1c1769eec82de8825 Mon Sep 17 00:00:00 2001 From: Jin wook Lee Date: Wed, 3 Nov 2021 16:54:47 -0700 Subject: [PATCH 4/7] update doc --- README.md | 81 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 61 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 52c324a..9db41d6 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,33 @@ ## Download new Caper>=2.0 -New Caper is out. You need to update your Caper to work with the latest ENCODE ATAC-seq pipeline. +New Caper is out. You need to update your Caper to work with the latest ENCODE ChIP-seq pipeline. ```bash $ pip install caper --upgrade ``` +## Local/HPC users and new Caper>=2.0 + +There are tons of changes for local/HPC backends: `local`, `slurm`, `sge`, `pbs` and `lsf`(added). Make a backup of your current Caper configuration file `~/.caper/default.conf` and run `caper init`. Local/HPC users need to reset/initialize Caper's configuration file according to your chosen backend. Edit the configuration file and follow instructions in there. +```bash +$ cd ~/.caper +$ cp default.conf default.conf.bak +$ caper init [YOUR_BACKEND] +``` + +In order to run a pipeline, you need to add one of the following flags to specify the environment to run each task within. i.e. `--conda`, `--singularity` and `--docker`. These flags are not required for cloud backend users (`aws` and `gcp`). +```bash +# for example +$ caper run ... --singularity +``` + +For Conda users, **RE-INSTALL PIPELINE'S CONDA ENVIRONMENT AND DO NOT ACTIVATE CONDA ENVIRONMENT BEFORE RUNNING PIPELINES**. Caper will internally call `conda run -n ENV_NAME CROMWELL_JOB_SCRIPT`. Just make sure that pipeline's new Conda environments are correctly installed. +```bash +$ scripts/uninstall_conda_env.sh +$ scripts/install_conda_env.sh +``` + + ## Introduction This ChIP-Seq pipeline is based off the ENCODE (phase-3) transcription factor and histone ChIP-seq pipeline specifications (by Anshul Kundaje) in [this google doc](https://docs.google.com/document/d/1lG_Rd7fnYgRpSIqrIfuVlAz2dW1VaSQThzk836Db99c/edit#). @@ -21,34 +43,35 @@ This ChIP-Seq pipeline is based off the ENCODE (phase-3) transcription factor an ## Installation - 1) Make sure that you have Python>=3.6. Caper does not work with Python2. Install Caper and check its version >=2.0. ```bash $ python --version $ pip install caper - $ caper -v ``` +2) Make a backup of your Caper configuration file `~/.caper/default.conf` if you are upgrading from old Caper(<2.0.0). Reset/initialize Caper's configuration file. Read Caper's [README](https://github.com/ENCODE-DCC/caper/blob/master/README.md) carefully to choose a backend for your system. Follow the instruction in the configuration file. + ```bash + # make a backup of ~/.caper/default.conf if you already have it + $ caper init [YOUR_BACKEND] -2) Git clone this pipeline. - > **IMPORTANT**: use `~/chip-seq-pipeline2/chip.wdl` as `[WDL]` in Caper's documentation. + # then edit ~/.caper/default.conf + $ vi ~/.caper/default.conf + ``` +3) Git clone this pipeline. + > **IMPORTANT**: use `~/chip-seq-pipeline2/chip.wdl` as `[WDL]` in Caper's documentation. ```bash $ cd $ git clone https://github.com/ENCODE-DCC/chip-seq-pipeline2 ``` - -3) (Optional for Conda) Install pipeline's Conda environments if you don't have Singularity or Docker installed on your system. We recommend to use Singularity instead of Conda. If you don't have Conda on your system, install [Miniconda3](https://docs.conda.io/en/latest/miniconda.html). +4) (Optional for Conda) Install pipeline's Conda environments if you don't have Singularity or Docker installed on your system. We recommend to use Singularity instead of Conda. If you don't have Conda on your system, install [Miniconda3](https://docs.conda.io/en/latest/miniconda.html). ```bash $ cd chip-seq-pipeline2 + # uninstall old environments (<2.0.0) + $ bash scripts/uninstall_conda_env.sh $ bash scripts/install_conda_env.sh ``` -4) Follow [Caper's README](https://github.com/ENCODE-DCC/caper) carefully. Find an instruction for your platform and run `caper init`. Edit the initialized Caper's configuration file (`~/.caper/default.conf`). - ```bash - $ caper init [YOUR_PLATFORM] - $ vi ~/.caper/default.conf - ``` ## Test run @@ -63,10 +86,35 @@ The followings are just examples. Please read [Caper's README](https://github.co # Or submit it as a leader job (with long/enough resources) to SLURM (Stanford Sherlock) with Singularity # It will fail if you directly run the leader job on login nodes - $ sbatch -p [SLURM_PARTITON] -J [WORKFLOW_NAME] --export=ALL --mem 4G -t 4-0 --wrap "caper chip atac.wdl -i https://storage.googleapis.com/encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI_subsampled_chr19_only.json --singularity" + $ sbatch -p [SLURM_PARTITON] -J [WORKFLOW_NAME] --export=ALL --mem 4G -t 4-0 --wrap "caper chip chip.wdl -i https://storage.googleapis.com/encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI_subsampled_chr19_only.json --singularity" ``` +## Running a pipeline on Terra/Anvil (using Dockstore) + +Visit our pipeline repo on [Dockstore](https://dockstore.org/workflows/github.com/ENCODE-DCC/chip-seq-pipeline2). Click on `Terra` or `Anvil`. Follow Terra's instruction to create a workspace on Terra and add Terra's billing bot to your Google Cloud account. + +Download this [test input JSON for Terra](https://storage.googleapis.com/encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI_subsampled_chr19_only.terra.json) and upload it to Terra's UI and then run analysis. + +If you want to use your own input JSON file, then make sure that all files in the input JSON are on a Google Cloud Storage bucket (`gs://`). URLs will not work. + + +## Running a pipeline on DNAnexus (using Dockstore) + +Sign up for a new account on [DNAnexus](https://platform.dnanexus.com/) and create a new project on either AWS or Azure. Visit our pipeline repo on [Dockstore](https://dockstore.org/workflows/github.com/ENCODE-DCC/chip-seq-pipeline2). Click on `DNAnexus`. Choose a destination directory on your DNAnexus project. Click on `Submit` and visit DNAnexus. This will submit a conversion job so that you can check status of it on `Monitor` on DNAnexus UI. + +Once conversion is done download one of the following input JSON files according to your chosen platform (AWS or Azure) for your DNAnexus project: +- AWS: https://storage.googleapis.com/encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI_subsampled_chr19_only_dx.json +- Azure: https://storage.googleapis.com/encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI_subsampled_chr19_only_dx_azure.json + +You cannot use these input JSON files directly. Go to the destination directory on DNAnexus and click on the converted workflow `chip`. You will see input file boxes in the left-hand side of the task graph. Expand it and define FASTQs (`fastq_repX_R1`) and `genome_tsv` as in the downloaded input JSON file. Click on the `common` task box and define other non-file pipeline parameters. + + +## Running a pipeline on DNAnexus (using our pre-built workflows) + +See [this](docs/tutorial_dx_web.md) for details. + + ## Input JSON file @@ -82,13 +130,6 @@ You can run this pipeline on [truwl.com](https://truwl.com/). This provides a we If you do not run the pipeline on Truwl, you can still share your use-case/job on the platform by getting in touch at [info@truwl.com](mailto:info@truwl.com) and providing your inputs.json file. -## Running a pipeline on DNAnexus - -You can also run this pipeline on DNAnexus without using Caper or Cromwell. There are two ways to build a workflow on DNAnexus based on our WDL. - -1) [dxWDL CLI](docs/tutorial_dx_cli.md) -2) [DNAnexus Web UI](docs/tutorial_dx_web.md) - ## How to organize outputs Install [Croo](https://github.com/ENCODE-DCC/croo#installation). **You can skip this installation if you have installed pipeline's Conda environment and activated it**. Make sure that you have python3(> 3.4.1) installed on your system. Find a `metadata.json` on Caper's output directory. From 28caf5a062c1eb2efe061728404fab6c268dcc15 Mon Sep 17 00:00:00 2001 From: Jin wook Lee Date: Thu, 4 Nov 2021 10:10:12 -0700 Subject: [PATCH 5/7] bump ver --- chip.wdl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/chip.wdl b/chip.wdl index f8419d3..58cd0fc 100644 --- a/chip.wdl +++ b/chip.wdl @@ -7,16 +7,16 @@ struct RuntimeEnvironment { } workflow chip { - String pipeline_ver = 'v2.0.0' + String pipeline_ver = 'v2.0.1' meta { - version: 'v2.0.0' + version: 'v2.0.1' author: 'Jin wook Lee (leepc12@gmail.com) at ENCODE-DCC' description: 'ENCODE TF/Histone ChIP-Seq pipeline' specification_document: 'https://docs.google.com/document/d/1lG_Rd7fnYgRpSIqrIfuVlAz2dW1VaSQThzk836Db99c/edit?usp=sharing' - default_docker: 'encodedcc/chip-seq-pipeline:v2.0.0' - default_singularity: 'library://leepc12/default/chip-seq-pipeline:v2.0.0' + default_docker: 'encodedcc/chip-seq-pipeline:v2.0.1' + default_singularity: 'library://leepc12/default/chip-seq-pipeline:v2.0.1' croo_out_def: 'https://storage.googleapis.com/encode-pipeline-output-definition/chip.croo.v5.json' parameter_group: { @@ -67,8 +67,8 @@ workflow chip { } input { # group: runtime_environment - String docker = 'encodedcc/chip-seq-pipeline:v2.0.0' - String singularity = 'library://leepc12/default/chip-seq-pipeline:v2.0.0' + String docker = 'encodedcc/chip-seq-pipeline:v2.0.1' + String singularity = 'library://leepc12/default/chip-seq-pipeline:v2.0.1' String conda = 'encode-chip-seq-pipeline' String conda_macs2 = 'encode-chip-seq-pipeline-macs2' String conda_spp = 'encode-chip-seq-pipeline-spp' From 620185b5e6ef6054288f16dcee0d08d47a0c9ef2 Mon Sep 17 00:00:00 2001 From: Jin wook Lee Date: Thu, 4 Nov 2021 12:08:53 -0700 Subject: [PATCH 6/7] fix test: add docker to input json for workflow tests --- .circleci/config.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fb7c207..38d3cd5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -59,11 +59,17 @@ commands: echo ${GCLOUD_SERVICE_ACCOUNT_SECRET_JSON} > tmp_secret_key.json export GOOGLE_APPLICATION_CREDENTIALS=$PWD/tmp_secret_key.json + + # add docker image to input JSON + cat ${INPUT} | jq ".+{\"chip.docker\": \"${TAG}\"}" > input_with_docker.json + caper run ../../../chip.wdl \ --backend gcp --gcp-prj ${GOOGLE_PROJECT_ID} \ --gcp-service-account-key-json $PWD/tmp_secret_key.json \ --out-gcs-bucket ${CAPER_OUT_DIR} --tmp-gcs-bucket ${CAPER_TMP_DIR} \ - -i ${INPUT} -m metadata.json --docker ${TAG} + -i input_with_docker.json -m metadata.json --docker ${TAG} + + rm -f input_with_docker.json res=$(jq '.outputs["chip.qc_json_ref_match"]' metadata.json) [[ "$res" != true ]] && exit 100 From 769ca5a66e76b49bbb5fc6774cf12f6a2dbb8a2b Mon Sep 17 00:00:00 2001 From: Jin wook Lee Date: Thu, 4 Nov 2021 13:12:37 -0700 Subject: [PATCH 7/7] metadata (author, description) for dockstore --- chip.wdl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/chip.wdl b/chip.wdl index 58cd0fc..1e1e652 100644 --- a/chip.wdl +++ b/chip.wdl @@ -11,8 +11,12 @@ workflow chip { meta { version: 'v2.0.1' - author: 'Jin wook Lee (leepc12@gmail.com) at ENCODE-DCC' - description: 'ENCODE TF/Histone ChIP-Seq pipeline' + + author: 'Jin wook Lee' + email: 'leepc12@gmail.com' + description: 'ENCODE TF/Histone ChIP-Seq pipeline. See https://github.com/ENCODE-DCC/chip-seq-pipeline2 for more details. e.g. example input JSON for Terra/Anvil.' + organization: 'ENCODE DCC' + specification_document: 'https://docs.google.com/document/d/1lG_Rd7fnYgRpSIqrIfuVlAz2dW1VaSQThzk836Db99c/edit?usp=sharing' default_docker: 'encodedcc/chip-seq-pipeline:v2.0.1'