From ca92c1bf641dc37022dd9137ed3c4f47f6948565 Mon Sep 17 00:00:00 2001
From: Robrecht Cannoodt <rcannood@gmail.com>
Date: Thu, 21 Mar 2024 14:02:44 +0100
Subject: [PATCH] update scripts

---
 scripts/resources_s3.sh                       |  3 ++
 ...-code_remote-input_s3-output_s3-work_s3.sh | 54 +++++++++++++++++++
 ...al-input_local-output_local-work_local.sh} |  8 +--
 ...te-input_local-output_local-work_local.sh} |  9 ++--
 ...remote-input_s3-output_local-work_local.sh | 24 +++++++++
 ...de_remote-input_s3-output_s3-work_local.sh | 33 ++++++++++++
 scripts/{test.sh => verify_output.sh}         |  0
 7 files changed, 125 insertions(+), 6 deletions(-)
 create mode 100644 scripts/resources_s3.sh
 create mode 100755 scripts/run_batch-code_remote-input_s3-output_s3-work_s3.sh
 rename scripts/{run_local.sh => run_local-code_local-input_local-output_local-work_local.sh} (64%)
 rename scripts/{run_local_with_remote.sh => run_local-code_remote-input_local-output_local-work_local.sh} (66%)
 create mode 100755 scripts/run_local-code_remote-input_s3-output_local-work_local.sh
 create mode 100755 scripts/run_local-code_remote-input_s3-output_s3-work_local.sh
 rename scripts/{test.sh => verify_output.sh} (100%)

diff --git a/scripts/resources_s3.sh b/scripts/resources_s3.sh
new file mode 100644
index 0000000..233c0b5
--- /dev/null
+++ b/scripts/resources_s3.sh
@@ -0,0 +1,3 @@
+set -e
+
+aws s3 sync resources s3://data-intuitive-tmp/test-nextflow-wave-fusion/resources --delete
\ No newline at end of file
diff --git a/scripts/run_batch-code_remote-input_s3-output_s3-work_s3.sh b/scripts/run_batch-code_remote-input_s3-output_s3-work_s3.sh
new file mode 100755
index 0000000..0f66b23
--- /dev/null
+++ b/scripts/run_batch-code_remote-input_s3-output_s3-work_s3.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -e
+
+script_name=$(basename "$0")
+TMPOUT=s3://data-intuitive-tmp/test-nextflow-wave-fusion/output/${script_name%.sh}
+OUT=output/${script_name%.sh}
+RES=s3://data-intuitive-tmp/test-nextflow-wave-fusion/resources
+WORK=s3://data-intuitive-tmp/test-nextflow-wave-fusion/work/${script_name%.sh}
+NXF_CONFIG=/tmp/${script_name%.sh}.config
+
+# clear output dir
+echo "Clearing output directory"
+aws s3 rm $TMPOUT --recursive
+[ -d "$OUT" ] && rm -r "$OUT"
+
+# run component
+echo "Running component"
+
+cat > $NXF_CONFIG <<EOF
+process {
+    executor = 'awsbatch'
+    queue = 'di_compute'
+    container = 'bash:4.0'
+}
+
+aws {
+    region = 'eu-west-1'
+    batch {
+        cliPath = '/home/ec2-user/miniconda/bin/aws'
+    }
+}
+EOF
+
+
+NXF_VER=23.10.0 nextflow run \
+  viash-io/test-nextflow-wave-fusion \
+  -r main_build \
+  -main-script target/nextflow/method/main.nf \
+  -profile docker \
+  -w $WORK \
+  -c $NXF_CONFIG \
+  -latest \
+  --input $RES/input1.txt \
+  --multiple_input "$RES/input1.txt;$RES/input2.txt" \
+  --publish_dir "$TMPOUT"
+
+# sync output
+echo "Syncing output"
+aws s3 sync $TMPOUT $OUT
+
+# check if output is correct
+echo "Checking output"
+scripts/verify_output.sh "$OUT"
diff --git a/scripts/run_local.sh b/scripts/run_local-code_local-input_local-output_local-work_local.sh
similarity index 64%
rename from scripts/run_local.sh
rename to scripts/run_local-code_local-input_local-output_local-work_local.sh
index 753c300..12c6d9a 100755
--- a/scripts/run_local.sh
+++ b/scripts/run_local-code_local-input_local-output_local-work_local.sh
@@ -2,7 +2,9 @@
 
 set -e
 
-OUT=output/local
+script_name=$(basename "$0")
+OUT=output/${script_name%.sh}
+RES=resources
 
 # clear output dir
 [ -d "$OUT" ] && rm -r "$OUT"
@@ -12,8 +14,8 @@ NXF_VER=23.10.0 nextflow run \
   . \
   -main-script target/nextflow/method/main.nf \
   -profile docker \
-  --input resources/input1.txt \
-  --multiple_input "resources/input1.txt;resources/input2.txt" \
+  --input $RES/input1.txt \
+  --multiple_input "$RES/input1.txt;$RES/input2.txt" \
   --publish_dir "$OUT"
 
 # check if output is correct
diff --git a/scripts/run_local_with_remote.sh b/scripts/run_local-code_remote-input_local-output_local-work_local.sh
similarity index 66%
rename from scripts/run_local_with_remote.sh
rename to scripts/run_local-code_remote-input_local-output_local-work_local.sh
index c2bbe4b..9021f3b 100755
--- a/scripts/run_local_with_remote.sh
+++ b/scripts/run_local-code_remote-input_local-output_local-work_local.sh
@@ -2,7 +2,9 @@
 
 set -e
 
-OUT=output/local_with_remote
+script_name=$(basename "$0")
+OUT=output/${script_name%.sh}
+RES=resources
 
 # clear output dir
 [ -d "$OUT" ] && rm -r "$OUT"
@@ -13,8 +15,9 @@ NXF_VER=23.10.0 nextflow run \
   -r main_build \
   -main-script target/nextflow/method/main.nf \
   -profile docker \
-  --input resources/input1.txt \
-  --multiple_input "resources/input1.txt;resources/input2.txt" \
+  -latest \
+  --input $RES/input1.txt \
+  --multiple_input "$RES/input1.txt;$RES/input2.txt" \
   --publish_dir "$OUT"
 
 # check if output is correct
diff --git a/scripts/run_local-code_remote-input_s3-output_local-work_local.sh b/scripts/run_local-code_remote-input_s3-output_local-work_local.sh
new file mode 100755
index 0000000..3a68a7a
--- /dev/null
+++ b/scripts/run_local-code_remote-input_s3-output_local-work_local.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+script_name=$(basename "$0")
+OUT=output/${script_name%.sh}
+RES=s3://data-intuitive-tmp/test-nextflow-wave-fusion/resources
+
+# clear output dir
+[ -d "$OUT" ] && rm -r "$OUT"
+
+# run component
+NXF_VER=23.10.0 nextflow run \
+  viash-io/test-nextflow-wave-fusion \
+  -r main_build \
+  -main-script target/nextflow/method/main.nf \
+  -profile docker \
+  -latest \
+  --input $RES/input1.txt \
+  --multiple_input "$RES/input1.txt;$RES/input2.txt" \
+  --publish_dir "$OUT"
+
+# check if output is correct
+scripts/verify_output.sh "$OUT"
diff --git a/scripts/run_local-code_remote-input_s3-output_s3-work_local.sh b/scripts/run_local-code_remote-input_s3-output_s3-work_local.sh
new file mode 100755
index 0000000..333a6ea
--- /dev/null
+++ b/scripts/run_local-code_remote-input_s3-output_s3-work_local.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -e
+
+script_name=$(basename "$0")
+TMPOUT=s3://data-intuitive-tmp/test-nextflow-wave-fusion/output/${script_name%.sh}
+OUT=output/${script_name%.sh}
+RES=s3://data-intuitive-tmp/test-nextflow-wave-fusion/resources
+
+# clear output dir
+echo "Clearing output directory"
+aws s3 rm $TMPOUT --recursive
+[ -d "$OUT" ] && rm -r "$OUT"
+
+# run component
+echo "Running component"
+NXF_VER=23.10.0 nextflow run \
+  viash-io/test-nextflow-wave-fusion \
+  -r main_build \
+  -main-script target/nextflow/method/main.nf \
+  -profile docker \
+  -latest \
+  --input $RES/input1.txt \
+  --multiple_input "$RES/input1.txt;$RES/input2.txt" \
+  --publish_dir "$TMPOUT"
+
+# sync output
+echo "Syncing output"
+aws s3 sync $TMPOUT $OUT
+
+# check if output is correct
+echo "Checking output"
+scripts/verify_output.sh "$OUT"
diff --git a/scripts/test.sh b/scripts/verify_output.sh
similarity index 100%
rename from scripts/test.sh
rename to scripts/verify_output.sh