add nextflow schema

CHANGELOG.md

@@ -1,3 +1,11 @@
+# viash_project_template 0.3.0
+
+* Update to Viash 0.8.2
+
+    * `demo_pipeline` is now built from `src/workflows/demo_pipeline`
+    * State management overhaul
+    * Use of helper functions (dependencies) from [`vsh_utils`](https://viash-hub.com/data-intuitive/vsh_utils)
+
 # viash_project_template 0.2.0
 
 * Update to Viash 0.7.1.
@@ -22,4 +30,4 @@ Components:
 
 Pipelines:
 
-* `workflows/demo_pipeline/main.nf`: A demo pipeline which uses abovementioned components.
+* `workflows/demo_pipeline/main.nf`: A demo pipeline which uses abovementioned components.

_viash.yaml

@@ -1,10 +1,10 @@
-viash_version: 0.7.1
+viash_version: 0.8.4
 
 source: src
 target: target
 
 config_mods: |
-  .functionality.version := 'dev'
+  .functionality.version := '0.2.2'
   .platforms[.type == 'docker'].target_registry := 'ghcr.io'
   .platforms[.type == 'docker'].target_organization := 'viash-io/viash_project_template'
   .platforms[.type == 'docker'].target_image_source := 'https://github.com/viash-io/viash_project_template'

resources_test/input.csv

@@ -0,0 +1,3 @@
+"id","input","column"
+"1","file1.tsv","2"
+"2","file2.tsv","1"

src/demo/combine_columns/config.vsh.yaml → src/template/combine_columns/config.vsh.yaml

@@ -1,6 +1,6 @@
 functionality:
   name: combine_columns
-  namespace: demo
+  namespace: template
   arguments:
     - name: "--input"
       alternatives: [ "-i" ]
@@ -12,13 +12,10 @@ functionality:
       type: file
       required: true
       direction: output
-    - type: string
-      name: --id
-      default: "combine_columns"
   resources:
     - type: r_script
       path: script.R
 platforms:
   - type: docker
     image: eddelbuettel/r2u:22.04
-  - type: nextflow
+  - type: nextflow

src/demo/remove_comments/config.vsh.yaml → src/template/remove_comments/config.vsh.yaml

@@ -1,6 +1,6 @@
 functionality:
   name: remove_comments
-  namespace: demo
+  namespace: template
   arguments:
     - name: "--input"
       alternatives: [ "-i" ]
@@ -13,15 +13,12 @@ functionality:
       required: true
       direction: output
       example: "file.tsv"
-    - type: string
-      name: --id
-      default: "remove_comments"
   resources:
-  - type: bash_script
-    path: script.sh
+    - type: bash_script
+      path: script.sh
   test_resources:
-  - type: bash_script
-    path: test.sh
+    - type: bash_script
+      path: test.sh
 platforms:
   - type: docker
     image: ubuntu:20.04

src/demo/take_column/config.vsh.yaml → src/template/take_column/config.vsh.yaml

@@ -1,6 +1,6 @@
 functionality:
   name: take_column
-  namespace: demo
+  namespace: template
   arguments:
     - name: "--input"
       alternatives: [ "-i" ]
@@ -15,9 +15,6 @@ functionality:
       type: integer
       required: false
       default: 2
-    - type: string
-      name: --id
-      default: "take_column"
   resources:
   - type: python_script
     path: script.py
@@ -26,5 +23,9 @@ platforms:
     image: python:3.10-slim
     setup:
       - type: python
-        packages: pandas
+        packages: 
+          - pandas
+      - type: apt
+        packages: 
+          - procps
   - type: nextflow

src/template/workflow/config.vsh.yaml

@@ -0,0 +1,54 @@
+functionality:
+  name: workflow
+  namespace: template
+  description: |
+    An example pipeline and project template.
+
+    Multiple TSV files can be input, each with a 'column' identifier that
+    should be extracted. All extracted columns are then collated again.
+
+  arguments:
+    - name: "--input"
+      alternatives: [ "-i" ]
+      type: file
+      required: true
+      description: Input TSV file
+      example: file1.tar.gz 
+    - name: "--column"
+      type: integer
+      required: false
+      default: 2
+      description: The column index to extract from the TSV input file
+    - name: "--output"
+      alternatives: [ "-o" ]
+      type: file
+      direction: output
+      required: true
+      description: Output TSV file
+      example: output.tsv
+
+  resources:
+    - type: nextflow_script
+      path: main.nf
+      entrypoint: run_wf
+
+  dependencies: 
+    - name: template/combine_columns
+      repository: local
+    - name: template/remove_comments
+      repository: local
+    - name: template/take_column
+      repository: local
+    - name: join/vsh_toList
+      repository: vsh-pipeline-operators
+
+  repositories:
+    - name: local
+      type: local
+    - name: vsh-pipeline-operators
+      type: vsh
+      repo: data-intuitive/vsh-pipeline-operators
+      tag: v0.2.0
+
+platforms:
+  - type: nextflow

src/template/workflow/main.nf

@@ -0,0 +1,53 @@
+workflow run_wf {
+  take:
+    input_ch
+
+  main:
+
+    output_ch =
+
+      input_ch
+
+        // Remove comments from each TSV input file
+        | remove_comments.run(
+            fromState: [ input: "input" ],
+            toState: { id, result, state -> state + result }
+          )
+
+        // Extract a single column from each TSV
+        // The column to extract is specified in the sample sheet
+        | take_column.run(
+            fromState:
+              [
+                input: "output",
+                column: "column"
+              ],
+            toState: { id, result, state -> result }
+          )
+
+        // Helper module with extra functionality around
+        // nextflow's `toSortedList` operator to reformat 
+        // its output list into a channel item that can be used
+        // directly with downstream components.
+        | vsh_toList.run(
+            args: [ id: "combined" ],
+            fromState: [ input: "output" ],
+            toState: [ output: "output" ]
+          )
+
+        // Concatenate TSVs into one
+        // and prep the output state.
+        | combine_columns.run(
+            auto: [ publish: true ],
+            fromState: [ input: "output" ],
+            toState: { id, result, state ->
+              [
+                output: result.output,
+                _meta: [ join_id: "1" ]
+              ]
+            }
+          )
+
+  emit:
+    output_ch
+}