Various fixes (#4)

* fix compute env * bump memory reqs * use main openproblems repo * update submodule * bump labels
openproblems-bio · Nov 25, 2024 · bda48b6 · bda48b6
1 parent 0bd597e
commit bda48b6
Show file tree

Hide file tree

Showing 14 changed files with 24 additions and 30 deletions.
diff --git a/_viash.yaml b/_viash.yaml
@@ -86,16 +86,10 @@ info:
       dest: resources_test/task_predict_modality
 
 repositories:
-  - name: openproblems_v2
+  - name: openproblems
     type: github
-    repo: openproblems-bio/openproblems-v2
+    repo: openproblems-bio/openproblems
     tag: main_build
-  - name: core
-    type: github
-    repo: openproblems-bio/core
-    # tag: build/main
-    tag: build/add_common_components
-    path: viash/core
 
 config_mods: |
   .runners[.type == "nextflow"].config.labels := { lowmem : "memory = 20.Gb", midmem : "memory = 50.Gb", highmem : "memory = 100.Gb", lowcpu : "cpus = 5", midcpu : "cpus = 15", highcpu : "cpus = 30", lowtime : "time = 1.h", midtime : "time = 4.h", hightime : "time = 8.h", veryhightime : "time = 24.h" }
diff --git a/common b/common
diff --git a/scripts/create_datasets/resources.sh b/scripts/create_datasets/resources.sh
@@ -19,7 +19,7 @@ tw launch https://github.com/openproblems-bio/task_predict_modality.git \
   --pull-latest \
   --main-script target/nextflow/workflows/process_datasets/main.nf \
   --workspace 53907369739130 \
-  --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
+  --compute-env 5DwwhQoBi0knMSGcwThnlF \
   --params-file /tmp/params.yaml \
   --entry-name auto \
   --config common/nextflow_helpers/labels_tw.config \

diff --git a/scripts/run_benchmark/run_full_seqeracloud.sh b/scripts/run_benchmark/run_full_seqeracloud.sh
@@ -14,8 +14,8 @@ publish_dir="s3://openproblems-data/resources/task_predict_modality/results/${RU
 
 # write the parameters to file
 cat > /tmp/params.yaml << HERE
-input_states: s3://openproblems-data/resources/task_predict_modality/datasets/**/state.yaml
-rename_keys: 'input_train_mod1:output_train_mod1;input_train_mod2:output_train_mod2;input_test_mod1:output_test_mod1;input_test_mod2:output_test_mod2"
+input_states: "s3://openproblems-data/resources/task_predict_modality/datasets/**/state.yaml"
+rename_keys: "input_train_mod1:output_train_mod1;input_train_mod2:output_train_mod2;input_test_mod1:output_test_mod1;input_test_mod2:output_test_mod2"
 output_state: "state.yaml"
 publish_dir: "$publish_dir"
 HERE
@@ -25,7 +25,7 @@ tw launch https://github.com/openproblems-bio/task_predict_modality.git \
   --pull-latest \
   --main-script target/nextflow/workflows/run_benchmark/main.nf \
   --workspace 53907369739130 \
-  --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
+  --compute-env 5DwwhQoBi0knMSGcwThnlF \
   --params-file /tmp/params.yaml \
   --entry-name auto \
   --config common/nextflow_helpers/labels_tw.config \

diff --git a/scripts/run_benchmark/run_test_seqeracloud.sh b/scripts/run_benchmark/run_test_seqeracloud.sh
@@ -34,7 +34,7 @@ tw launch https://github.com/openproblems-bio/task_predict_modality.git \
   --pull-latest \
   --main-script target/nextflow/workflows/run_benchmark/main.nf \
   --workspace 53907369739130 \
-  --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
+  --compute-env 5DwwhQoBi0knMSGcwThnlF \
   --params-file /tmp/params.yaml \
   --config common/nextflow_helpers/labels_tw.config \
   --labels task_predict_modality,test
diff --git a/src/control_methods/mean_per_gene/config.vsh.yaml b/src/control_methods/mean_per_gene/config.vsh.yaml
@@ -13,5 +13,5 @@ runners:
   - type: executable
   - type: nextflow
     directives:
-      label: [midtime, lowmem, lowcpu]
+      label: [midtime, midmem, lowcpu]
 
diff --git a/src/control_methods/random_predict/config.vsh.yaml b/src/control_methods/random_predict/config.vsh.yaml
@@ -13,4 +13,4 @@ runners:
   - type: executable
   - type: nextflow
     directives:
-      label: [midtime, lowmem, lowcpu]
+      label: [midtime, midmem, lowcpu]
diff --git a/src/control_methods/solution/config.vsh.yaml b/src/control_methods/solution/config.vsh.yaml
@@ -13,4 +13,4 @@ runners:
   - type: executable
   - type: nextflow
     directives:
-      label: [midtime, lowmem, lowcpu]
+      label: [midtime, midmem, lowcpu]
diff --git a/src/control_methods/zeros/config.vsh.yaml b/src/control_methods/zeros/config.vsh.yaml
@@ -13,4 +13,4 @@ runners:
   - type: executable
   - type: nextflow
     directives:
-      label: [midtime, lowmem, lowcpu]
+      label: [midtime, midmem, lowcpu]
diff --git a/src/methods/knnr_py/config.vsh.yaml b/src/methods/knnr_py/config.vsh.yaml
@@ -32,4 +32,4 @@ runners:
   - type: executable
   - type: nextflow
     directives:
-      label: [hightime, lowmem, lowcpu]
+      label: [hightime, midmem, midcpu]
diff --git a/src/methods/knnr_r/config.vsh.yaml b/src/methods/knnr_r/config.vsh.yaml
@@ -35,4 +35,4 @@ runners:
   - type: executable
   - type: nextflow
     directives:
-      label: [hightime, lowmem, lowcpu]
+      label: [hightime, midmem, midcpu]
diff --git a/src/workflows/process_datasets/config.vsh.yaml b/src/workflows/process_datasets/config.vsh.yaml
@@ -7,10 +7,10 @@ resources:
     entrypoint: run_wf
   - path: /common/nextflow_helpers/helper.nf
 dependencies:
-  - name: schema/verify_data_structure
-    repository: core
-  - name: h5ad/extract_uns_metadata
-    repository: core
+  - name: validation/check_dataset_with_schema
+    repository: openproblems
+  - name: utils/extract_uns_metadata
+    repository: openproblems
   - name: data_processors/process_dataset
 runners:
   - type: nextflow
diff --git a/src/workflows/process_datasets/main.nf b/src/workflows/process_datasets/main.nf
@@ -16,8 +16,8 @@ workflow run_wf {
   output_ch = input_ch
 
     // Check if the input datasets match the desired format --------------------------------
-    | verify_data_structure.run(
-      key: "verify_data_structure_mod1",
+    | check_dataset_with_schema.run(
+      key: "check_dataset_with_schema_mod1",
       fromState: { id, state ->
         def schema = findArgumentSchema(meta.config, "input_mod1")
         def schemaYaml = tempFile("schema.yaml")
@@ -36,8 +36,8 @@ workflow run_wf {
       }
     )
 
-    | verify_data_structure.run(
-      key: "verify_data_structure_mod2",
+    | check_dataset_with_schema.run(
+      key: "check_dataset_with_schema_mod2",
       fromState: { id, state ->
         def schema = findArgumentSchema(meta.config, "input_mod2")
         def schemaYaml = tempFile("schema.yaml")

diff --git a/src/workflows/run_benchmark/config.vsh.yaml b/src/workflows/run_benchmark/config.vsh.yaml
@@ -60,8 +60,8 @@ resources:
   - type: file
     path: /_viash.yaml
 dependencies:
-  - name: h5ad/extract_uns_metadata
-    repository: core
+  - name: utils/extract_uns_metadata
+    repository: openproblems
   - name: control_methods/mean_per_gene
   - name: control_methods/random_predict
   - name: control_methods/zeros
+1 −1		component_tests/run_and_check_output.py
+17 −0		nextflow_helpers/helper.nf
+4 −0		nextflow_helpers/labels_tw.config
+1 −0		schemas/api_file_format.yaml
+136 −28		schemas/schema_openproblems_definitions.yaml
+0 −2		schemas/task_control_method.yaml
+0 −2		schemas/task_method.yaml
+1 −9		schemas/task_metric.yaml
+13 −1		scripts/create_component
+13 −1		scripts/create_task_readme
+40 −0		scripts/fetch_task_run
+13 −1		scripts/sync_resources
+13 −1		scripts/upgrade_config