update to documentation (#349)

* update components

* update create pull request

* update submodule

* update test resources path

* [WIP] update common commands

* Make website text more readable

* update common commands

* change info circle to question-circle

* upodate link to github issues tracker

* Make single line code bg more visible

* update add a method page

* Fix dependencies inlcude

_extensions/debruine/glossary/glossary.lua

@@ -103,7 +103,7 @@ return {
       end
 
       local url = "/documentation/reference/glossary/index.qmd#" .. term
-      local infoIcon = pandoc.RawInline('html', '<i class="bi bi-info-circle info-icon"></i>')
+      local infoIcon = pandoc.RawInline('html', '<i class="bi bi-question-circle info-icon"></i>')
       local combinedLabel = pandoc.Span({pandoc.Str(label), infoIcon})
       local link = pandoc.Link(combinedLabel, url, "", pandoc.Attr("class", {"quarto-xref"}))
 

documentation/create_component/_include/_add_dependencies.qmd

@@ -42,7 +42,8 @@ platforms:
           - pyyaml
           - requests
           - jsonschema
-        github: openproblems-bio/core#subdirectory=packages/python/openproblems
+        github: 
+          - "openproblems-bio/core#subdirectory=packages/python/openproblems"
 ```
 
 ## R
@@ -69,7 +70,8 @@ platforms:
           - pyyaml
           - requests
           - jsonschema
-          github: openproblems-bio/core#subdirectory=packages/python/openproblems
+        github: 
+          - "openproblems-bio/core#subdirectory=packages/python/openproblems"
       - type: r
         packages:
           - anndata

documentation/create_component/add_a_control.qmd

@@ -244,9 +244,9 @@ You can also run your component on local files using the `viash run` command. Fo
 
 ```bash
 viash run src/tasks/label_projection/control_methods/my_python_method/config.vsh.yaml -- \
-  --input_train resources_test/task_template/pancreas/train.h5ad \
-  --input_test resources_test/task_template/pancreas/test.h5ad \
-  --input_solution resources_test/task_template/pancreas/solution.h5ad \
+  --input_train resources_test/task_template/cxg_mouse_pancreas_atlas/train.h5ad \
+  --input_test resources_test/task_template/cxg_mouse_pancreas_atlas/test.h5ad \
+  --input_solution resources_test/task_template/cxg_mouse_pancreas_atlas/solution.h5ad \
   --output output.h5ad
 ```
 

documentation/create_component/add_a_method.qmd

@@ -41,7 +41,7 @@ common/scripts/create_component \
 
 This creates a new folder at `src/methods/my_python_method` containing a Viash config and a script.
 
-    tree src/tasks/label_projection/methods/my_python_method
+    tree src/methods/my_python_method
         ├── script.py         Script for running the method.
         ├── config.vsh.yaml   Config file for method.
         └── ...               Optional additional resources.
@@ -62,7 +62,7 @@ common/scripts/create_component \
 
 This creates a new folder at `src/methods/my_r_method` containing a Viash config and a script.
 
-    tree src/tasks/label_projection/methods/my_r_method
+    tree src/methods/my_r_method
         ├── script.R          Script for running the method.
         ├── config.vsh.yaml   Config file for method.
         └── ...               Optional additional resources.
@@ -121,13 +121,13 @@ Please edit `info` section in the config file to fill in the necessary metadata.
 
 {{< include _include/_render_schema_required_fields.qmd >}}
 
-<!-- ```{r echo=FALSE, warning=FALSE, error=FALSE}
-info <- read_and_merge_yaml("src/common/schemas/task_method.yaml")
+```{r echo=FALSE, warning=FALSE, error=FALSE}
+info <- openproblems::read_nested_yaml("../_core/schemas/task_method.yaml")
 
 info_str <- paste(render_schema_required_fields(info, required = TRUE), collapse = "\n\n")
 
 knitr::asis_output(info_str)
-``` -->
+```
 
 
 ## Step 3: Add dependencies
@@ -243,8 +243,8 @@ You can also run your component on local files using the `viash run` command. Fo
 
 ```bash
 viash run src/methods/my_python_method/config.vsh.yaml -- \
-  --input_train resources_test/task_template/pancreas/train.h5ad \
-  --input_test resources_test/task_template/pancreas/test.h5ad \
+  --input_train resources_test/task_template/cxg_mouse_pancreas_atlas/train.h5ad \
+  --input_test resources_test/task_template/cxg_mouse_pancreas_atlas/test.h5ad \
   --output output.h5ad
 ```
 

documentation/create_component/add_a_metric.qmd

@@ -238,8 +238,8 @@ You can also run your component on local files using the `viash run` command. Fo
 
 ```bash
 viash run src/metrics/my_python_metric/config.vsh.yaml -- \
-  --input_prediction resources_test/task_template/pancreas/prediction.h5ad \
-  --input_solution resources_test/task_template/pancreas/solution.h5ad \
+  --input_prediction resources_test/task_template/cxg_mouse_pancreas_atlas/prediction.h5ad \
+  --input_solution resources_test/task_template/cxg_mouse_pancreas_atlas/solution.h5ad \
   --output output.h5ad
 ```
 

documentation/create_component/create_pull_request.qmd

@@ -14,7 +14,7 @@ Before you contribute your changes need to merge the upstream main branch into y
 To do this, enter the following commands adapted from [Syncing a Fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork#syncing-a-fork-branch-from-the-command-line) in your terminal or command prompt:
 
 ```bash
-git remote add upstream https://github.com/openproblems-bio/openproblems.git
+git remote add upstream <original repository URL>
 git fetch upstream
 git checkout main
 git merge upstream/main
@@ -28,14 +28,14 @@ Add an entry to the CHANGELOG.md file describing the proposed changes.
 
 The following steps were adapted from [Creating a pull request from a fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork)
 
-1. Go to <https://github.com/openproblems-bio/openproblems/pulls>.
+1. Go to the original repo where you want to request a change https://github.com/openproblems-bio.
 2. Click on the `New pull request` button.
 3. On the compare page click on the link `compare across forks` below the title. ![](../images/compare_across_forks.png)
 4. On the right side in the `head` section select your fork repo and the correct branch you want to merge.
 5. Click on `Create pull request`.
 6. Construct your PR by giving it a title and description.
 7. Make sure you select the box below the description `Allow edits from maintainers`.
-8. If the PR is ready for review click the button `Create Pull Request`. Otherwise you can click the arrow next to the button and select `Create Draft Pull Request` and click the button when it changes.
+8. If the PR is ready for review click the button `Create Pull Request`. (_If available_) Otherwise you can click the arrow next to the button and select `Create Draft Pull Request` and click the button when it changes.
 
 ## Next steps
 

documentation/create_component/first_contributions.qmd

@@ -8,7 +8,7 @@ page-navigation: true
 
 To contribute new functionality to OpenProblems, please follow the steps below:
 
-1. Search through the [GitHub Issues](https://github.com/openproblems-bio/openproblems/issues)  tracker to ensure that no one else is already working on the feature you would like to add. 
+1. Search through the [GitHub Issues](https://github.com/openproblems-bio/) tracker of the task to ensure that no one else is already working on the feature you would like to add. 
   - If someone is already working on it, post in that issue to express your interest or reach out to the contributors working on the issue directly.
   - If there is no existing issue tracking the feature, create a new one.
 

documentation/create_component/getting_started.qmd

@@ -76,17 +76,17 @@ Use the `viash run` command to run a Viash component. Everything after the `--`
 
 ```bash
 viash run src/methods/logistic_regression/config.vsh.yaml -- \
-  --input_train resources_test/task_template/pancreas/train.h5ad \
-  --input_test resources_test/task_template/pancreas/test.h5ad \
+  --input_train resources_test/task_template/cxg_mouse_pancreas_atlas/train.h5ad \
+  --input_test resources_test/task_template/cxg_mouse_pancreas_atlas/test.h5ad \
   --output output.h5ad
 ```
 <details>
   <summary>Output</summary>
 ```{bash}
 #| echo: false
 viash run src/methods/logistic_regression/config.vsh.yaml -- \
-  --input_train resources_test/task_template/pancreas/train.h5ad \
-  --input_test resources_test/task_template/pancreas/test.h5ad \
+  --input_train resources_test/task_template/cxg_mouse_pancreas_atlas/train.h5ad \
+  --input_test resources_test/task_template/cxg_mouse_pancreas_atlas/test.h5ad \
   --output output.h5ad
 ```
 </details>

documentation/create_task/dataset_processor.qmd

@@ -125,7 +125,7 @@ import anndata as ad
 
 ## VIASH START
 par = {
-    "input": "resources_test/common/pancreas/dataset.h5ad",
+    "input": "resources_test/common/cxg_mouse_pancreas_atlas/dataset.h5ad",
     "output_dataset": "dataset.h5ad",
     "output_solution": "solution.h5ad",
 }
@@ -159,7 +159,7 @@ library(anndata)
 
 ## VIASH START
 par <- list(
-  input = "resources_test/common/pancreas/dataset.h5ad",
+  input = "resources_test/common/cxg_mouse_pancreas_atlas/dataset.h5ad",
   output_dataset = "dataset.h5ad",
   output_solution = "solution.h5ad",
 )
@@ -225,23 +225,23 @@ nextflow run . \
   -profile docker \
   --publish_dir "$DATASET_DIR" \
   --id "pancreas" \
-  --input "$RAW_DATA/pancreas/dataset.h5ad" \
+  --input "$RAW_DATA/cxg_mouse_pancreas_atlas/dataset.h5ad" \
   --output_train '$id/train.h5ad' \
   --output_test '$id/test.h5ad' \
   --output_solution '$id/solution.h5ad' \
   --output_state '$id/state.yaml'
 
 # run one method
 viash run src/methods/knn/config.vsh.yaml -- \
-    --input_train $DATASET_DIR/pancreas/train.h5ad \
-    --input_test $DATASET_DIR/pancreas/test.h5ad \
-    --output $DATASET_DIR/pancreas/prediction.h5ad
+    --input_train $DATASET_DIR/cxg_mouse_pancreas_atlas/train.h5ad \
+    --input_test $DATASET_DIR/cxg_mouse_pancreas_atlas/test.h5ad \
+    --output $DATASET_DIR/cxg_mouse_pancreas_atlas/prediction.h5ad
 
 # run one metric
 viash run src/metrics/accuracy/config.vsh.yaml -- \
-    --input_prediction $DATASET_DIR/pancreas/prediction.h5ad \
-    --input_solution $DATASET_DIR/pancreas/solution.h5ad \
-    --output $DATASET_DIR/pancreas/score.h5ad
+    --input_prediction $DATASET_DIR/cxg_mouse_pancreas_atlas/prediction.h5ad \
+    --input_solution $DATASET_DIR/cxg_mouse_pancreas_atlas/solution.h5ad \
+    --output $DATASET_DIR/cxg_mouse_pancreas_atlas/score.h5ad
 
 # only run this if you have access to the openproblems-data bucket #<2>
 # aws s3 sync --profile op \

documentation/create_task/design_api.qmd

@@ -113,7 +113,7 @@ Let's start by creating one for the solution object:
 
 ```{.yaml filename="src/api/file_solution.yaml"}
 type: file # <1>
-example: "resources_test/<task_id>/pancreas/solution.h5ad"  # <2>
+example: "resources_test/<task_id>/cxg_mouse_pancreas_atlas/solution.h5ad"  # <2>
 label: Solution  # <3>
 summary: "FILL IN: what this file represents" # <4>
 info:
@@ -181,7 +181,7 @@ Below is the slot information of the solution AnnData object:
 ```{.yaml filename="src/api/file_solution.yaml"}
 type: file
 description: "FILL IN: what this file represents"
-example: "resources_test/<task_id>/pancreas/solution.h5ad"
+example: "resources_test/<task_id>/cxg_mouse_pancreas_atlas/solution.h5ad"
 info:
   label: Solution
   format: # <1>
@@ -248,5 +248,5 @@ Each required or optional slot in the file format should have the following fiel
 Go through each file format specification file and add the expected slots accordingly.
 
 :::{.callout-tip}
-Look at the [Common dataset](/documentation/reference/openproblems/src-datasets.qmd#file-format-common-dataset) reference docs to see which slots the common datasets have. The AnnData file at `resources_test/common/pancreas/dataset.h5ad` is also an example of a Common dataset, though note that this object contains _more_ slots than what is defined by the spec.
+Look at the [Common dataset](/documentation/reference/openproblems/src-datasets.qmd#file-format-common-dataset) reference docs to see which slots the common datasets have. The AnnData file at `resources_test/common/cxg_mouse_pancreas_atlas/dataset.h5ad` is also an example of a Common dataset, though note that this object contains _more_ slots than what is defined by the spec.
 :::

documentation/create_task/getting_started.qmd

@@ -105,17 +105,17 @@ Use the `viash run` command to run a Viash component. Everything after the `--`
 
 ```bash
 viash run src/methods/logistic_regression/config.vsh.yaml -- \
-  --input_train resources_test/task_template/pancreas/train.h5ad \
-  --input_test resources_test/task_template/pancreas/test.h5ad \
+  --input_train resources_test/task_template/cxg_mouse_pancreas_atlas/train.h5ad \
+  --input_test resources_test/task_template/cxg_mouse_pancreas_atlas/test.h5ad \
   --output output.h5ad
 ```
 <details>
   <summary>Output</summary>
 ```{bash}
 #| echo: false
 viash run src/methods/logistic_regression/config.vsh.yaml -- \
-  --input_train resources_test/task_template/pancreas/train.h5ad \
-  --input_test resources_test/task_template/pancreas/test.h5ad \
+  --input_train resources_test/task_template/cxg_mouse_pancreas_atlas/train.h5ad \
+  --input_test resources_test/task_template/cxg_mouse_pancreas_atlas/test.h5ad \
   --output output.h5ad
 ```
 </details>

documentation/fundamentals/common_commands.qmd

@@ -8,6 +8,9 @@ engine: knitr
 Below is a list of common Viash commands to develop and test components. 
 Check out this [Cheat sheet](/documentation/more_information/cheat_sheets.qmd) to get a printable version of the same information. Please visit the [Viash guide](https://viash.io/guide) or [reference documentation](https://viash.io/reference/) for more in-depth information.
 
+:::{.callout-important}
+Most of the common commands are added to a script in the `scripts` directory. You can run these scripts from the root directory of the project. Some scripts might need some additional arguments, which you can find in the script itself.
+:::
 
 ## Download test data
 
@@ -25,8 +28,8 @@ scripts/create_datasets/test_resources.sh
 Build all components in a namespace.
 
 :::{.small}
-```bash
-scripts/project/build_all_components.sh
+```{.bash filename="scripts/project/build_all_components.sh"}
+viash ns build --parallel
 ```
 :::
 
@@ -35,8 +38,8 @@ scripts/project/build_all_components.sh
 Build all docker containers.
 
 :::{.small}
-```bash
-scripts/project/build_all_docker_containers.sh
+```{.bash filename="scripts/project/build_all_docker_containers.sh"}
+viash ns build --parallel --setup cachedbuild
 ```
 :::
 
@@ -53,8 +56,8 @@ viash test path/to/config.vsh.yaml
 Run all tests in a task:
 
 :::{.small}
-```bash
-scripts/project/test_all_components.sh
+```{.bash filename="scripts/project/test_all_components.sh"}
+viash ns test --parallel
 ```
 :::
 
@@ -89,7 +92,6 @@ viash run path/to/config.vsh.yaml -- --input dataset.h5ad --output output.h5ad
 
 ## View Dockerfile
 
-
 Use this command to view the component's Dockerfile.
 
 :::{.small}

documentation/fundamentals/components.qmd

@@ -60,35 +60,37 @@ Next, we'll write a Viash config file for this component.
 
 :::{.small}
 ```{.yaml filename="config.vsh.yaml"}
-functionality:
-  name: mycomponent
-  description: |
-    A multiline description.
-  arguments:
-    - name: "--input"
-      type: file
-      description: Input h5ad
-      example: input.h5ad
-      required: true
-    - name: "--output"
-      type: file
-      direction: output
-      description: Output g5ad
-      example: output.h5ad
-      required: true
-  resources:
-    - type: python_script
-      path: script.py
-    tests:
-    - type: python_script
-      path: test.py
-platforms:
+name: mycomponent
+label: My Component
+summary: A short summary of the component.
+description: |
+  A multiline description.
+arguments:
+  - name: "--input"
+    type: file
+    description: Input h5ad
+    example: input.h5ad
+    required: true
+  - name: "--output"
+    type: file
+    direction: output
+    description: Output g5ad
+    example: output.h5ad
+    required: true
+resources:
+  - type: python_script
+    path: script.py
+  tests:
+  - type: python_script
+    path: test.py
+engines:
   - type: docker
       image: "python:3.10"
       setup:
         - type: python
           pypi: anndata
-  - type: native
+runners:
+  - type: executable
   - type: nextflow
 ```
 :::
@@ -99,4 +101,4 @@ After adding more arguments to the config, you can update your script using the
 
 ### Component format specifications
 
-OpenProblems tasks contain specifications for the common config fields between the same component types. These specifications are defined in the `src/tasks/*/comp_*.yaml` files. For more information on how these specifications are formatted, see ["Design the API"](/documentation/create_task/design_api.qmd#step-3).
+OpenProblems tasks contain specifications for the common config fields between the same component types. These specifications are defined in the `src/api/comp_*.yaml` files. For more information on how these specifications are formatted, see ["Design the API"](/documentation/create_task/design_api.qmd#step-3).

style/styles.css

@@ -4,9 +4,14 @@
   --bs-code-color: #ff414b;
 }
 
+p code:not(.sourceCode), li code:not(.sourceCode), td code:not(.sourceCode) {
+  background-color: #eef1f3;
+  padding: .2em;
+}
+
 .quarto-dark {
   --bs-code-color: #ffb0b4;
-  --bs-code-bgcolor: #272729;
+  --bs-code-bgcolor: #3b3b3b;
 
   & code,
   & p code:not(.sourceCode),

style/theme-dark.scss

@@ -1,5 +1,4 @@
 /*-- scss:defaults --*/
-$font-family-sans-serif: "Open Sans", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
 $body-bg: #1e1e20;
 
 :root {