From 176065b371e757da9cfdba76a98226a6c71b0bda Mon Sep 17 00:00:00 2001 From: Jalil Nourisa Date: Thu, 19 Sep 2024 10:17:08 +0200 Subject: [PATCH] normalize as non default for control models --- runs copy.ipynb | 2970 ----------------- runs.ipynb | 2158 ++---------- ...k_single_omics.sh => run_benchmark_all.sh} | 40 +- scripts/sbatch/batch_grnboost2.sh | 11 - scripts/sbatch/batch_scenic.sh | 7 +- src/api/comp_method.yaml | 4 + src/control_methods/pearson/script.py | 10 +- src/control_methods/pearson_causal/script.py | 9 +- .../positive_control/script.py | 7 +- src/methods/multi_omics/celloracle/script.py | 13 +- .../celloracle_ns/config.novsh.yaml | 29 - src/methods/multi_omics/celloracle_ns/main.nf | 21 - src/methods/multi_omics/celloracle_ns/run.sh | 61 - .../multi_omics/granie_ns/config.novsh.yaml | 27 - src/methods/multi_omics/granie_ns/main.nf | 22 - src/methods/multi_omics/granie_ns/run.sh | 35 - .../scenicplus_ns/config.novsh.yaml | 39 - src/methods/multi_omics/scenicplus_ns/main.nf | 23 - src/methods/multi_omics/scenicplus_ns/run.sh | 44 - .../multi_omics/scglue_ns/config.novsh.yaml | 20 - src/methods/multi_omics/scglue_ns/main.nf | 22 - src/methods/multi_omics/scglue_ns/run.sh | 32 - .../{config.vsh.yaml => config.novsh.yaml} | 0 src/methods/single_omics/genie3/script.py | 18 +- src/methods/single_omics/ppcor/script.R | 12 +- .../single_omics/scgpt/config.vsh.yaml | 2 +- .../config.vsh.yaml | 24 +- .../main.nf | 3 +- test.sh | 20 - 29 files changed, 308 insertions(+), 5375 deletions(-) delete mode 100644 runs copy.ipynb rename scripts/{run_benchmark_single_omics.sh => run_benchmark_all.sh} (58%) delete mode 100644 scripts/sbatch/batch_grnboost2.sh delete mode 100644 src/methods/multi_omics/celloracle_ns/config.novsh.yaml delete mode 100644 src/methods/multi_omics/celloracle_ns/main.nf delete mode 100644 src/methods/multi_omics/celloracle_ns/run.sh delete mode 100644 src/methods/multi_omics/granie_ns/config.novsh.yaml delete mode 100644 src/methods/multi_omics/granie_ns/main.nf delete mode 100644 src/methods/multi_omics/granie_ns/run.sh delete mode 100644 src/methods/multi_omics/scenicplus_ns/config.novsh.yaml delete mode 100644 src/methods/multi_omics/scenicplus_ns/main.nf delete mode 100644 src/methods/multi_omics/scenicplus_ns/run.sh delete mode 100644 src/methods/multi_omics/scglue_ns/config.novsh.yaml delete mode 100644 src/methods/multi_omics/scglue_ns/main.nf delete mode 100644 src/methods/multi_omics/scglue_ns/run.sh rename src/methods/single_omics/ennet/{config.vsh.yaml => config.novsh.yaml} (100%) rename src/workflows/{run_benchmark_all => run_benchmark}/config.vsh.yaml (92%) rename src/workflows/{run_benchmark_all => run_benchmark}/main.nf (99%) delete mode 100644 test.sh diff --git a/runs copy.ipynb b/runs copy.ipynb deleted file mode 100644 index d727dfcbe..000000000 --- a/runs copy.ipynb +++ /dev/null @@ -1,2970 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Commands" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "upload: resources/grn-benchmark/multiomics_rna_qc.h5ad to s3://openproblems-data/resources/grn/grn-benchmark/multiomics_rna_qc.h5ad\n", - "upload: resources/grn-benchmark/multiomics_rna_0.h5ad to s3://openproblems-data/resources/grn/grn-benchmark/multiomics_rna_0.h5ad\n", - "upload: resources/grn-benchmark/multiomics_atac_0.h5ad to s3://openproblems-data/resources/grn/grn-benchmark/multiomics_atac_0.h5ad\n" - ] - } - ], - "source": [ - "!aws s3 sync resources/grn-benchmark s3://openproblems-data/resources/grn/grn-benchmark " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import yaml\n", - "import pandas as pd \n", - "import matplotlib.pyplot as plt\n", - "import anndata as ad\n", - "controls = ['negative_control','positive_control']\n", - "grn_models = ['collectri','granie', 'figr', 'celloracle', 'scglue', 'scenicplus']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Test" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Unnamed: 0sourcetargetweight
00BCL6FBP10.544815
11BCL6FAM157C1.530639
22BCL6OLIG10.508618
33BCL6BIRC50.915793
44BCL6TMED50.523122
...............
56865686ZNF816FLII0.160327
56875687ZNF816KNOP10.167249
56885688ZNF816SCO10.184201
56895689ZNF816SH2D4B0.072754
56905690ZNF816SLC13A30.313595
\n", - "

5691 rows × 4 columns

\n", - "
" - ], - "text/plain": [ - " Unnamed: 0 source target weight\n", - "0 0 BCL6 FBP1 0.544815\n", - "1 1 BCL6 FAM157C 1.530639\n", - "2 2 BCL6 OLIG1 0.508618\n", - "3 3 BCL6 BIRC5 0.915793\n", - "4 4 BCL6 TMED5 0.523122\n", - "... ... ... ... ...\n", - "5686 5686 ZNF816 FLII 0.160327\n", - "5687 5687 ZNF816 KNOP1 0.167249\n", - "5688 5688 ZNF816 SCO1 0.184201\n", - "5689 5689 ZNF816 SH2D4B 0.072754\n", - "5690 5690 ZNF816 SLC13A3 0.313595\n", - "\n", - "[5691 rows x 4 columns]" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df = pd.read_csv('output/scenic/scenic.csv')\n", - "df # default" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# To be classified " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import anndata as ad \n", - "import numpy as np \n", - "adata_rna = ad.read_h5ad('resources/grn-benchmark/multiomics_rna.h5ad')\n", - "X = adata_rna.X.todense()\n", - "\n", - "if True:\n", - " print('qc')\n", - " print('Data shape before QC: ', adata_rna.shape)\n", - " # Remove genes with >=90% of zeros\n", - " mask_gene = (np.mean(X == 0, axis=0) >= 0.9)\n", - " adata_rna = adata_rna[:, ~mask_gene]\n", - " # Remove samples with >=90% of zeros\n", - " mask_cells = (np.mean(X == 0, axis=1) >= 0.9)\n", - " adata_rna = adata_rna[~mask_cells, :]\n", - " print('Data shape after QC: ', adata_rna.shape)\n", - "adata_rna.write('resources/grn-benchmark/multiomics_rna_qc.h5ad')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## GRN inference: multiomics" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### scGLUE" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "download: s3://openproblems-data/resources/grn/results/scglue/state.yaml to resources/results/scglue/state.yaml\n", - "download: s3://openproblems-data/resources/grn/results/scglue/trace.txt to resources/results/scglue/trace.txt\n", - "download: s3://openproblems-data/resources/grn/results/scglue/output/grn.csv to resources/results/scglue/output/grn.csv\n" - ] - } - ], - "source": [ - "!aws s3 sync s3://openproblems-data/resources/grn/results/scglue ./resources/results/scglue" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[37mReading input files\u001b[0m\n", - "\u001b[37mCalculate basic stats\u001b[0m\n", - "\u001b[37mOutputting stats to : /viash_automount/mnt/c/Users/nourisa/Documents/testProjs/ongoing/task_grn_benchmark/output/stats.json\u001b[0m\n", - "\u001b[37mTopological analysis\u001b[0m\n", - "\u001b[37mPlotting tf-gene in degree, dir: /viash_automount/mnt/c/Users/nourisa/Documents/testProjs/ongoing/task_grn_benchmark/output/tf_gene_indegree.png\u001b[0m\n", - "\u001b[37mPlotting tf-gene out degree, dir: /viash_automount/mnt/c/Users/nourisa/Documents/testProjs/ongoing/task_grn_benchmark/output/tf_gene_outdegree.png\u001b[0m\n" - ] - } - ], - "source": [ - "!viash run src/exp_analysis/config.vsh.yaml -- --tf_gene_net resources/results/scglue/output/grn.csv" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## GRN evaluation for multiomics methods" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "temporaryFolder: /tmp/viash_hub_repo10489538114835231235 uri: https://github.com/openproblems-bio/openproblems-v2.git\n", - "Cloning into '.'...\n", - "checkout out: List(git, checkout, origin/main_build, --, .) 0 \n", - "\u001b[37mExporting run_grn_evaluation (workflows) =nextflow=> /mnt/c/Users/nourisa/Documents/testProjs/ongoing/task_grn_inference/target/nextflow/workflows/run_grn_evaluation\u001b[0m\n", - "\u001b[33mNot all configs built successfully\u001b[0m\n", - "\u001b[33m 39 configs were disabled\u001b[0m\n", - "\u001b[32m 1/1 configs built successfully\u001b[0m\n" - ] - } - ], - "source": [ - "!viash ns build --setup cb -q run_grn_evaluation" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "!bash scripts/run_grn_evaluation.sh \"ridge\"\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "download: s3://openproblems-data/resources/grn/results/grn_evaluation_ridge/scores.yaml to resources/results/grn_evaluation_ridge/scores.yaml\n", - "download: s3://openproblems-data/resources/grn/results/grn_evaluation_ridge/trace.txt to resources/results/grn_evaluation_ridge/trace.txt\n", - "download: s3://openproblems-data/resources/grn/results/grn_evaluation_ridge/metric_configs.yaml to resources/results/grn_evaluation_ridge/metric_configs.yaml\n" - ] - } - ], - "source": [ - "!aws s3 sync s3://openproblems-data/resources/grn/results/grn_evaluation_ridge ./resources/results/grn_evaluation_ridge" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Robustness analysis" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "temporaryFolder: /tmp/viash_hub_repo10268443552668753296 uri: https://github.com/openproblems-bio/openproblems-v2.git\n", - "Cloning into '.'...\n", - "checkout out: List(git, checkout, origin/main_build, --, .) 0 \n", - "\u001b[37mExporting run_robustness_analysis (workflows) =nextflow=> /mnt/c/Users/nourisa/Documents/testProjs/ongoing/task_grn_inference/target/nextflow/workflows/run_robustness_analysis\u001b[0m\n", - "\u001b[33mNot all configs built successfully\u001b[0m\n", - "\u001b[33m 39 configs were disabled\u001b[0m\n", - "\u001b[32m 1/1 configs built successfully\u001b[0m\n" - ] - } - ], - "source": [ - "# !viash ns build --setup cb --parallel\n", - "!viash ns build --setup cb -q run_robustness_analysis\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Shuffle net" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "!bash scripts/run_robust_analys.sh \"net\"" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_net/state.yaml to resources/results/robust_analy_reg2_net/state.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_net/scores.yaml to resources/results/robust_analy_reg2_net/scores.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_net/metric_configs.yaml to resources/results/robust_analy_reg2_net/metric_configs.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_net/trace.txt to resources/results/robust_analy_reg2_net/trace.txt\n" - ] - } - ], - "source": [ - "!aws s3 sync s3://openproblems-data/resources/grn/results/robust_analy_reg2_net ./resources/results/robust_analy_reg2_net" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Add noise to weights" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "weight\n", - "\u001b[33mNextflow 24.04.4 is available - Please consider updating your version to it\u001b[m\n", - "N E X T F L O W ~ version 23.10.4\n", - "Launching `target/nextflow/workflows/run_robustness_analysis/main.nf` [dreamy_davinci] DSL2 - revision: 6bd5cda1f4\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "\u001b[3A\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[5A\n", - "executor > local (1)\u001b[K\n", - "[5f/aa4675] process > run_robustness_analysis:run... [ 0%] 0 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (6)\u001b[K\n", - "[f3/689356] process > run_robustness_analysis:run... [ 0%] 0 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (7)\u001b[K\n", - "[5f/aa4675] process > run_robustness_analysis:run... [ 3%] 1 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 1\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (9)\u001b[K\n", - "[bc/144211] process > run_robustness_analysis:run... [ 10%] 3 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 3\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (10)\u001b[K\n", - "[b2/e135c4] process > run_robustness_analysis:run... [ 13%] 4 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 4\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (10)\u001b[K\n", - "[b2/e135c4] process > run_robustness_analysis:run... [ 13%] 4 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 4\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (11)\u001b[K\n", - "[41/99eca2] process > run_robustness_analysis:run... [ 16%] 5 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 5\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (13)\u001b[K\n", - "[c2/0fe413] process > run_robustness_analysis:run... [ 23%] 7 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 6\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (13)\u001b[K\n", - "[c2/0fe413] process > run_robustness_analysis:run... [ 23%] 7 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 7\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (14)\u001b[K\n", - "[e1/58ddaa] process > run_robustness_analysis:run... [ 26%] 8 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 8\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (15)\u001b[K\n", - "[7b/b975d3] process > run_robustness_analysis:run... [ 30%] 9 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 8\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (15)\u001b[K\n", - "[7b/b975d3] process > run_robustness_analysis:run... [ 30%] 9 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 9\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (16)\u001b[K\n", - "[64/f3e1eb] process > run_robustness_analysis:run... [ 33%] 10 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 10\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (18)\u001b[K\n", - "[a9/8325fb] process > run_robustness_analysis:run... [ 40%] 12 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 11\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (19)\u001b[K\n", - "[d4/d30a10] process > run_robustness_analysis:run... [ 43%] 13 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 12\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (19)\u001b[K\n", - "[d4/d30a10] process > run_robustness_analysis:run... [ 43%] 13 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 13\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (20)\u001b[K\n", - "[48/b949bd] process > run_robustness_analysis:run... [ 46%] 14 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 14\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (21)\u001b[K\n", - "[40/2c9923] process > run_robustness_analysis:run... [ 50%] 15 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 15\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (22)\u001b[K\n", - "[45/952ee5] process > run_robustness_analysis:run... [ 53%] 16 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 16\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (23)\u001b[K\n", - "[57/dece12] process > run_robustness_analysis:run... [ 56%] 17 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 17\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (24)\u001b[K\n", - "[8b/7868a3] process > run_robustness_analysis:run... [ 60%] 18 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 18\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (25)\u001b[K\n", - "[a9/8325fb] process > run_robustness_analysis:run... [ 63%] 19 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 19\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (25)\u001b[K\n", - "[a9/8325fb] process > run_robustness_analysis:run... [ 63%] 19 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 19\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (26)\u001b[K\n", - "[3e/bd85b3] process > run_robustness_analysis:run... [ 66%] 20 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 20\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (27)\u001b[K\n", - "[58/cbdb13] process > run_robustness_analysis:run... [ 70%] 21 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 21\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (28)\u001b[K\n", - "[55/ec24c0] process > run_robustness_analysis:run... [ 73%] 22 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 22\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (29)\u001b[K\n", - "[cd/555e42] process > run_robustness_analysis:run... [ 76%] 23 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 23\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (30)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [ 80%] 24 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 24\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (31)\u001b[K\n", - "[09/ecd838] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[4f/726118] process > run_robustness_analysis:run... [ 0%] 0 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (32)\u001b[K\n", - "[3e/bd85b3] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[c7/828453] process > run_robustness_analysis:run... [ 0%] 0 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (33)\u001b[K\n", - "[58/cbdb13] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[7f/f16766] process > run_robustness_analysis:run... [ 0%] 0 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (33)\u001b[K\n", - "[58/cbdb13] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[7f/f16766] process > run_robustness_analysis:run... [ 0%] 0 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (34)\u001b[K\n", - "[55/ec24c0] process > run_robustness_analysis:run... [ 93%] 28 of 30\u001b[K\n", - "[80/dcac1b] process > run_robustness_analysis:run... [ 0%] 0 of 28\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (35)\u001b[K\n", - "[cd/555e42] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[a5/a9db21] process > run_robustness_analysis:run... [ 0%] 0 of 28\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (35)\u001b[K\n", - "[cd/555e42] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[a5/a9db21] process > run_robustness_analysis:run... [ 0%] 0 of 29\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (36)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e9/90d374] process > run_robustness_analysis:run... [ 0%] 0 of 29\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (36)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e9/90d374] process > run_robustness_analysis:run... [ 0%] 0 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (37)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7f/f16766] process > run_robustness_analysis:run... [ 3%] 1 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 1\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (37)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/726118] process > run_robustness_analysis:run... [ 6%] 2 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 1\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (38)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7e/fced9a] process > run_robustness_analysis:run... [ 6%] 2 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 2\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (39)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[a5/a9db21] process > run_robustness_analysis:run... [ 10%] 3 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 3\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (40)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7f/6484f9] process > run_robustness_analysis:run... [ 13%] 4 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 4\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (40)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7f/6484f9] process > run_robustness_analysis:run... [ 13%] 4 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 4\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (41)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[02/0a1ff4] process > run_robustness_analysis:run... [ 16%] 5 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 5\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (42)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[80/dcac1b] process > run_robustness_analysis:run... [ 20%] 6 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 6\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (43)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7f/355350] process > run_robustness_analysis:run... [ 23%] 7 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 6\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (43)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7f/355350] process > run_robustness_analysis:run... [ 23%] 7 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 7\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (44)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[02/0a1ff4] process > run_robustness_analysis:run... [ 26%] 8 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 8\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (44)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[02/0a1ff4] process > run_robustness_analysis:run... [ 26%] 8 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 8\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (45)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[c0/7f3f8c] process > run_robustness_analysis:run... [ 30%] 9 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 9\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (46)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7e/fced9a] process > run_robustness_analysis:run... [ 33%] 10 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 10\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (46)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7e/fced9a] process > run_robustness_analysis:run... [ 33%] 10 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 10\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (47)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[c0/7f3f8c] process > run_robustness_analysis:run... [ 36%] 11 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 11\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (48)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e2/5dd3e9] process > run_robustness_analysis:run... [ 40%] 12 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 11\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (48)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e2/5dd3e9] process > run_robustness_analysis:run... [ 40%] 12 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 12\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (49)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[06/b5bead] process > run_robustness_analysis:run... [ 43%] 13 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 12\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (49)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[06/b5bead] process > run_robustness_analysis:run... [ 43%] 13 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 13\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (50)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[29/84b62b] process > run_robustness_analysis:run... [ 46%] 14 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 14\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (50)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[11/a529f0] process > run_robustness_analysis:run... [ 50%] 15 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 14\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (51)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e4/2df648] process > run_robustness_analysis:run... [ 50%] 15 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 15\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (52)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[71/62b2be] process > run_robustness_analysis:run... [ 53%] 16 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 15\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (52)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[71/62b2be] process > run_robustness_analysis:run... [ 53%] 16 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 16\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (53)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 56%] 17 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 17\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (54)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ce/14a13f] process > run_robustness_analysis:run... [ 60%] 18 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 18\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (55)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e2/5dd3e9] process > run_robustness_analysis:run... [ 63%] 19 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 19\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (55)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e2/5dd3e9] process > run_robustness_analysis:run... [ 63%] 19 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 19\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (56)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[71/62b2be] process > run_robustness_analysis:run... [ 66%] 20 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 19\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (56)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[71/62b2be] process > run_robustness_analysis:run... [ 66%] 20 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 20\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (57)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ee/2134b6] process > run_robustness_analysis:run... [ 70%] 21 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 20\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (57)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ee/2134b6] process > run_robustness_analysis:run... [ 70%] 21 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 21\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (58)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/e10dd4] process > run_robustness_analysis:run... [ 73%] 22 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 22\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (58)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/e10dd4] process > run_robustness_analysis:run... [ 73%] 22 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 22\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (59)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/1ffa8d] process > run_robustness_analysis:run... [ 76%] 23 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 22\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (59)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/1ffa8d] process > run_robustness_analysis:run... [ 76%] 23 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 23\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (60)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ce/14a13f] process > run_robustness_analysis:run... [ 80%] 24 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 24\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (61)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[db/6af869] process > run_robustness_analysis:run... [ 0%] 0 of 24\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (61)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[db/6af869] process > run_robustness_analysis:run... [ 0%] 0 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (62)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[97/46ce3e] process > run_robustness_analysis:run... [ 4%] 1 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (62)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[97/46ce3e] process > run_robustness_analysis:run... [ 8%] 2 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (63)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[13/dd0594] process > run_robustness_analysis:run... [ 8%] 2 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (64)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[cc/a14f0b] process > run_robustness_analysis:run... [ 12%] 3 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (65)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[dd/df4db4] process > run_robustness_analysis:run... [ 16%] 4 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (65)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[dd/df4db4] process > run_robustness_analysis:run... [ 16%] 4 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (66)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[1e/d146ba] process > run_robustness_analysis:run... [ 20%] 5 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (66)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[1e/d146ba] process > run_robustness_analysis:run... [ 20%] 5 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (67)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[88/7b4b56] process > run_robustness_analysis:run... [ 24%] 6 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (67)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[88/7b4b56] process > run_robustness_analysis:run... [ 24%] 6 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (68)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[24/082c3f] process > run_robustness_analysis:run... [ 28%] 7 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (68)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[24/082c3f] process > run_robustness_analysis:run... [ 28%] 7 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (69)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[f2/cf0248] process > run_robustness_analysis:run... [ 32%] 8 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (70)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[be/464085] process > run_robustness_analysis:run... [ 36%] 9 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (71)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[f5/03f97a] process > run_robustness_analysis:run... [ 40%] 10 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (71)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[f5/03f97a] process > run_robustness_analysis:run... [ 40%] 10 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (72)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[78/138a40] process > run_robustness_analysis:run... [ 44%] 11 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (73)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[d7/1ed34a] process > run_robustness_analysis:run... [ 48%] 12 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (74)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[09/a046e9] process > run_robustness_analysis:run... [ 52%] 13 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (75)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[c1/99784d] process > run_robustness_analysis:run... [ 56%] 14 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (75)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[c1/99784d] process > run_robustness_analysis:run... [ 56%] 14 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (76)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[3d/6f5339] process > run_robustness_analysis:run... [ 60%] 15 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (77)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[cd/7ad7f5] process > run_robustness_analysis:run... [ 64%] 16 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (78)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[34/a92115] process > run_robustness_analysis:run... [ 68%] 17 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (79)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[00/e4e2ee] process > run_robustness_analysis:run... [ 72%] 18 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (80)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[f5/b4e24a] process > run_robustness_analysis:run... [ 76%] 19 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (81)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[87/3bc33c] process > run_robustness_analysis:run... [ 80%] 20 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (81)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[87/3bc33c] process > run_robustness_analysis:run... [ 80%] 20 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (82)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[c8/fae13c] process > run_robustness_analysis:run... [ 84%] 21 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (83)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[e0/204b2c] process > run_robustness_analysis:run... [ 88%] 22 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (83)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[e0/204b2c] process > run_robustness_analysis:run... [ 88%] 22 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (84)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[14/9941fb] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[f8/b7bf95] process > run_robustness_analysis:run... [ 92%] 23 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (85)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/1ffa8d] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[bb/db3f8e] process > run_robustness_analysis:run... [ 92%] 23 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (86)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/e10dd4] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[40/90153e] process > run_robustness_analysis:run... [ 88%] 23 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (86)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/e10dd4] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[40/90153e] process > run_robustness_analysis:run... [ 85%] 23 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (87)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/e10dd4] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[07/730549] process > run_robustness_analysis:run... [ 88%] 24 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (87)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/e10dd4] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[bb/db3f8e] process > run_robustness_analysis:run... [ 92%] 25 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (87)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/e10dd4] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[40/90153e] process > run_robustness_analysis:run... [ 96%] 26 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (87)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4f/e10dd4] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[07/730549] process > run_robustness_analysis:run... [100%] 27 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (88)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[10/f830da] process > run_robustness_analysis:run... [ 93%] 28 of 30\u001b[K\n", - "[02/400b59] process > run_robustness_analysis:run... [ 96%] 27 of 28\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (88)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[10/f830da] process > run_robustness_analysis:run... [ 93%] 28 of 30\u001b[K\n", - "[02/400b59] process > run_robustness_analysis:run... [100%] 28 of 28\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (89)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[1a/9c2b76] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[5e/8cbd1a] process > run_robustness_analysis:run... [ 96%] 28 of 29\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (89)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[1a/9c2b76] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[5e/8cbd1a] process > run_robustness_analysis:run... [ 96%] 28 of 29\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (89)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7d/6ca956] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/8cbd1a] process > run_robustness_analysis:run... [ 96%] 28 of 29\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (90)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7d/6ca956] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e6/69e750] process > run_robustness_analysis:run... [ 93%] 28 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (90)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7d/6ca956] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/8cbd1a] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (90)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7d/6ca956] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e6/69e750] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (91)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7d/6ca956] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e6/69e750] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[71/51e091] process > run_robustness_analysis:pub... [ 0%] 0 of 1\u001b[K\n", - "\u001b[6A\n", - "executor > local (91)\u001b[K\n", - "[46/d03fc1] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7d/6ca956] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e6/69e750] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[71/51e091] process > run_robustness_analysis:pub... [100%] 1 of 1 ✔\u001b[K\n", - "\u001b[32;1mCompleted at: 25-Aug-2024 22:35:49\n", - "Duration : 9m 39s\n", - "CPU hours : 1.7\n", - "Succeeded : 91\n", - "\u001b[22;39m\u001b[K\n", - "\n" - ] - } - ], - "source": [ - "!bash scripts/run_robust_analys.sh \"weight\"" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_weight/scores.yaml to resources/results/robust_analy_reg2_weight/scores.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_weight/state.yaml to resources/results/robust_analy_reg2_weight/state.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_weight/metric_configs.yaml to resources/results/robust_analy_reg2_weight/metric_configs.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_weight/trace.txt to resources/results/robust_analy_reg2_weight/trace.txt\n" - ] - } - ], - "source": [ - "!aws s3 sync s3://openproblems-data/resources/grn/results/robust_analy_reg2_weight ./resources/results/robust_analy_reg2_weight" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Negative vs positive controls" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "!bash scripts/run_pc_vs_nc.sh \"pc_vs_nc\"" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "!aws s3 sync s3://openproblems-data/resources/grn/results/robust_analy_reg2_pc_vs_nc ./resources/results/robust_analy_reg2_pc_vs_nc" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## regulatory sign" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "sign\n", - "\u001b[33mNextflow 24.04.4 is available - Please consider updating your version to it\u001b[m\n", - "N E X T F L O W ~ version 23.10.4\n", - "Launching `target/nextflow/workflows/run_robustness_analysis/main.nf` [soggy_sinoussi] DSL2 - revision: d55e1b9116\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "\u001b[2A\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "\u001b[3A\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[5A\n", - "executor > local (2)\u001b[K\n", - "[2f/3f5479] process > run_robustness_analysis:run... [ 0%] 0 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (6)\u001b[K\n", - "[f9/652d7d] process > run_robustness_analysis:run... [ 0%] 0 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (7)\u001b[K\n", - "[d7/bd6de3] process > run_robustness_analysis:run... [ 3%] 1 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 1\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (9)\u001b[K\n", - "[61/b7d70f] process > run_robustness_analysis:run... [ 10%] 3 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 2\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (10)\u001b[K\n", - "[86/319727] process > run_robustness_analysis:run... [ 13%] 4 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 4\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (11)\u001b[K\n", - "[ad/5bccfc] process > run_robustness_analysis:run... [ 16%] 5 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 5\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (12)\u001b[K\n", - "[55/295e22] process > run_robustness_analysis:run... [ 20%] 6 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 5\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (12)\u001b[K\n", - "[55/295e22] process > run_robustness_analysis:run... [ 20%] 6 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 6\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (14)\u001b[K\n", - "[60/1b8a95] process > run_robustness_analysis:run... [ 26%] 8 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 8\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (15)\u001b[K\n", - "[a4/0f774f] process > run_robustness_analysis:run... [ 30%] 9 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 9\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (16)\u001b[K\n", - "[a4/bf1a59] process > run_robustness_analysis:run... [ 33%] 10 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 10\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (17)\u001b[K\n", - "[f0/3f59fb] process > run_robustness_analysis:run... [ 36%] 11 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 11\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (18)\u001b[K\n", - "[2d/97cfa6] process > run_robustness_analysis:run... [ 40%] 12 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 12\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (19)\u001b[K\n", - "[82/e5f963] process > run_robustness_analysis:run... [ 43%] 13 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 13\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (20)\u001b[K\n", - "[16/08d5fc] process > run_robustness_analysis:run... [ 46%] 14 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 14\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (21)\u001b[K\n", - "[17/a55008] process > run_robustness_analysis:run... [ 50%] 15 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 15\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (23)\u001b[K\n", - "[aa/63fbb5] process > run_robustness_analysis:run... [ 56%] 17 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 16\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (24)\u001b[K\n", - "[f0/3f59fb] process > run_robustness_analysis:run... [ 60%] 18 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 18\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (25)\u001b[K\n", - "[83/2cd2c1] process > run_robustness_analysis:run... [ 63%] 19 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 18\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (25)\u001b[K\n", - "[83/2cd2c1] process > run_robustness_analysis:run... [ 63%] 19 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 19\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (26)\u001b[K\n", - "[25/668fba] process > run_robustness_analysis:run... [ 66%] 20 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 20\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (27)\u001b[K\n", - "[c9/c54c3a] process > run_robustness_analysis:run... [ 70%] 21 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 21\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (28)\u001b[K\n", - "[45/4ba6a2] process > run_robustness_analysis:run... [ 73%] 22 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 22\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (28)\u001b[K\n", - "[45/4ba6a2] process > run_robustness_analysis:run... [ 73%] 22 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 22\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (29)\u001b[K\n", - "[2a/6e29ce] process > run_robustness_analysis:run... [ 76%] 23 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 23\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (30)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [ 80%] 24 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 24\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (31)\u001b[K\n", - "[83/2cd2c1] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[a5/757052] process > run_robustness_analysis:run... [ 0%] 0 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (32)\u001b[K\n", - "[25/668fba] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[0e/0accdb] process > run_robustness_analysis:run... [ 0%] 0 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (33)\u001b[K\n", - "[c9/c54c3a] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[f6/a2c7dd] process > run_robustness_analysis:run... [ 0%] 0 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (34)\u001b[K\n", - "[45/4ba6a2] process > run_robustness_analysis:run... [ 93%] 28 of 30\u001b[K\n", - "[81/2ff6fc] process > run_robustness_analysis:run... [ 0%] 0 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (34)\u001b[K\n", - "[45/4ba6a2] process > run_robustness_analysis:run... [ 93%] 28 of 30\u001b[K\n", - "[81/2ff6fc] process > run_robustness_analysis:run... [ 0%] 0 of 28\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (35)\u001b[K\n", - "[2a/6e29ce] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[e5/dd1da2] process > run_robustness_analysis:run... [ 0%] 0 of 29\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (36)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[c9/cc55af] process > run_robustness_analysis:run... [ 0%] 0 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (36)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[c9/cc55af] process > run_robustness_analysis:run... [ 0%] 0 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... -\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (37)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[55/b2845d] process > run_robustness_analysis:run... [ 3%] 1 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 1\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (38)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[0e/0accdb] process > run_robustness_analysis:run... [ 6%] 2 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 1\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (38)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[0e/0accdb] process > run_robustness_analysis:run... [ 6%] 2 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 2\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (39)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[81/2ff6fc] process > run_robustness_analysis:run... [ 10%] 3 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 2\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (39)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[81/2ff6fc] process > run_robustness_analysis:run... [ 10%] 3 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 3\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (40)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e5/dd1da2] process > run_robustness_analysis:run... [ 13%] 4 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 3\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (40)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[e5/dd1da2] process > run_robustness_analysis:run... [ 13%] 4 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 4\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (41)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[17/4cc270] process > run_robustness_analysis:run... [ 16%] 5 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 4\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (41)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[17/4cc270] process > run_robustness_analysis:run... [ 16%] 5 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 5\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (42)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[c9/cc55af] process > run_robustness_analysis:run... [ 20%] 6 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 6\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (43)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[4b/034b41] process > run_robustness_analysis:run... [ 23%] 7 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 7\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (44)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7a/54021d] process > run_robustness_analysis:run... [ 26%] 8 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 8\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (45)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[f6/a2c7dd] process > run_robustness_analysis:run... [ 30%] 9 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 9\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (46)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[dc/a6fcac] process > run_robustness_analysis:run... [ 33%] 10 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 10\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (47)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[d9/fb7083] process > run_robustness_analysis:run... [ 36%] 11 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 11\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (47)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[d9/fb7083] process > run_robustness_analysis:run... [ 36%] 11 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 11\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (48)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7b/5eddfc] process > run_robustness_analysis:run... [ 40%] 12 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 12\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (48)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[7b/5eddfc] process > run_robustness_analysis:run... [ 40%] 12 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 12\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (49)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[80/28e78a] process > run_robustness_analysis:run... [ 43%] 13 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 12\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (49)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[80/28e78a] process > run_robustness_analysis:run... [ 43%] 13 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 13\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (50)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[6b/9b2bad] process > run_robustness_analysis:run... [ 46%] 14 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 14\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (51)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[1b/00118d] process > run_robustness_analysis:run... [ 50%] 15 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 15\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (52)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[69/3cbc30] process > run_robustness_analysis:run... [ 53%] 16 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 15\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (52)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[69/3cbc30] process > run_robustness_analysis:run... [ 53%] 16 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 16\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (53)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[28/41b50f] process > run_robustness_analysis:run... [ 56%] 17 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 17\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (53)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[28/41b50f] process > run_robustness_analysis:run... [ 56%] 17 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 17\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (54)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 60%] 18 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 17\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (54)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 60%] 18 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 18\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (55)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5a/c20a99] process > run_robustness_analysis:run... [ 63%] 19 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 19\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (56)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/b50eb7] process > run_robustness_analysis:run... [ 66%] 20 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 20\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (57)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 70%] 21 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 21\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (58)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5a/c20a99] process > run_robustness_analysis:run... [ 73%] 22 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 22\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (59)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[69/3cbc30] process > run_robustness_analysis:run... [ 76%] 23 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 23\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (60)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[b0/1cd463] process > run_robustness_analysis:run... [ 80%] 24 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:run... [ 0%] 0 of 24\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (61)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[60/c4fb7e] process > run_robustness_analysis:run... [ 0%] 0 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (61)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[60/c4fb7e] process > run_robustness_analysis:run... [ 0%] 0 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (62)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[60/c4fb7e] process > run_robustness_analysis:run... [ 4%] 1 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (63)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[d8/3dccf5] process > run_robustness_analysis:run... [ 8%] 2 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (64)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[3a/c82a35] process > run_robustness_analysis:run... [ 12%] 3 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (65)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[fe/12f35c] process > run_robustness_analysis:run... [ 16%] 4 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (65)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[fe/12f35c] process > run_robustness_analysis:run... [ 16%] 4 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (66)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[fe/12f35c] process > run_robustness_analysis:run... [ 20%] 5 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (67)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[59/b22c05] process > run_robustness_analysis:run... [ 24%] 6 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (68)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[b5/f43d8a] process > run_robustness_analysis:run... [ 28%] 7 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (69)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[d7/23f6c7] process > run_robustness_analysis:run... [ 32%] 8 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (69)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[d7/23f6c7] process > run_robustness_analysis:run... [ 32%] 8 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (70)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[c0/e1ade3] process > run_robustness_analysis:run... [ 36%] 9 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (70)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[c0/e1ade3] process > run_robustness_analysis:run... [ 36%] 9 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (71)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[b6/e860a7] process > run_robustness_analysis:run... [ 40%] 10 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (71)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[b6/e860a7] process > run_robustness_analysis:run... [ 40%] 10 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (72)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[47/70e155] process > run_robustness_analysis:run... [ 44%] 11 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (72)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[47/70e155] process > run_robustness_analysis:run... [ 44%] 11 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (73)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[47/70e155] process > run_robustness_analysis:run... [ 48%] 12 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (73)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ca/0b682c] process > run_robustness_analysis:run... [ 83%] 25 of 30\u001b[K\n", - "[47/70e155] process > run_robustness_analysis:run... [ 48%] 12 of 25\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (74)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[6d/11513e] process > run_robustness_analysis:run... [ 46%] 12 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (75)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[da/de5833] process > run_robustness_analysis:run... [ 50%] 13 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (76)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[78/0ba11b] process > run_robustness_analysis:run... [ 53%] 14 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (77)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[88/5c09d9] process > run_robustness_analysis:run... [ 57%] 15 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (78)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[22/57f530] process > run_robustness_analysis:run... [ 61%] 16 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (79)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[4d/ed1ab5] process > run_robustness_analysis:run... [ 65%] 17 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (79)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[4d/ed1ab5] process > run_robustness_analysis:run... [ 65%] 17 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (80)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[22/57f530] process > run_robustness_analysis:run... [ 69%] 18 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (81)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[51/3cf21c] process > run_robustness_analysis:run... [ 73%] 19 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (82)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[c5/8c073d] process > run_robustness_analysis:run... [ 76%] 20 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (83)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[8e/6c35e8] process > run_robustness_analysis:run... [ 80%] 21 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (84)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[c9/214a53] process > run_robustness_analysis:run... [ 84%] 22 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (85)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[77/f748f1] process > run_robustness_analysis:run... [ 86%] 26 of 30\u001b[K\n", - "[20/538eef] process > run_robustness_analysis:run... [ 88%] 23 of 26\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (86)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/5e2fe1] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[ee/5dbe09] process > run_robustness_analysis:run... [ 85%] 23 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (86)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/5e2fe1] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[ee/5dbe09] process > run_robustness_analysis:run... [ 85%] 23 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (87)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/5e2fe1] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[db/cf8d69] process > run_robustness_analysis:run... [ 88%] 24 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (87)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/5e2fe1] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[20/538eef] process > run_robustness_analysis:run... [ 92%] 25 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (87)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/5e2fe1] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[ee/5dbe09] process > run_robustness_analysis:run... [ 96%] 26 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (87)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[be/5e2fe1] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[db/cf8d69] process > run_robustness_analysis:run... [100%] 27 of 27\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (88)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[94/a96236] process > run_robustness_analysis:run... [ 93%] 28 of 30\u001b[K\n", - "[62/4e8c10] process > run_robustness_analysis:run... [ 96%] 27 of 28\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (88)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[b0/1cd463] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[62/4e8c10] process > run_robustness_analysis:run... [ 96%] 27 of 28\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (89)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[b0/1cd463] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[ea/f3975f] process > run_robustness_analysis:run... [ 93%] 27 of 29\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (90)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/b50eb7] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[68/cffa95] process > run_robustness_analysis:run... [ 90%] 27 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (90)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/b50eb7] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[62/4e8c10] process > run_robustness_analysis:run... [ 93%] 28 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (90)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/b50eb7] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[ea/f3975f] process > run_robustness_analysis:run... [ 96%] 29 of 30\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (90)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/b50eb7] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[68/cffa95] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[- ] process > run_robustness_analysis:pub... -\u001b[K\n", - "\u001b[6A\n", - "executor > local (91)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/b50eb7] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[68/cffa95] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[27/96b063] process > run_robustness_analysis:pub... [ 0%] 0 of 1\u001b[K\n", - "\u001b[6A\n", - "executor > local (91)\u001b[K\n", - "[63/274a7b] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[5e/b50eb7] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[68/cffa95] process > run_robustness_analysis:run... [100%] 30 of 30 ✔\u001b[K\n", - "[27/96b063] process > run_robustness_analysis:pub... [100%] 1 of 1 ✔\u001b[K\n", - "\u001b[32;1mCompleted at: 26-Aug-2024 10:56:46\n", - "Duration : 9m 30s\n", - "CPU hours : 1.7\n", - "Succeeded : 91\n", - "\u001b[22;39m\u001b[K\n", - "\n" - ] - } - ], - "source": [ - "!bash scripts/run_robust_analys.sh \"sign\"" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_sign/trace.txt to resources/results/robust_analy_reg2_sign/trace.txt\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_sign/state.yaml to resources/results/robust_analy_reg2_sign/state.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_sign/scores.yaml to resources/results/robust_analy_reg2_sign/scores.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_reg2_sign/metric_configs.yaml to resources/results/robust_analy_reg2_sign/metric_configs.yaml\n" - ] - } - ], - "source": [ - "!aws s3 sync s3://openproblems-data/resources/grn/results/robust_analy_reg2_sign ./resources/results/robust_analy_reg2_sign" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Corr vs causal" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!bash scripts/run_robust_analys_causal.sh " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "download: s3://openproblems-data/resources/grn/results/robust_analy_causal_1/scores.yaml to resources/results/robust_analy_causal_1/scores.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_causal_1/state.yaml to resources/results/robust_analy_causal_1/state.yaml\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_causal_1/trace.txt to resources/results/robust_analy_causal_1/trace.txt\n", - "download: s3://openproblems-data/resources/grn/results/robust_analy_causal_1/metric_configs.yaml to resources/results/robust_analy_causal_1/metric_configs.yaml\n" - ] - } - ], - "source": [ - "!aws s3 sync s3://openproblems-data/resources/grn/results/robust_analy_causal_1 ./resources/results/robust_analy_causal_1" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "reg1-corr-5\n", - "reg1-corr-3\n", - "reg1-corr-2\n", - "reg1-corr-10\n", - "reg1-corr-1\n", - "reg1-corr-9\n", - "reg1-corr-8\n", - "reg1-corr-6\n", - "reg1-corr-7\n", - "reg1-corr-4\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ex(False)_tf(-1)ex(True)_tf(-1)Mean
corr_50.1604330.1687580.164596
corr_30.1918520.2021960.197024
corr_20.2026650.2135070.208086
corr_100.1664900.1752330.170861
corr_10.2226730.2346880.228680
corr_90.1825400.1922680.187404
corr_80.1867100.1966920.191701
corr_60.2172030.2290860.223144
corr_70.2380840.2509730.244528
corr_40.1788510.1879960.183423
\n", - "
" - ], - "text/plain": [ - " ex(False)_tf(-1) ex(True)_tf(-1) Mean\n", - "corr_5 0.160433 0.168758 0.164596\n", - "corr_3 0.191852 0.202196 0.197024\n", - "corr_2 0.202665 0.213507 0.208086\n", - "corr_10 0.166490 0.175233 0.170861\n", - "corr_1 0.222673 0.234688 0.228680\n", - "corr_9 0.182540 0.192268 0.187404\n", - "corr_8 0.186710 0.196692 0.191701\n", - "corr_6 0.217203 0.229086 0.223144\n", - "corr_7 0.238084 0.250973 0.244528\n", - "corr_4 0.178851 0.187996 0.183423" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[37mRead data\u001b[0m\n", - "\u001b[37m\u001b[0m\n", - "\u001b[37mNoramlize data\u001b[0m\n", - "\u001b[37mCreate corr net\u001b[0m\n", - "\u001b[37mProcessing groups: 0%| | 0/4 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "\n", - "import matplotlib.pyplot as plt\n", - "aa = plt.hist((df_==0).sum(axis=1)/df.shape[1], bins=100)" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/tmp/ipykernel_247612/3262953603.py:17: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n", - " groups = adata.obs.groupby([cell_type_col, donor_id_col])\n" - ] - }, - { - "ename": "ValueError", - "evalue": "invalid literal for int() with base 10: 'GTCTTTAGTTAGTGAT-3'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[34], line 53\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m meta_adata\n\u001b[1;32m 51\u001b[0m \u001b[38;5;66;03m# Usage\u001b[39;00m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;66;03m# Assuming you already have an AnnData object 'adata' with cell_type and donor_id in 'obs'\u001b[39;00m\n\u001b[0;32m---> 53\u001b[0m meta_adata \u001b[38;5;241m=\u001b[39m \u001b[43mcreate_meta_cells\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmultiomics_rna\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcell_type_col\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mcell_type\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdonor_id_col\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mdonor_id\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn_cells\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m15\u001b[39;49m\u001b[43m)\u001b[49m\n", - "Cell \u001b[0;32mIn[34], line 22\u001b[0m, in \u001b[0;36mcreate_meta_cells\u001b[0;34m(adata, cell_type_col, donor_id_col, n_cells)\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[38;5;66;03m# For each group (cell_type, donor_id combination)\u001b[39;00m\n\u001b[1;32m 20\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m (cell_type, donor_id), indices \u001b[38;5;129;01min\u001b[39;00m groups\u001b[38;5;241m.\u001b[39mgroups\u001b[38;5;241m.\u001b[39mitems():\n\u001b[1;32m 21\u001b[0m \u001b[38;5;66;03m# Convert indices to a numpy array of integers\u001b[39;00m\n\u001b[0;32m---> 22\u001b[0m cell_indices \u001b[38;5;241m=\u001b[39m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mindices\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mint\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 24\u001b[0m \u001b[38;5;66;03m# Filter indices to ensure they are within bounds\u001b[39;00m\n\u001b[1;32m 25\u001b[0m cell_indices \u001b[38;5;241m=\u001b[39m cell_indices[cell_indices \u001b[38;5;241m<\u001b[39m n_cells_in_adata]\n", - "\u001b[0;31mValueError\u001b[0m: invalid literal for int() with base 10: 'GTCTTTAGTTAGTGAT-3'" - ] - } - ], - "source": [ - "import numpy as np\n", - "import pandas as pd\n", - "import scanpy as sc\n", - "\n", - "def create_meta_cells(adata, cell_type_col='cell_type', donor_id_col='donor_id', n_cells=15):\n", - " # Initialize lists to store the meta cell data\n", - " meta_cell_data = []\n", - " meta_cell_obs = []\n", - "\n", - " # Get the raw counts matrix from the AnnData object\n", - " raw_counts = adata.X\n", - "\n", - " # Get the total number of rows in raw_counts\n", - " n_cells_in_adata = raw_counts.shape[0]\n", - "\n", - " # Group by cell_type and donor_id\n", - " groups = adata.obs.groupby([cell_type_col, donor_id_col])\n", - "\n", - " # For each group (cell_type, donor_id combination)\n", - " for (cell_type, donor_id), indices in groups.groups.items():\n", - " # Convert indices to a numpy array of integers\n", - " cell_indices = np.array(list(indices), dtype=int)\n", - "\n", - " # Filter indices to ensure they are within bounds\n", - " cell_indices = cell_indices[cell_indices < n_cells_in_adata]\n", - "\n", - " # Process in chunks of 'n_cells' (15)\n", - " for i in range(0, len(cell_indices), n_cells):\n", - " meta_cell_indices = cell_indices[i:i + n_cells]\n", - " \n", - " # Sum the raw counts for the meta cell\n", - " meta_cell_counts = raw_counts[meta_cell_indices].sum(axis=0)\n", - "\n", - " # Create observation data for the meta cell\n", - " meta_cell_obs.append({\n", - " cell_type_col: cell_type,\n", - " donor_id_col: donor_id\n", - " })\n", - " \n", - " # Append the meta cell data\n", - " meta_cell_data.append(meta_cell_counts)\n", - "\n", - " # Create a new AnnData object for the meta cells\n", - " meta_cell_data = np.array(meta_cell_data)\n", - " meta_cell_obs = pd.DataFrame(meta_cell_obs)\n", - "\n", - " meta_adata = sc.AnnData(X=meta_cell_data, obs=meta_cell_obs)\n", - "\n", - " return meta_adata\n", - "\n", - "# Usage\n", - "# Assuming you already have an AnnData object 'adata' with cell_type and donor_id in 'obs'\n", - "meta_adata = create_meta_cells(multiomics_rna, cell_type_col='cell_type', donor_id_col='donor_id', n_cells=15)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "AnnData object with n_obs × n_vars = 25551 × 22787\n", - " obs: 'cell_type', 'donor_id'\n", - " var: 'gene_ids', 'interval', 'mean', 'std'\n", - " uns: 'log1p'" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Read data\n", - "Noramlize data\n" - ] - } - ], - "source": [ - "import os\n", - "import pandas as pd\n", - "import numpy as np\n", - "import anndata as ad\n", - "import scanpy as sc\n", - "from tqdm import tqdm\n", - "from scipy.stats import spearmanr\n", - "\n", - "## VIASH START\n", - "par = {\n", - " \"multiomics_rna\": \"resources/grn-benchmark/multiomics_rna.h5ad\",\n", - " \"tf_all\": \"resources_test/prior/tf_all.csv\",\n", - " \"corr_method\": \"spearman\"\n", - "}\n", - "## VIASH END\n", - "print('Read data')\n", - "multiomics_rna = ad.read_h5ad(par[\"multiomics_rna\"])\n", - "gene_names = multiomics_rna.var_names.to_numpy()\n", - "tf_all = np.loadtxt(par['tf_all'], dtype=str)\n", - "tf_all = np.intersect1d(tf_all, gene_names)\n", - "groups = multiomics_rna.obs.cell_type\n", - "print('Noramlize data')\n", - "sc.pp.normalize_total(multiomics_rna)\n", - "sc.pp.log1p(multiomics_rna)\n", - "sc.pp.scale(multiomics_rna)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Processing groups: 0%| | 0/4 [00:00 15\u001b[0m \u001b[43maa\u001b[49m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;66;03m# grn = X_sub.T.dot(X_sub)\u001b[39;00m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;66;03m# grn = grn/stds\u001b[39;00m\n\u001b[1;32m 18\u001b[0m grn\n", - "\u001b[0;31mNameError\u001b[0m: name 'aa' is not defined" - ] - } - ], - "source": [ - "X = multiomics_rna.X\n", - "for group in tqdm(np.unique(groups), desc=\"Processing groups\"):\n", - " mask_group = groups == group\n", - " X_sub = X[mask_group, :]\n", - " # stds = np.std(X_sub, axis=0)\n", - " # mask_g = stds != 0\n", - " # stds = stds[mask_g]\n", - " # X_sub = X_sub[:, mask_g]\n", - " # g_names = gene_names[mask_g]\n", - " \n", - " # grn = X_sub.T.dot(X_sub)/np.std(X_sub, axis=0)\n", - " grn = np.corrcoef(X_sub, rowvar=False)\n", - " # grn = np.nan_to_num(grn, nan=0.0, posinf=0.0, neginf=0.0)\n", - " print(grn)\n", - " aa\n", - " # grn = X_sub.T.dot(X_sub)\n", - " # grn = grn/stds\n", - "grn" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Processing groups: 0%| | 0/4 [00:00 25\u001b[0m net \u001b[38;5;241m=\u001b[39m \u001b[43mcreate_corr_net\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmultiomics_rna\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgroups\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpar\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mcorr_method\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n", - "Cell \u001b[0;32mIn[34], line 22\u001b[0m, in \u001b[0;36mcreate_corr_net\u001b[0;34m(X, groups, method)\u001b[0m\n\u001b[1;32m 20\u001b[0m grn \u001b[38;5;241m=\u001b[39m spearmanr(X_sub)\u001b[38;5;241m.\u001b[39mstatistic\n\u001b[1;32m 21\u001b[0m \u001b[38;5;28mprint\u001b[39m(grn)\n\u001b[0;32m---> 22\u001b[0m \u001b[43maa\u001b[49m \n\u001b[1;32m 23\u001b[0m grns\u001b[38;5;241m.\u001b[39mappend(grn)\n\u001b[1;32m 24\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39mmean(grns, axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m)\n", - "\u001b[0;31mNameError\u001b[0m: name 'aa' is not defined" - ] - } - ], - "source": [ - "from scipy.stats import pearsonr\n", - "\n", - "def create_corr_net(X: np.ndarray, groups: np.ndarray, method=\"pearson\"):\n", - " grns = []\n", - " for group in tqdm(np.unique(groups), desc=\"Processing groups\"):\n", - " mask_group = groups == group\n", - " X_sub = X[mask_group, :]\n", - " stds = np.std(X_sub, axis=0)\n", - " mask_g = stds != 0\n", - " stds = stds[mask_g]\n", - " X_sub = X_sub[:, mask_g]\n", - " g_names = gene_names[mask_g]\n", - " if method == \"pearson\":\n", - " # grn = np.corrcoef(X_sub.T)\n", - " grn = X_sub.T.dot(X_sub)\n", - " grn = grn/stds\n", - "\n", - " elif method == \"spearman\":\n", - " grn = spearmanr(X_sub).statistic\n", - "\n", - " grns.append(grn)\n", - " return np.mean(grns, axis=0)\n", - "net = create_corr_net(multiomics_rna.X, groups, par['corr_method'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "\n", - "print('Read data')\n", - "\n", - "\n", - "print('Create corr net')\n", - "\n", - "net = pd.DataFrame(net, index=gene_names, columns=gene_names)\n", - "\n", - "if par['causal']:\n", - " net = net[tf_all]\n", - "else:\n", - " net = net.sample(len(tf_all), axis=1, random_state=par['seed'])\n", - " \n", - "net = net.reset_index().melt(id_vars='index', var_name='source', value_name='weight')\n", - "net.rename(columns={'index': 'target'}, inplace=True)\n", - "\n", - "\n", - "print('Output GRN')\n", - "net.to_csv(par['prediction'])\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Sinlge omics" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!bash scripts/run_benchmark_single_omics.sh" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "download: s3://openproblems-data/resources/grn/results/single_omics_inference/ridge.scenic.scenic.prediction.csv to resources/results/single_omics_inference/ridge.scenic.scenic.prediction.csv\n" - ] - } - ], - "source": [ - "!aws s3 sync s3://openproblems-data/resources/grn/results/single_omics_inference resources/results/single_omics_inference" - ] - }, - { - "cell_type": "code", - "execution_count": 116, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "download: s3://openproblems-data/resources/grn/results/grn_evaluation_all_ridge/scores.yaml to resources/results/grn_evaluation_all_ridge/scores.yaml\n", - "download: s3://openproblems-data/resources/grn/results/grn_evaluation_all_ridge/metric_configs.yaml to resources/results/grn_evaluation_all_ridge/metric_configs.yaml\n", - "download: s3://openproblems-data/resources/grn/results/grn_evaluation_all_ridge/trace.txt to resources/results/grn_evaluation_all_ridge/trace.txt\n" - ] - } - ], - "source": [ - "!aws s3 sync s3://openproblems-data/resources/grn/results/grn_evaluation_all_ridge resources/results/grn_evaluation_all_ridge" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
 ex(False)_tf(-1)ex(True)_tf(-1)static-theta-0.0static-theta-0.5Rank
grnboost20.2656090.4032310.8310480.5883931
portia0.0146770.0353050.5804740.5352892
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 42, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "\n", - "models_all = ['negative_control', 'baseline_pearson', \n", - " 'baseline_pearson_causal', 'baseline_pearson_causal_celltype', \n", - " 'baseline_pearson_causal_metacell', 'positive_control', 'collectri','granie', 'figr', 'celloracle', \n", - " 'scglue', 'scenicplus', 'portia','ppcor', 'grnboost2', 'genie3']\n", - "\n", - "def extract_data(data, reg='reg1', dataset_id='scgen_pearson'):\n", - " i = 0\n", - " for entry in data:\n", - " if entry['dataset_id']!=dataset_id:\n", - " continue\n", - " try:\n", - " rg, method_id = entry['method_id'].split('-')\n", - " except:\n", - " rg, method_id, _ = entry['method_id'].split('-')\n", - " if rg != reg:\n", - " continue\n", - " dataset_id = entry['dataset_id']\n", - " metric_ids = entry['metric_ids']\n", - " metric_values = entry['metric_values']\n", - " \n", - " df = pd.DataFrame([metric_values], index=[method_id], columns=metric_ids)\n", - " if i==0:\n", - " df_reg = df\n", - " else:\n", - " df_reg = pd.concat([df_reg, df], axis=0)\n", - " i+=1\n", - " return df_reg\n", - "import yaml\n", - "import pandas as pd\n", - "# base_folder = 'resources/results/grn_evaluation_all_ridge/'\n", - "base_folder = 'resources/results/single_omics_inference'\n", - "\n", - "result_file = f'{base_folder}/scores.yaml'\n", - "with open(result_file, 'r') as file:\n", - " data = yaml.safe_load(file)\n", - "if False:\n", - " df_reg1 = extract_data(data, reg='reg1').reindex(models_all).drop(columns=['Mean'])\n", - " df_reg2 = extract_data(data, reg='reg2').reindex(models_all).drop(columns=['Mean'])\n", - "else:\n", - " df_reg1 = extract_data(data, reg='reg1').drop(columns=['Mean'])\n", - " df_reg2 = extract_data(data, reg='reg2').drop(columns=['Mean'])\n", - "df_all = pd.concat([df_reg1,df_reg2], axis=1).fillna(0)\n", - "df_all[df_all<0]=0\n", - "df_all_n = (df_all-df_all.min(axis=0))/(df_all.max(axis=0)-df_all.min(axis=0))\n", - "df_all['Rank'] = df_all_n.mean(axis=1).rank(ascending=False).astype(int)\n", - "df_all.style.background_gradient()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "py10", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/runs.ipynb b/runs.ipynb index 3fe3bd62c..51afd6a06 100644 --- a/runs.ipynb +++ b/runs.ipynb @@ -4,1204 +4,367 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Commands" + "# Import" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 23, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "upload: resources/grn-benchmark/multiomics_rna_qc.h5ad to s3://openproblems-data/resources/grn/grn-benchmark/multiomics_rna_qc.h5ad\n", - "upload: resources/grn-benchmark/multiomics_rna_0.h5ad to s3://openproblems-data/resources/grn/grn-benchmark/multiomics_rna_0.h5ad\n", - "upload: resources/grn-benchmark/multiomics_atac_0.h5ad to s3://openproblems-data/resources/grn/grn-benchmark/multiomics_atac_0.h5ad\n" - ] - } - ], + "outputs": [], "source": [ - "!aws s3 sync resources/grn-benchmark s3://openproblems-data/resources/grn/grn-benchmark " + "import yaml\n", + "import pandas as pd\n", + "import anndata as ad \n", + "import numpy as np\n", + "from src.exp_analysis.helper import plot_cumulative_density\n", + "def extract_data(data, reg='reg1', dataset_id='scgen_pearson'):\n", + " i = 0\n", + " for entry in data:\n", + " if entry['dataset_id']!=dataset_id:\n", + " continue\n", + " try:\n", + " rg, method_id = entry['method_id'].split('-')\n", + " except:\n", + " rg, method_id, _ = entry['method_id'].split('-')\n", + " if rg != reg:\n", + " continue\n", + " dataset_id = entry['dataset_id']\n", + " metric_ids = entry['metric_ids']\n", + " metric_values = entry['metric_values']\n", + " \n", + " df = pd.DataFrame([metric_values], index=[method_id], columns=metric_ids)\n", + " if i==0:\n", + " df_reg = df\n", + " else:\n", + " df_reg = pd.concat([df_reg, df], axis=0)\n", + " i+=1\n", + " return df_reg\n", + "def process_data(RUN_ID, models_all):\n", + " !aws s3 sync s3://openproblems-data/resources/grn/results/{RUN_ID} resources/results/{RUN_ID} \n", + " base_folder = f'resources/results/{RUN_ID}'\n", + " result_file = f'{base_folder}/scores.yaml'\n", + " \n", + "\n", + " with open(result_file, 'r') as file:\n", + " data = yaml.safe_load(file)\n", + " if True:\n", + " df_reg1 = extract_data(data, reg='reg1').reindex(models_all).drop(columns=['Mean'])\n", + " df_reg2 = extract_data(data, reg='reg2').reindex(models_all).drop(columns=['Mean'])\n", + " else:\n", + " df_reg1 = extract_data(data, reg='reg1').drop(columns=['Mean'])\n", + " df_reg2 = extract_data(data, reg='reg2').drop(columns=['Mean'])\n", + " df_all = pd.concat([df_reg1, df_reg2], axis=1).fillna(0)\n", + " return df_all" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ - "import yaml\n", - "import pandas as pd \n", - "import matplotlib.pyplot as plt\n", - "import anndata as ad\n", - "controls = ['negative_control','positive_control']\n", - "grn_models = ['collectri','granie', 'figr', 'celloracle', 'scglue', 'scenicplus']" + "from sklearn.preprocessing import StandardScaler \n", + "adata = ad.read_h5ad('resources/grn-benchmark/multiomics_rna.h5ad')" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 17, "metadata": {}, + "outputs": [], "source": [ - "# Test" + "X_tran = StandardScaler().fit_transform(adata.X.todense().A)" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 27, "metadata": {}, "outputs": [ { "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Unnamed: 0sourcetargetweight
00BCL6FBP10.544815
11BCL6FAM157C1.530639
22BCL6OLIG10.508618
33BCL6BIRC50.915793
44BCL6TMED50.523122
...............
56865686ZNF816FLII0.160327
56875687ZNF816KNOP10.167249
56885688ZNF816SCO10.184201
56895689ZNF816SH2D4B0.072754
56905690ZNF816SLC13A30.313595
\n", - "

5691 rows × 4 columns

\n", - "
" - ], "text/plain": [ - " Unnamed: 0 source target weight\n", - "0 0 BCL6 FBP1 0.544815\n", - "1 1 BCL6 FAM157C 1.530639\n", - "2 2 BCL6 OLIG1 0.508618\n", - "3 3 BCL6 BIRC5 0.915793\n", - "4 4 BCL6 TMED5 0.523122\n", - "... ... ... ... ...\n", - "5686 5686 ZNF816 FLII 0.160327\n", - "5687 5687 ZNF816 KNOP1 0.167249\n", - "5688 5688 ZNF816 SCO1 0.184201\n", - "5689 5689 ZNF816 SH2D4B 0.072754\n", - "5690 5690 ZNF816 SLC13A3 0.313595\n", - "\n", - "[5691 rows x 4 columns]" + "(
,\n", + " )" ] }, - "execution_count": 9, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAFzCAYAAADSc9khAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCCklEQVR4nO3deXxTVd4/8E+SJmlLV+hKKbRlK/tOLSqLtiw6jDCjIvoAg4rjUn9KwcEqUNFnLOOCuKCMKCIzo6jzqONIRWppkaWy75ssLWXpSkv3Nmlyfn+kjcaWkpSkN7n5vF8vXpCbc2++h7YfLueee65CCCFARERuQSl1AURE1HEY+kREboShT0TkRhj6RERuhKFPRORGGPpERG6EoU9E5EYY+kREbsRD6gI6mtFoxOXLl+Hr6wuFQiF1OUREN0wIgaqqKnTt2hVKZdvn8m4X+pcvX0ZkZKTUZRAR2d2FCxfQrVu3Ntu4Xej7+voCMP3l+Pn5SVxN++j1emzevBkTJ06EWq2WupwbJrf+APLrk9z6A8irT5WVlYiMjDTnW1vcLvSbh3T8/PxcOvS9vb3h5+fn8t+sgPz6A8ivT3LrDyDPPlkzZM0LuUREboShT0TkRhj6RERuhKFPRORGGPpERG6EoU9E5EYY+kREbkTS0P/xxx8xdepUdO3aFQqFAl9//fV198nOzsbw4cOh1WrRq1cvrFu3zuF1EhHJhaShX1NTgyFDhmDVqlVWtc/NzcWdd96JCRMm4ODBg3j66afx8MMP4/vvv3dwpURE8iDpHblTpkzBlClTrG6/evVqREdH4/XXXwcA9OvXD9u3b8cbb7yBSZMmOapMInIRQggIAQgARiFgbH4tTK+btwsj0KDXoVoPXKlugFA2wmAUMBgFGpt/NwjoDUbU6gxQqxQQ5s9o+pymPwOAgOlDLdrgl1rMtZkao+lP5tpE0/vbz5RiVFRn3NIrCIGdNA75O3KpZRhycnKQkJBgsW3SpEl4+umnr7lPQ0MDGhoazK8rKysBmG7B1uv1DqnT0ZrrdtX6f0tu/QFcq09Go0B9owH1eiNqdI2o1xlRrWtERZ0eSoUCeoMRDbpG7C9V4OKPZxHgrYXeYITOIHChvA7eahWUSphDs/mXzmDEz0XViOriDYMRMIhf3jOKX9qcLKxCr2AfGIUpcI1G0dQWMBiNOF1cg0BvNbzUKotjNAd0rc4ApQIWIWw9Dzy/d6sD/lbbb33OeXz+yGgMiwyweh9bvs9cKvQLCwsRGhpqsS00NBSVlZWoq6uDl5dXi33S0tKwbNmyFts3b94Mb29vh9XaETIyMqQuwa7k1h/Afn0yGIGaRqBKD5TrTAnXKIC6RqC2EVAogLpGBUrrTa/VSqDBCDQYFGg0mtoW15naeakAowAMAtAZbVleXAWcPmtz7QcuVFy3ze6a8jbfL6/VoxzXDjajzWHfOq1SQKkAlApA1fR7pc70e4AWaP7bMv+usHxt/rPC9LvF9mu0/e32YE+Bi4d2ouCI9XXX1tZa3dalQr89UlJSkJycbH7dvBrdxIkTXXrBtYyMDCQmJspioSi59Qewrk+1ukYUVjTg4tU65JfV4mRhFYqrGqBSKFBc3YDDFysR6K1Gea39/rcghOkfj2tRqxTQGwS6+nuiuqERSoUC3Tt7QaUAqiorENylM85dqcWQbv7QeiihVilRUtWAMH9P+Gg9oFIq4KFUQKVUQNWUZuW1OnQL9IKy6T2lwvS+6XfT59bpDAjy0Zq2KxVQKQCVUgmVEk3/2xDw9fSAR/Mxfv05TccyHdu06JhCYdrP/Bq/ea0ADI2NyPzhB0yc6Prfd80jGNZwqdAPCwtDUVGRxbaioiL4+fm1epYPAFqtFlqttsV2tVrt8l9oOfTh1+TUHyEEyhuA3fmVOHypCo0GI0qqG3ClWoeiqgZcKq9FabXuusf5beB37qSBp4cS5bV6DIrwh8ZDCV2jEVAAfUN9EeCthlKhgFqlQNcAL3hrPOClUUGjUkLjYQphX081PJQKqFWm0FYpFfDWqOCpVkGlbP3MX6/XIz09HXfcMUo2XyO9XgmFQh7fd7bU71KhHx8fj/T0dIttGRkZiI+Pl6gicmdCCBRW1mN3bhkOXahAbmk1Omk9UFRZjzPF1Siv9QD277vucTQqJYb3CEDnThoYjUCvEB/Ehvsi1M8Tnh4qBHir0UnrAX8v9TVDmchakoZ+dXU1zpw5Y36dm5uLgwcPonPnzujevTtSUlJw6dIlrF+/HgDw6KOP4p133sFf/vIXPPjgg9iyZQs+//xzbNy4UaoukBtoNBiRW1qDzJPFKK/R4djlSmw/U3rd/RQQiA7qhB5dOqFWZ0B8zy7o3EmDEF9PdAv0QmRnb/h5evCxndShJA39vXv3YsKECebXzWPvc+bMwbp161BQUID8/Hzz+9HR0di4cSPmz5+PN998E926dcMHH3zA6ZpkN1eqG3D4YgWOXKpA5sliXCirRVW9HnrDta8U9grxgYdSge6dvdEt0BuDu/kjMkCLcwd2YNrUW1x+6IDkRdLQHz9+PEQbc6xau9t2/PjxOHDggAOrInchhMD+/KvYm1eG3bllOHjhKq7UtD7OrlYp0DfMFxV1esyJj0KvEB+E+nmiZ7APNB4t73HU6/W4eNjRPSCynUuN6RPdqMKKemSeLMLOM1ew93wZiiobWrSJ6uKNQd0C0CvYB0G+GsTHdEFUl05QcjydZIChT7JmNArsySvDN4cuY3duGc6WVFvM6fZSq3BL7yAM6x6AUVGd0S/cDz5a/liQfPG7m2Snedhm2+kSfLH3Ii5drbN4f1j3AIzvE2IOei+NSqJKiToeQ59k49jlCnxz6DIyjhXhXGmNeXsnjQoTYkMQF9MFwyIDMDDCX8IqiaTF0CeXVlRZj4935mHHmVIcuvjL7f4aDyWiu3TCvLEx+N3gcHiqeTZPBDD0yQUZjAI7zpRi4+ECfHXwkumO1CaxYb54bHxP3BYbAl9PTpUk+i2GPrmMilo91ufk4Z+7zlvMuhkU4Y/E/qG4e0Q3dA1ofTkOIjJh6JPTK6lqQOo3R/HDiWLzWb2vpwfuGBiOPwyPwOjozryrlchKDH1yWuU1Ovz9x3NYn5OHWp0BABDiq8WzU2Jx5+BwaD04Tk9kK4Y+OZ1aXSP++dN5vLPlDCrrTesAD4zww6LJsbilVxDP6oluAEOfnEZJVQPW7zqLf+TkoabpzD42zBcLJvZFQr8Qhj2RHTD0SXJV9Xr8+5wSC3b9iMam22UjO3vhsXG9cO/IbvBQtVzbhojah6FPkhFCYMOeC3hl00mU1yoBCAzvHoBHxvbExP6hXOuGyAEY+iSJPXlleO7LIzhdXA0A6KwVWDFzJMbHhklcGZG8MfSpQ5VUNeDV70/i870XAQAqpQLzb++F8MoTuLlnF4mrI5I/hj51CF2jEWt35OLdrF9m5PxxeDcsmtIXgZ4qpKefkLhCIvfA0CeHK6yox5Of7seevHIApgd4L7trAG6KMZ3Z6/X6tnYnIjti6JPDCCHw6e4LePHbY6jXG+GpVmLR5Fj8z009oOaMHCJJMPTJIU4VVuGFb44h59wVAEDvEB+smT0SUUGdJK6MyL0x9Mnusk4W4/F/7Ued3gCNSokFE/tg3q0xnIJJ5AQY+mQ3Qgh8uD0Xf00/ASGAMT27YPkfBqN7F2+pSyOiJgx9souGRgMW/fswvj54GQBwz4hueGnaQD68hMjJMPTphv1cVIUFnx/CkUumJ1ctmhyLR8fFcK0cIifE0Kcb8s2hy3jmi0NoaDTCz9MDb80chvF9Q6Qui4iugaFP7aI3GPG3707ig+25AIBRUYF45/7hCPXzlLgyImoLQ59sVlWvx//79ACyTpUAAB66JRqLJsdC48G590TOjqFPNjl/pQZz1u5G3pVaqJQKvHr3YPxheDepyyIiKzH0yWpHL1XgTx/tRmm1DkE+Wqy6fxjiYrhIGpErYeiTVXaeKcWfPtoDncGIvqG++PjB0Qjz5/g9kath6NN1bT9digc/NgX+qKhAfDB7FPy91VKXRUTtwNCnNm39uQTz1u+FrtGI22JD8O4Dw3nDFZELY+jTNW0/XYpHmgJ/XJ9gvPc/w6H1YOATuTLOsaNW7Ttfjj//Yy8aGo24tXcQ3p89goFPJAMMfWrhVGEV5n60GzU6A0b0CMT7s0Yy8IlkgqFPFipq9Uj6ZD8q6xsxvHsAPpo7Cl4aBj6RXDD0yayh0YBH/7kPp4urEeSjwTv3D4efJ2fpEMkJQ58AAEajwDNfHEbOuSvwVCux/sE4dA3wkrosIrIzhj4BAN7JOoNvDl2GUgG898AI9O/qJ3VJROQADH3Cjz+XYEXGzwCAF+8aiAmxXBqZSK4Y+m7uaq0Oi/7vMADgvlGReCCuu8QVEZEjMfTd3PLvTqKgoh4RAV5YOrU/n3ZFJHMMfTd2/HIlvth3EQDw2j1D4K3hDdpEcsfQd1O6RiMW/d9hGIwCkwaEIr4nl0gmcgcMfTe1PicPRy5VwM/TAy/8foDU5RBRB2Hou6GrtTqsyjoDAHh2Sj+E+3M+PpG7kDz0V61ahaioKHh6eiIuLg67d+9us/3KlSvRt29feHl5ITIyEvPnz0d9fX0HVSsPq7eeQ3mtHn1CfXDvSD7qkMidSBr6n332GZKTk5Gamor9+/djyJAhmDRpEoqLi1tt/8knn+DZZ59FamoqTpw4gQ8//BCfffYZnnvuuQ6u3HVdKKvF+pw8AMDCiX3hoZL8330i6kCS/sSvWLEC8+bNw9y5c9G/f3+sXr0a3t7eWLt2bavtd+7ciZtvvhn3338/oqKiMHHiRMycOfO6/zugX7ycfgK1OgNGR3VGQr9Qqcshog4m2Rw9nU6Hffv2ISUlxbxNqVQiISEBOTk5re4zZswY/POf/8Tu3bsxevRonDt3Dunp6Zg1a9Y1P6ehoQENDQ3m15WVlQAAvV4PvV5vp950rOa6ba3/wIWr+O5oIZQKYOmdfWEwNMJgcESFtmlvf5yZ3Pokt/4A8uqTLX2QLPRLS0thMBgQGmp5thkaGoqTJ0+2us/999+P0tJS3HLLLRBCoLGxEY8++mibwztpaWlYtmxZi+2bN2+Gt7f3jXVCYhkZGVa3FQJ4+5gKgAIjg4w4u38bzjqutHaxpT+uQm59klt/AHn0qba21uq2LnU3TnZ2Nl5++WW8++67iIuLw5kzZ/DUU0/hpZdewpIlS1rdJyUlBcnJyebXlZWViIyMxMSJE+Hn55qLiun1emRkZCAxMRFqtXVLH+87X46zP+2BxkOJV2ff6lQraLanP85Obn2SW38AefWpeQTDGpKFflBQEFQqFYqKiiy2FxUVISwsrNV9lixZglmzZuHhhx8GAAwaNAg1NTV45JFH8Pzzz0OpbHmJQqvVQqvVttiuVqtd/gttSx/W77oAAJg2tCt6BDvnP3Zy+Jr8ltz6JLf+APLoky31S3YhV6PRYMSIEcjMzDRvMxqNyMzMRHx8fKv71NbWtgh2lcr0VCchhOOKdXFnS6qx6WghAODBW6IlroaIpCTp8E5ycjLmzJmDkSNHYvTo0Vi5ciVqamowd+5cAMDs2bMRERGBtLQ0AMDUqVOxYsUKDBs2zDy8s2TJEkydOtUc/tTSqqwzMArg9tgQxIY551k+EXUMSUN/xowZKCkpwdKlS1FYWIihQ4di06ZN5ou7+fn5Fmf2ixcvhkKhwOLFi3Hp0iUEBwdj6tSp+Otf/ypVF5zehbJa/OfgZQBA0m29JK6GiKQm+YXcpKQkJCUltfpedna2xWsPDw+kpqYiNTW1AyqTh7U7cmEwCozp2QXDugdKXQ4RSYy3Y8pYRa0en+8xXcB98GaO5RMRQ1/Wvj1yGTU6A3oGd8JtfAQiEYGhL1tCCKzfeR4AcPeISCiVfCIWETH0ZevAhas4VVQFT7US9/O5t0TUhKEvU5/sygcA3DEwHP5ern3jCRHZD0Nfhirr9fj2sGmaJs/yiejXGPoy9N9Dl1GvN6J3iA9G9OA0TSL6BUNfhr4+cAkA8Ifh3aBQ8AIuEf2CoS8z56/UYE9eOZQK4K6hXaUuh4icDENfZr5qOssf0zPIqZZPJiLnYHPoR0VF4cUXX0R+fr4j6qEbIITAfw+ZLuBOGxYhcTVE5IxsDv2nn34aX375JWJiYpCYmIgNGzZYPI6QpHO2pBpnS2qgVikwcQCff0tELbUr9A8ePIjdu3ejX79+ePLJJxEeHo6kpCTs37/fETWSlf57qAAAcHOvIPh5cm4+EbXU7jH94cOH46233sLly5eRmpqKDz74AKNGjcLQoUOxdu1aPtREAt80De1M59AOEV1Du5dW1uv1+Oqrr/DRRx8hIyMDN910Ex566CFcvHgRzz33HH744Qd88skn9qyV2nCupBq5paahHS6uRkTXYnPo79+/Hx999BE+/fRTKJVKzJ49G2+88QZiY2PNbaZPn45Ro0bZtVBqW9apEgBAXHQX+HJoh4iuwebQHzVqFBITE/Hee+9h2rRprT6QNzo6Gvfdd59dCiTrZJ8qBgDc0jtI4kqIyJnZHPrnzp1Djx492mzTqVMnfPTRR+0uimxTWt2AnWevAAAm9uesHSK6Npsv5E6YMAFXrlxpsf3q1auIiYmxS1Fkm01HC2EwCgzu5o+YYB+pyyEiJ2Zz6Ofl5cFgMLTY3tDQgEuXLtmlKLJNxvEiAMCkAWESV0JEzs7q4Z1vvvnG/Ofvv/8e/v7+5tcGgwGZmZmIioqya3F0ffV6A3KahnYm8YYsIroOq0N/2rRpAACFQoE5c+ZYvKdWqxEVFYXXX3/drsXR9e3JK4POYESQjxY9ObRDRNdhdegbjUYAppk5e/bsQVAQZ4k4g61NUzXH9gniMspEdF02z97Jzc11RB3UDkIIbG4az0/ox6EdIro+q0L/rbfewiOPPAJPT0+89dZbbbb9f//v/9mlMLq+k4XVyC+rhdZDiXF9gqUuh4hcgFWh/8Ybb+CBBx6Ap6cn3njjjWu2UygUDP0OlHmy6YasXkHopG33ihpE5EasSopfD+lweMd5ZJwwhT6XUSYia93wk7MMBgMOHjyI8vJye9RDViqpA44XVMFDqcDtHM8nIiu1az39Dz/8EIAp8MeOHYvhw4cjMjIS2dnZ9q6PruFkhWmmzogegQjy0UpcDRG5CptD/9///jeGDBkCAPjvf/+LvLw8nDx5EvPnz8fzzz9v9wKpdaebQv/mXpw6S0TWszn0S0tLERZmut0/PT0d99xzD/r06YMHH3wQR44csXuB1FKjwYifm0Kfq2oSkS1sDv3Q0FAcP34cBoMBmzZtQmJiIgCgtrYWKpXK7gVSS/vyr6LOoECgtxpDugVIXQ4RuRCb5/nNnTsX9957L8LDw6FQKJCQkAAA2LVrl8WDVMhxdp4tAwCM6dkFKiXvwiUi69kc+i+88AIGDhyICxcu4J577oFWa7qIqFKp8Oyzz9q9QGqp+SlZt/bqInElRORq2nVHz913391i228XYSPHKKlqwInCKgDA+L68C5eIbNOu0M/MzERmZiaKi4vNC7E1W7t2rV0Ko9Y1PxaxWyeBLp00EldDRK7G5tBftmwZXnzxRYwcOdI8rk8dZ+vPpqGd/gFC4kqIyBXZHPqrV6/GunXrMGvWLEfUQ20wGoX5gSl9A4zXaU1E1JLNUzZ1Oh3GjBnjiFroOo5cqsCVGh06aVWI4vNSiKgdbA79hx9+GJ988okjaqHraB7aGRPTBR43vGoSEbkjm4d36uvr8f777+OHH37A4MGDoVarLd5fsWKF3YojS9tPlwIAxvYOAkouS1wNEbkim0P/8OHDGDp0KADg6NGjFu/xoq7j1OoaceCCaSXT+JjOOFYicUFE5JJsDv2srCxH1EHXsTu3DHqDQLi/J7p39sIxqQsiIpfU7pHhM2fO4Pvvv0ddXR0A0/NayXF2nGke2gnm/6iIqN1sDv0rV67g9ttvR58+fXDHHXegoKAAAPDQQw9hwYIFdi+QTLY1jeffzFU1iegG2Bz68+fPh1qtRn5+Pry9vc3bZ8yYgU2bNtlcwKpVqxAVFQVPT0/ExcVh9+7dbba/evUqnnjiCYSHh0Or1aJPnz5IT0+3+XNdSWl1A04VmZZeiI/hejtE1H42j+lv3rwZ33//Pbp162axvXfv3jh//rxNx/rss8+QnJyM1atXIy4uDitXrsSkSZNw6tQphISEtGiv0+mQmJiIkJAQ/Pvf/0ZERATOnz+PgIAAW7vhUnaevQIhgH7hfgj21UKv10tdEhG5KJtDv6amxuIMv1lZWZl5xU1rrVixAvPmzcPcuXMBmO723bhxI9auXdvqip1r165FWVkZdu7caZ4qGhUVZWsXXM5P50x34d4U01niSojI1dkc+rfeeivWr1+Pl156CYBpmqbRaMQrr7yCCRMmWH0cnU6Hffv2ISUlxbxNqVQiISEBOTk5re7zzTffID4+Hk888QT+85//IDg4GPfffz8WLVp0zQe4NDQ0oKGhwfy6srISAKDX613ijFkIgR+bFlmLiwqwqNsV6reG3PoDyK9PcusPIK8+2dIHm0P/lVdewe233469e/dCp9PhL3/5C44dO4aysjLs2LHD6uOUlpbCYDAgNDTUYntoaChOnjzZ6j7nzp3Dli1b8MADDyA9PR1nzpzB448/Dr1ej9TU1Fb3SUtLw7Jly1ps37x5c6v/Y3E2RXXAxase8FAIVJ7ei/Rzv7yXkZEhXWEOILf+APLrk9z6A8ijT7W1tVa3tTn0Bw4ciJ9//hnvvPMOfH19UV1djT/84Q/mi6uOZDQaERISgvfffx8qlQojRozApUuX8Oqrr14z9FNSUpCcnGx+XVlZicjISEycOBF+fn4Ordce/rkrHzh4EsN7dMb0qaMAmP5Vz8jIQGJiYos7ol2R3PoDyK9PcusPIK8+NY9gWKNd6+n7+/vj+eefb8+uZkFBQVCpVCgqKrLYXlRUZH7w+m+Fh4dDrVZbDOX069cPhYWF0Ol00Ghari+v1WpbvdagVqtd4gudedI0VXNCbGiLel2lD9aSW38A+fVJbv0B5NEnW+q3OfRPnz6N//znP8jLy4NCoUBMTAymTZuG6Ohom46j0WgwYsQIZGZmYtq0aQBMZ/KZmZlISkpqdZ+bb74Zn3zyCYxGI5RK02zTn3/+GeHh4a0Gvqur0xmwO8/0PNzE/i1nMxER2cqmefppaWno378/Fi1ahP/7v//DF198gYULF6Jv37547bXXbP7w5ORkrFmzBh9//DFOnDiBxx57DDU1NebZPLNnz7a40PvYY4+hrKwMTz31FH7++Wds3LgRL7/8Mp544gmbP9sV7DxbCl2jEREBXugZzLWUiejGWX2mn5WVhcWLF2PJkiV46qmnEBgYCMA0VXPlypV49tlnMXr0aIwdO9bqD58xYwZKSkqwdOlSFBYWYujQodi0aZP54m5+fr75jB4AIiMj8f3332P+/PkYPHgwIiIi8NRTT2HRokVWf6Yr2XjYdLdzYv9QLr1ARHZhdeivXr0aDz/8MF544QWL7Z07d8aLL76IwsJCvPfeezaFPgAkJSVdczgnOzu7xbb4+Hj89NNPNn2GKxJCmOfnJ/QLvU5rIiLrWD28s3v37jYfkThr1iy3COOOkl9Wi8sV9VCrFBjeI0DqcohIJqwO/aKiojbvfo2OjkZhYaE9aiIA+86b1s7v39Uf3pp2TbIiImrB6tCvr69vc4aMWq2GTqezS1H0y1OyuMAaEdmTTaeQH3zwAXx8Wp9FUlVVZZeCyDSev71p/fxbuZQyEdmR1aHfvXt3rFmz5rpt6MadLalBcVUDNB5KjOgRKHU5RCQjVod+Xl6eA8ugX8tpmrUzonsgPNWtLyRHRNQe7X5cIjnO/qaLuKOjuZQyEdkXQ98J7Wo+0+fQDhHZGUPfyVwsN83P91AqMDKKoU9E9sXQdzI5Z01n+QMiOD+fiOyPoe9kdjRN1RzLqZpE5ADtCv2zZ89i8eLFmDlzJoqLTY/y++6773Ds2DG7FueODly4CgAYzvF8InIAm0N/69atGDRoEHbt2oUvv/wS1dXVAIBDhw5d8+lVZJ0LZbU4f6XWNJ7P0CciB7A59J999ln87//+LzIyMiyWZbjtttu44NoN2nve9MCUgRH+8PV07Sf5EJFzsjn0jxw5gunTp7fYHhISgtLSUrsU5a62nzZdxI3j/HwichCbQz8gIAAFBQUtth84cAARERF2KcodCSHMF3Fv7sWLuETkGDaH/n333YdFixahsLAQCoUCRqMRO3bswMKFCzF79mxH1OgW8stqUVhpWj9/VBTP9InIMWwO/ZdffhmxsbGIjIxEdXU1+vfvj7Fjx2LMmDFYvHixI2p0Cz82LaU8rHsgvDRcb4eIHMPmu380Gg3WrFmDJUuW4OjRo6iursawYcPQu3dvR9TnNvbmmS7icv18InIkm0N/+/btuOWWW9C9e3cupWxHhzg/n4g6gM3DO7fddhuio6Px3HPP4fjx446oye2U1+iQd6UWADA4wl/iaohIzmwO/cuXL2PBggXYunUrBg4ciKFDh+LVV1/FxYsXHVGfW9jbtJRyz+BOCOx07UdSEhHdKJtDPygoCElJSdixYwfOnj2Le+65Bx9//DGioqJw2223OaJG2Wu+KWt4dw7tEJFj3dCCa9HR0Xj22WexfPlyDBo0CFu3brVXXW7lwPmrAIA4XsQlIgdrd+jv2LEDjz/+OMLDw3H//fdj4MCB2Lhxoz1rcwsGo8CJgkoAwICufhJXQ0RyZ/PsnZSUFGzYsAGXL19GYmIi3nzzTdx1113w9vZ2RH2yd7KwElUNjfDReqB3iI/U5RCRzNkc+j/++COeeeYZ3HvvvQgK4nIBN2pf00XcoZEB8FDx8QZE5Fg2h/6OHTscUYfb+qnpebg3xXDpBSJyPKtC/5tvvsGUKVOgVqvxzTfftNn297//vV0KcxcH868CAEb0YOgTkeNZFfrTpk1DYWEhQkJCMG3atGu2UygUMBgM9qpN9sprdLhcUQ8AGBDBi7hE5HhWhb7RaGz1z3RjDjYtvRAT3Al+fGgKEXUAm68crl+/Hg0NDS2263Q6rF+/3i5FuYuThVUAgAFdufQCEXUMm0N/7ty5qKioaLG9qqoKc+fOtUtR7uJ0kSn0+3CqJhF1EJtDXwgBhULRYvvFixfh788zVls0n+n3CfOVuBIichdWT9kcNmwYFAoFFAoFbr/9dnh4/LKrwWBAbm4uJk+e7JAi5UjXaMSZ4moAQL8wXsQloo5hdeg3z9o5ePAgJk2aBB+fX4YkNBoNoqKi8Mc//tHuBcpVflkNdAYjOmlUiOzsJXU5ROQmrA791NRUAEBUVBRmzJgBT09PhxXlDn4uMp3l9wr1bXW4jIjIEWy+I3fOnDmOqMPtnG0a2ukZ3EniSojIndgc+gaDAW+88QY+//xz5OfnQ6fTWbxfVlZmt+Lk7GxJ05k+Z+4QUQeyefbOsmXLsGLFCsyYMQMVFRVITk7GH/7wByiVSrzwwgsOKFGecpsejxjdhWf6RNRxbA79f/3rX1izZg0WLFgADw8PzJw5Ex988AGWLl2Kn376yRE1yo4QAmea5ujzTJ+IOpLNoV9YWIhBgwYBAHx8fMw3av3ud7/jQ1SsVFTZgBqdASqlAlFBPNMnoo5jc+h369YNBQUFAICePXti8+bNAIA9e/ZAq9XatzqZah7P797ZG2quoU9EHcjmxJk+fToyMzMBAE8++SSWLFmC3r17Y/bs2XjwwQftXqAcNS+/0DOYQztE1LFsnr2zfPly859nzJiB7t27IycnB71798bUqVPtWpxcNS+/EMvlF4iog93w2EJ8fDySk5NvKPBXrVqFqKgoeHp6Ii4uDrt377Zqvw0bNkChULS5xr8zOldSAwDoHcozfSLqWFY/Octatj4567PPPkNycjJWr16NuLg4rFy5EpMmTcKpU6cQEhJyzf3y8vKwcOFC3HrrrTZ9njM4XczhHSKShtVPzrJGe56ctWLFCsybN8+8LPPq1auxceNGrF27Fs8++2yr+xgMBjzwwANYtmwZtm3bhqtXr9r0mVKqrNejvFYPAIjmzB0i6mA2PznLnnQ6Hfbt24eUlBTzNqVSiYSEBOTk5FxzvxdffBEhISF46KGHsG3btjY/o6GhweKhL5WVlQAAvV4PvV5/gz2w3dlC0+d36aSBRinaVUPzPlLU7why6w8gvz7JrT+AvPpkSx9svpBrT6WlpTAYDAgNDbXYHhoaipMnT7a6z/bt2/Hhhx/i4MGDVn1GWloali1b1mL75s2b4e3tbXPNN+rAFQUAFXwVDUhPT7+hY2VkZNinKCcht/4A8uuT3PoDyKNPtbW1Vre1OfRffPHFNt9funSprYe0WlVVFWbNmoU1a9YgKCjIqn1SUlKQnJxsfl1ZWYnIyEhMnDgRfn4dv479+a3ngJ/PYGjPrrjjjkHtOoZer0dGRgYSExOhVrv+s3Xl1h9Afn2SW38AefWpeQTDGjaH/ldffWXxWq/XIzc3Fx4eHujZs6dNoR8UFASVSoWioiKL7UVFRQgLC2vR/uzZs8jLy7OYKdQ89OTh4YFTp06hZ8+eFvtotdpWbxpTq9WSfKHPl9UDAHqG+N7w50vVB0eRW38A+fVJbv0B5NEnW+q3OfQPHDjQYltlZSX+9Kc/Yfr06TYdS6PRYMSIEcjMzDRfLDYajcjMzERSUlKL9rGxsThy5IjFtsWLF6OqqgpvvvkmIiMjbfp8KeRdMU3X5PILRCQFu4zp+/n5YdmyZZg6dSpmzZpl077JycmYM2cORo4cidGjR2PlypWoqakxz+aZPXs2IiIikJaWBk9PTwwcONBi/4CAAABosd1Z5ZeZxt6iuLomEUnAbhdyKyoqzIuv2WLGjBkoKSnB0qVLUVhYiKFDh2LTpk3mi7v5+flQKuWxPk293oCSKtNMoohAPiKRiDqezaH/1ltvWbwWQqCgoAD/+Mc/MGXKlHYVkZSU1OpwDgBkZ2e3ue+6deva9ZlSuHS1DgDgrVEh0Nu1xxCJyDXZHPpvvPGGxWulUong4GDMmTPHYr49tZTf9OCU7p29+VxcIpKEzaGfm5vriDrcwrnSpou4HM8nIonIY7DcReQ3zdyJ5sPQiUgiNp/p19fX4+2330ZWVhaKi4tbLNGwf/9+uxUnNwUVpjn6Xf09Ja6EiNyVzaH/0EMPYfPmzbj77rsxevRojk3boLDSFPqhfgx9IpKGzaH/7bffIj09HTfffLMj6pG1y02zd7oGcLomEUnD5jH9iIgI+PryiU+2qmloRGm1DgAQ2bnjF3ojIgLaEfqvv/46Fi1ahPPnzzuiHtkqqDCd5ftqPeDvxTn6RCQNm4d3Ro4cifr6esTExMDb27vFQj9lZWV2K05OLl1tuojLoR0ikpDNoT9z5kxcunQJL7/8MkJDQ3kh10oXmtbciezM0Cci6dgc+jt37kROTg6GDBniiHpk60K5KfS7BXI8n4ikY/OYfmxsLOrq6hxRi6xdbhre6caF1ohIQjaH/vLly7FgwQJkZ2fjypUrqKystPhFrSuq4Bx9IpKezcM7kydPBgDcfvvtFtuFEFAoFDAYDPapTGZ4YxYROQObQz8rK8sRdciaEALFVc2h3/LRjUREHcXm0B83bpwj6pC1yvpG1OtNaxSF+PJMn4ikY3Po//jjj22+P3bs2HYXI1fFTUM7fp4e8NKoJK6GiNyZzaE/fvz4Ftt+PVefY/otNa+uGcbVNYlIYjbP3ikvL7f4VVxcjE2bNmHUqFHYvHmzI2p0eZyjT0TOwuYzfX9//xbbEhMTodFokJycjH379tmlMDkpMC/BwDN9IpKW3Z6cFRoailOnTtnrcLLSPHMnjNM1iUhiNp/pHz582OK1EAIFBQVYvnw5hg4daq+6ZOVK05LKXXw4XZOIpGVz6A8dOhQKhQJCCIvtN910E9auXWu3wuSktLoBANClk0biSojI3dkc+rm5uRavlUolgoOD4enJoYtrKakyhX6wL8/0iUhaNod+jx49HFGHbBmNAsVNoc8pm0QkNasv5G7ZsgX9+/dvdVG1iooKDBgwANu2bbNrcXJwpUaHRqOAQgEEcUyfiCRmdeivXLkS8+bNg5+fX4v3/P398ec//xkrVqywa3FyUNh0Y1aQjxZqld0mSxERtYvVKXTo0CHzCputmThxIufot6L52bhdObRDRE7A6tAvKipq8TzcX/Pw8EBJSYldipKToqbxfC6pTETOwOrQj4iIwNGjR6/5/uHDhxEeHm6XouSkeeZOEGfuEJETsDr077jjDixZsgT19fUt3qurq0Nqaip+97vf2bU4OSi4ahreCeeZPhE5AaunbC5evBhffvkl+vTpg6SkJPTt2xcAcPLkSaxatQoGgwHPP/+8wwp1VSVNN2aF8OEpROQErA790NBQ7Ny5E4899hhSUlLMd+QqFApMmjQJq1atQmhoqMMKdVVFlc2hzzN9IpKeTTdn9ejRA+np6SgvL8eZM2cghEDv3r0RGBjoqPpcXvMDVEL5xCwicgI235ELAIGBgRg1apS9a5GdRoMRZbWmxdY4vENEzoB3CznQlRodhACUCiDQm4utEZH0GPoO1Dxds4uPFiql4jqtiYgcj6HvQFdqmtbR55LKROQkGPoOxCWVicjZMPQdqKhp5k4IZ+4QkZNg6DtQ85k+Z+4QkbNg6DvQL2f6DH0icg4MfQdqDn2usElEzoKh70DNSzAw9InIWTD0HcT0bFzTmT6fjUtEzsIpQn/VqlWIioqCp6cn4uLisHv37mu2XbNmDW699VYEBgYiMDAQCQkJbbaXypUaHfQG07NxOaZPRM5C8tD/7LPPkJycjNTUVOzfvx9DhgzBpEmTUFxc3Gr77OxszJw5E1lZWcjJyUFkZCQmTpyIS5cudXDlbWt+TGIwn41LRE5E8jRasWIF5s2bh7lz56J///5YvXo1vL29sXbt2lbb/+tf/8Ljjz+OoUOHIjY2Fh988AGMRiMyMzM7uPK2NT8QPZxDO0TkRNq1yqa96HQ67Nu3DykpKeZtSqUSCQkJyMnJseoYtbW10Ov16Ny5c6vvNzQ0oKGhwfy6srISAKDX66HX62+g+rZdLq8BYBrasffnNB/PkfV3JLn1B5Bfn+TWH0BefbKlD5KGfmlpKQwGQ4uHr4SGhuLkyZNWHWPRokXo2rUrEhISWn0/LS0Ny5Yta7F98+bN8Pb2tr1oK+3MVwJQor6sEOnp6Q75jIyMDIccVypy6w8gvz7JrT+APPpUW1trdVtJQ/9GLV++HBs2bEB2djY8PVsfRklJSUFycrL5dWVlpfk6gJ+fn8Nq2/bVMeDSJYwY2Ad3jI+x67H1ej0yMjKQmJgItVpt12NLQW79AeTXJ7n1B5BXn5pHMKwhaegHBQVBpVKhqKjIYntRURHCwsLa3Pe1117D8uXL8cMPP2Dw4MHXbKfVaqHVtpw9o1arHfqFLmpagqFroLfDPsfRfehocusPIL8+ya0/gDz6ZEv9kl7I1Wg0GDFihMVF2OaLsvHx8dfc75VXXsFLL72ETZs2YeTIkR1Rqs2a78YN441ZROREJB/eSU5Oxpw5czBy5EiMHj0aK1euRE1NDebOnQsAmD17NiIiIpCWlgYA+Nvf/oalS5fik08+QVRUFAoLCwEAPj4+8PHxkawfv9U8e4c3ZhGRM5E89GfMmIGSkhIsXboUhYWFGDp0KDZt2mS+uJufnw+l8pf/kLz33nvQ6XS4++67LY6TmpqKF154oSNLv6aqej0q6xsBcMomETkXyUMfAJKSkpCUlNTqe9nZ2Rav8/LyHF/QDWpec8fX0wO+nq49VkhE8iL5zVlyZH5ilg+XXyAi58LQd4Dmhdb4mEQicjYMfQe4UGa6UaJboONu/iIiag+GvgNcLDctthbZ2UviSoiILDH0HeBy03TNrgEMfSJyLgx9B7h81XSmH8HQJyInw9B3AN6YRUTOiqFvZ5X1elQ38MYsInJODH07u9R0ETfAWw1vjVPc+0ZEZMbQt7Pm0O8WyPF8InI+DH07u1DeNEc/gHP0icj5MPTt7CLP9InIiTH07aygomm6JkOfiJwQQ9/OmqdrcuYOETkjhr6dXb7aPEefZ/pE5HwY+nbU0GhAUdMKmxzTJyJnxNC3o8KKeggBaD2U6NJJI3U5REQtMPTt6NfLLygUComrISJqiaFvR5eaFlrryvF8InJSDH074t24ROTsGPp2dLlpjn44l1QmIifF0Lej3NIaAEBUFy7BQETOiaFvRxfKTGf63Tsz9InIOTH07aRebzAP70QFdZK4GiKi1jH07SS3tAZCAL6eHpyjT0ROi6FvJ2eKqwEAPYN9OEefiJwWQ99OThdVAQBiw3wlroSI6NoY+nZyuulMv1eIj8SVEBFdG0PfTo4XVAIA+oX7SVwJEdG1MfTtQNdoNN+NGxPMmTtE5LwY+nZwtqQajUYBP08PhPnx4SlE5LwY+nbwc9NF3F4hnLlDRM6NoW8HJwpMod+/K8fzici5MfTt4NjlCgBAbBhDn4icG0P/BgkhcOyyaebOoAh/iashImobQ/8GXSirQ1mNDhqVErHhvDGLiJwbQ/8GHS8wDe30DvWB1kMlcTVERG1j6N+ggxdMoc+hHSJyBQz9G3Tk0lUAwJDIAEnrICKyBkP/BugNRhzIvwoAGNyNZ/pE5PwY+jfg+OVK1OoM8PdSox+naxKRC2Do34Btp0sAACN6BEKp5J24ROT8GPo34PtjRQCASQNCJa6EiMg6DP12ulBWiyOXTDN3JvQNkbgaIiLrMPTbafXWswCAW3oFIYQraxKRi3CK0F+1ahWioqLg6emJuLg47N69u832X3zxBWJjY+Hp6YlBgwYhPT29gyo1qajV4+sDlwAA88bGdOhnExHdCMlD/7PPPkNycjJSU1Oxf/9+DBkyBJMmTUJxcXGr7Xfu3ImZM2fioYcewoEDBzBt2jRMmzYNR48e7ZB6hRD4343HUaMzoE+oD8b2DuqQzyUisgfJQ3/FihWYN28e5s6di/79+2P16tXw9vbG2rVrW23/5ptvYvLkyXjmmWfQr18/vPTSSxg+fDjeeecdh9Z5oqASX+6/iFkf7sYX+y4CAFKnDuD6+UTkUjyk/HCdTod9+/YhJSXFvE2pVCIhIQE5OTmt7pOTk4Pk5GSLbZMmTcLXX3/davuGhgY0NDSYX1dWmlbE1Ov10Ov1Vtf68sbj2HbmCgBAoQCem9IXo3v423QMe2n+TCk+2xHk1h9Afn2SW38AefXJlj5IGvqlpaUwGAwIDbWc8hgaGoqTJ0+2uk9hYWGr7QsLC1ttn5aWhmXLlrXYvnnzZnh7e1tdq3e9Ej19FejaSWBMiBEh5ceQnn7M6v0dISMjQ9LPtze59QeQX5/k1h9AHn2qra21uq2kod8RUlJSLP5nUFlZicjISEycOBF+ftbfRXuHI4prJ71ej4yMDCQmJkKtVktdzg2TW38A+fVJbv0B5NWn5hEMa0ga+kFBQVCpVCgqKrLYXlRUhLCwsFb3CQsLs6m9VquFVqttsV2tVrv8F1oOffg1ufUHkF+f5NYfQB59sqV+SS/kajQajBgxApmZmeZtRqMRmZmZiI+Pb3Wf+Ph4i/aA6b9n12pPRES/kHx4Jzk5GXPmzMHIkSMxevRorFy5EjU1NZg7dy4AYPbs2YiIiEBaWhoA4KmnnsK4cePw+uuv484778SGDRuwd+9evP/++1J2g4jIJUge+jNmzEBJSQmWLl2KwsJCDB06FJs2bTJfrM3Pz4dS+ct/SMaMGYNPPvkEixcvxnPPPYfevXvj66+/xsCBA6XqAhGRy5A89AEgKSkJSUlJrb6XnZ3dYts999yDe+65x8FVERHJj+Q3ZxERUcdh6BMRuRGGPhGRG2HoExG5EYY+EZEbYegTEbkRp5iy2ZGEEABsW6vC2ej1etTW1qKystLlbx8H5NcfQH59klt/AHn1qTnPmvOtLW4X+lVVVQCAyMhIiSshIrKvqqoq+Pv7t9lGIaz5p0FGjEYjLl++DF9fX5d9AErzSqEXLlywaaVQZyW3/gDy65Pc+gPIq09CCFRVVaFr164WKxi0xu3O9JVKJbp16yZ1GXbh5+fn8t+svya3/gDy65Pc+gPIp0/XO8Nvxgu5RERuhKFPRORGGPouSKvVIjU1tdWHw7giufUHkF+f5NYfQJ59sobbXcglInJnPNMnInIjDH0iIjfC0CciciMMfSIiN8LQdzGrVq1CVFQUPD09ERcXh927d0tdEgDghRdegEKhsPgVGxtrfr++vh5PPPEEunTpAh8fH/zxj39EUVGRxTHy8/Nx5513wtvbGyEhIXjmmWfQ2Nho0SY7OxvDhw+HVqtFr169sG7dOrvU/+OPP2Lq1Kno2rUrFAoFvv76a4v3hRBYunQpwsPD4eXlhYSEBJw+fdqiTVlZGR544AH4+fkhICAADz30EKqrqy3aHD58GLfeeis8PT0RGRmJV155pUUtX3zxBWJjY+Hp6YlBgwYhPT3dIX3605/+1OJrNnnyZKftU1paGkaNGgVfX1+EhIRg2rRpOHXqlEWbjvw+c9afxesS5DI2bNggNBqNWLt2rTh27JiYN2+eCAgIEEVFRVKXJlJTU8WAAQNEQUGB+VdJSYn5/UcffVRERkaKzMxMsXfvXnHTTTeJMWPGmN9vbGwUAwcOFAkJCeLAgQMiPT1dBAUFiZSUFHObc+fOCW9vb5GcnCyOHz8u3n77baFSqcSmTZtuuP709HTx/PPPiy+//FIAEF999ZXF+8uXLxf+/v7i66+/FocOHRK///3vRXR0tKirqzO3mTx5shgyZIj46aefxLZt20SvXr3EzJkzze9XVFSI0NBQ8cADD4ijR4+KTz/9VHh5eYm///3v5jY7duwQKpVKvPLKK+L48eNi8eLFQq1WiyNHjti9T3PmzBGTJ0+2+JqVlZVZtHGmPk2aNEl89NFH4ujRo+LgwYPijjvuEN27dxfV1dXmNh31febMP4vXw9B3IaNHjxZPPPGE+bXBYBBdu3YVaWlpElZlkpqaKoYMGdLqe1evXhVqtVp88cUX5m0nTpwQAEROTo4QwhRQSqVSFBYWmtu89957ws/PTzQ0NAghhPjLX/4iBgwYYHHsGTNmiEmTJtm1L78NSKPRKMLCwsSrr75q0SetVis+/fRTIYQQx48fFwDEnj17zG2+++47oVAoxKVLl4QQQrz77rsiMDDQ3B8hhFi0aJHo27ev+fW9994r7rzzTot64uLixJ///Ge79kkIU+jfdddd19zH2ftUXFwsAIitW7cKITr2+8yZfxavh8M7LkKn02Hfvn1ISEgwb1MqlUhISEBOTo6Elf3i9OnT6Nq1K2JiYvDAAw8gPz8fALBv3z7o9XqL2mNjY9G9e3dz7Tk5ORg0aBBCQ0PNbSZNmoTKykocO3bM3ObXx2hu4+j+5+bmorCw0OKz/f39ERcXZ1F/QEAARo4caW6TkJAApVKJXbt2mduMHTsWGo3Gov5Tp06hvLzc3KYj+5idnY2QkBD07dsXjz32GK5cuWJ+z9n7VFFRAQDo3LkzgI77PnOFn8W2MPRdRGlpKQwGg8U3KwCEhoaisLBQoqp+ERcXh3Xr1mHTpk147733kJubi1tvvRVVVVUoLCyERqNBQECAxT6/rr2wsLDVvjW/11abyspK1NXVOahnv3x+W3/3hYWFCAkJsXjfw8MDnTt3tksfHfE1njx5MtavX4/MzEz87W9/w9atWzFlyhQYDAan75PRaMTTTz+Nm2++GQMHDjR/Tkd8nzn7z+L1uN0qm+QYU6ZMMf958ODBiIuLQ48ePfD555/Dy8tLwsroWu677z7znwcNGoTBgwejZ8+eyM7Oxu233y5hZdf3xBNP4OjRo9i+fbvUpbgcnum7iKCgIKhUqhYzEYqKihAWFiZRVdcWEBCAPn364MyZMwgLC4NOp8PVq1ct2vy69rCwsFb71vxeW238/Pwc+g9L8+e39XcfFhaG4uJii/cbGxtRVlZmlz52xNc4JiYGQUFBOHPmjLkWZ+xTUlISvv32W2RlZVksk95R32eu9rP4Wwx9F6HRaDBixAhkZmaatxmNRmRmZiI+Pl7CylpXXV2Ns2fPIjw8HCNGjIBarbao/dSpU8jPzzfXHh8fjyNHjliETEZGBvz8/NC/f39zm18fo7mNo/sfHR2NsLAwi8+urKzErl27LOq/evUq9u3bZ26zZcsWGI1GxMXFmdv8+OOP0Ov1FvX37dsXgYGB5jZS9BEALl68iCtXriA8PNxcizP1SQiBpKQkfPXVV9iyZQuio6Mt3u+o7zNX+1lsQeoryWS9DRs2CK1WK9atWyeOHz8uHnnkEREQEGAxE0EqCxYsENnZ2SI3N1fs2LFDJCQkiKCgIFFcXCyEME2l6969u9iyZYvYu3eviI+PF/Hx8eb9m6fSTZw4URw8eFBs2rRJBAcHtzqV7plnnhEnTpwQq1atstuUzaqqKnHgwAFx4MABAUCsWLFCHDhwQJw/f14IYZqyGRAQIP7zn/+Iw4cPi7vuuqvVKZvDhg0Tu3btEtu3bxe9e/e2mN549epVERoaKmbNmiWOHj0qNmzYILy9vVtMb/Tw8BCvvfaaOHHihEhNTW33lM22+lRVVSUWLlwocnJyRG5urvjhhx/E8OHDRe/evUV9fb1T9umxxx4T/v7+Ijs722KaaW1trblNR32fOfPP4vUw9F3M22+/Lbp37y40Go0YPXq0+Omnn6QuSQhhmtIWHh4uNBqNiIiIEDNmzBBnzpwxv19XVycef/xxERgYKLy9vcX06dNFQUGBxTHy8vLElClThJeXlwgKChILFiwQer3eok1WVpYYOnSo0Gg0IiYmRnz00Ud2qT8rK0sAaPFrzpw5QgjTtM0lS5aI0NBQodVqxe233y5OnTplcYwrV66ImTNnCh8fH+Hn5yfmzp0rqqqqLNocOnRI3HLLLUKr1YqIiAixfPnyFrV8/vnnok+fPkKj0YgBAwaIjRs32r1PtbW1YuLEiSI4OFio1WrRo0cPMW/evBah5Ux9aq0vACy+Bzry+8xZfxavh0srExG5EY7pExG5EYY+EZEbYegTEbkRhj4RkRth6BMRuRGGPhGRG2HoExG5EYY+kZ1lZ2dDoVC0WAOmLS+88AKGDh3qsJqImjH0ya2tXr0avr6+Fo/Lq66uhlqtxvjx4y3aNof52bNn2zzmmDFjUFBQAH9/f7vWOn78eDz99NN2PSa5H4Y+ubUJEyaguroae/fuNW/btm0bwsLCsGvXLtTX15u3Z2VloXv37ujZs2ebx9RoNAgLC4NCoXBY3UTtxdAnt9a3b1+Eh4cjOzvbvC07Oxt33XUXoqOj8dNPP1lsnzBhAoxGI9LS0hAdHQ0vLy8MGTIE//73vy3a/XZ4Z82aNYiMjIS3tzemT5+OFStWtHjYBwD84x//QFRUFPz9/XHfffehqqoKgOkh5lu3bsWbb75pfoh5Xl6evf86yA0w9MntTZgwAVlZWebXWVlZGD9+PMaNG2feXldXh127dmHChAlIS0vD+vXrsXr1ahw7dgzz58/H//zP/2Dr1q2tHn/Hjh149NFH8dRTT+HgwYNITEzEX//61xbtzp49i6+//hrffvstvv32W2zduhXLly8HALz55puIj4/HvHnzUFBQgIKCAkRGRjrgb4Pkjk/OIrc3YcIEPP3002hsbERdXR0OHDiAcePGQa/XY/Xq1QBMz01taGjA+PHj0b9/f/zwww/mtdNjYmKwfft2/P3vf8e4ceNaHP/tt9/GlClTsHDhQgBAnz59sHPnTnz77bcW7YxGI9atWwdfX18AwKxZs5CZmYm//vWv8Pf3h0ajgbe3t0s8qIOcF0Of3N748eNRU1ODPXv2oLy8HH369EFwcDDGjRuHuXPnor6+HtnZ2YiJiUF1dTVqa2uRmJhocQydTodhw4a1evxTp05h+vTpFttGjx7dIvSjoqLMgQ8A4eHhLZ5cRXSjGPrk9nr16oVu3bohKysL5eXl5rP1rl27IjIyEjt37kRWVhZuu+02VFdXAwA2btyIiIgIi+NotdobqkOtVlu8VigUMBqNN3RMot9i6BPBNMSTnZ2N8vJyPPPMM+btY8eOxXfffYfdu3fjscceQ//+/aHVapGfn9/qUE5r+vbtiz179lhs++1ra2g0GhgMBpv3I/o1hj4RTKH/xBNPQK/XW4T5uHHjkJSUBJ1OhwkTJsDX1xcLFy7E/PnzYTQaccstt6CiogI7duyAn58f5syZ0+LYTz75JMaOHYsVK1Zg6tSp2LJlC7777jubp3RGRUVh165dyMvLg4+PDzp37gylknMxyDb8jiGCKfTr6urQq1cvhIaGmrePGzcOVVVV5qmdAPDSSy9hyZIlSEtLQ79+/TB58mRs3LixxYO6m918881YvXo1VqxYgSFDhmDTpk2YP38+PD09bapx4cKFUKlU6N+/P4KDg5Gfn9/+DpPb4uMSiSQwb948nDx5Etu2bZO6FHIzHN4h6gCvvfYaEhMT0alTJ3z33Xf4+OOP8e6770pdFrkhnukTdYB7770X2dnZqKqqQkxMDJ588kk8+uijUpdFboihT0TkRnghl4jIjTD0iYjcCEOfiMiNMPSJiNwIQ5+IyI0w9ImI3AhDn4jIjTD0iYjcCEOfiMiN/H9Cri4JPFdI7wAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "df = pd.read_csv('output/scenic/scenic.csv')\n", - "df # default" + "plot_cumulative_density(np.sum(X_tran, axis=1))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "# Evaluate priors" + "# Pending runs " ] }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "# !pip install loompy" + "RUN_ID=\"benchmark_donor_0_default\" -> grnboost2, check how this compares to when there was 5000 genes \n" ] }, { - "cell_type": "code", - "execution_count": 1, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "import pandas as pd \n", - "df = pd.read_csv('output/scglue_ext/draft_grn.csv')" + "# Test analysis" ] }, { - "cell_type": "code", - "execution_count": 7, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAGdCAYAAAD+JxxnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAuN0lEQVR4nO3df3RUdX7/8VcSkkn4MRMDJkOWAHFRMcgPCSZM1W2pKaNNPVLSbaSUjYBa6ECFVBH61eB6tg3F7hEsv9bSY+hZf0DOqboQhGbDAsclAoalAmKKFhtcnABqZiCFBJLP9489uWUASQbBgXyej3PuOdz7ed/PfTvjPfM6N/fOxBljjAAAACwUH+sGAAAAYoUgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwVo9YN3A9a29v19GjR9WnTx/FxcXFuh0AANAFxhidPHlSmZmZio+//DUfgtBlHD16VFlZWbFuAwAAXIEjR45owIABl60hCF1Gnz59JP3uhXS73THuBgAAdEU4HFZWVpbzOX45BKHL6PhzmNvtJggBAHCD6cptLdwsDQAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGCtHrFuwGaD51dFrH+2qDBGnQAAYCeuCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtaIKQoMHD1ZcXNxFSyAQkCSdOXNGgUBAffv2Ve/evVVUVKTGxsaIORoaGlRYWKiePXsqPT1dTz/9tM6dOxdRs3XrVo0ePVoul0tDhgxRRUXFRb0sX75cgwcPVnJysvLz87Vr166I8a70AgAA7BZVENq9e7e++OILZ6murpYk/fCHP5QkzZ07V+vXr1dlZaW2bdumo0ePauLEic7+bW1tKiwsVGtrq3bs2KE1a9aooqJCZWVlTs3hw4dVWFiocePGae/evZozZ44ee+wxbd682alZu3atSktLtXDhQu3Zs0cjR46U3+/XsWPHnJrOegEAAJD5Fp588knz/e9/37S3t5umpiaTmJhoKisrnfGDBw8aSaa2ttYYY8zGjRtNfHy8CQaDTs3KlSuN2+02LS0txhhj5s2bZ4YNGxZxnOLiYuP3+531vLw8EwgEnPW2tjaTmZlpysvLjTGmS710RSgUMpJMKBTq8j7RGPTMhogFAAB8e9F8fl/xPUKtra36+c9/rmnTpikuLk51dXU6e/asCgoKnJqhQ4dq4MCBqq2tlSTV1tZq+PDhysjIcGr8fr/C4bAOHDjg1Jw/R0dNxxytra2qq6uLqImPj1dBQYFT05VeLqWlpUXhcDhiAQAA3dcVB6G3335bTU1NevTRRyVJwWBQSUlJSk1NjajLyMhQMBh0as4PQR3jHWOXqwmHwzp9+rROnDihtra2S9acP0dnvVxKeXm5PB6Ps2RlZXX+QgAAgBvWFQehf/3Xf9WDDz6ozMzMq9lPTC1YsEChUMhZjhw5EuuWAADANdTjSnb6n//5H/3yl7/Uv//7vzvbvF6vWltb1dTUFHElprGxUV6v16m58Omujie5zq+58OmuxsZGud1upaSkKCEhQQkJCZesOX+Oznq5FJfLJZfL1cVXAQAA3Oiu6IrQq6++qvT0dBUWFjrbcnNzlZiYqJqaGmdbfX29Ghoa5PP5JEk+n0/79u2LeLqrurpabrdbOTk5Ts35c3TUdMyRlJSk3NzciJr29nbV1NQ4NV3pBQAAIOorQu3t7Xr11VdVUlKiHj3+b3ePx6Pp06ertLRUaWlpcrvdmj17tnw+n8aOHStJGj9+vHJycjRlyhQtXrxYwWBQzz77rAKBgHMlZsaMGVq2bJnmzZunadOmacuWLVq3bp2qqqqcY5WWlqqkpERjxoxRXl6elixZoubmZk2dOrXLvQAAAEQdhH75y1+qoaFB06ZNu2jspZdeUnx8vIqKitTS0iK/368VK1Y44wkJCdqwYYNmzpwpn8+nXr16qaSkRC+88IJTk52draqqKs2dO1dLly7VgAEDtHr1avn9fqemuLhYx48fV1lZmYLBoEaNGqVNmzZF3EDdWS8AAABxxhgT6yauV+FwWB6PR6FQSG63+6rPP3h+VcT6Z4sKv6ESAAB0VTSf3/zWGAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrRR2Efvvb3+ov//Iv1bdvX6WkpGj48OH64IMPnHFjjMrKytS/f3+lpKSooKBAhw4dipjjq6++0uTJk+V2u5Wamqrp06fr1KlTETUffvih7rvvPiUnJysrK0uLFy++qJfKykoNHTpUycnJGj58uDZu3Bgx3pVeAACAvaIKQl9//bXuueceJSYm6t1339VHH32kn/70p7rpppucmsWLF+vll1/WqlWrtHPnTvXq1Ut+v19nzpxxaiZPnqwDBw6ourpaGzZs0Pbt2/XEE0844+FwWOPHj9egQYNUV1enF198Uc8//7xeeeUVp2bHjh2aNGmSpk+frt/85jeaMGGCJkyYoP3790fVCwAAsJiJwjPPPGPuvffebxxvb283Xq/XvPjii862pqYm43K5zBtvvGGMMeajjz4ykszu3budmnfffdfExcWZ3/72t8YYY1asWGFuuukm09LSEnHs22+/3Vn/8z//c1NYWBhx/Pz8fPNXf/VXXe6lM6FQyEgyoVCoS/XRGvTMhogFAAB8e9F8fkd1RegXv/iFxowZox/+8IdKT0/XXXfdpX/5l39xxg8fPqxgMKiCggJnm8fjUX5+vmprayVJtbW1Sk1N1ZgxY5yagoICxcfHa+fOnU7ND37wAyUlJTk1fr9f9fX1+vrrr52a84/TUdNxnK70cqGWlhaFw+GIBQAAdF9RBaH//u//1sqVK3Xrrbdq8+bNmjlzpv7mb/5Ga9askSQFg0FJUkZGRsR+GRkZzlgwGFR6enrEeI8ePZSWlhZRc6k5zj/GN9WcP95ZLxcqLy+Xx+NxlqysrM5eEgAAcAOLKgi1t7dr9OjR+od/+AfdddddeuKJJ/T4449r1apV16q/79SCBQsUCoWc5ciRI7FuCQAAXENRBaH+/fsrJycnYtsdd9yhhoYGSZLX65UkNTY2RtQ0NjY6Y16vV8eOHYsYP3funL766quImkvNcf4xvqnm/PHOermQy+WS2+2OWAAAQPcVVRC65557VF9fH7Htv/7rvzRo0CBJUnZ2trxer2pqapzxcDisnTt3yufzSZJ8Pp+amppUV1fn1GzZskXt7e3Kz893arZv366zZ886NdXV1br99tudJ9R8Pl/EcTpqOo7TlV4AAIDlorkLe9euXaZHjx7m7//+782hQ4fMa6+9Znr27Gl+/vOfOzWLFi0yqamp5p133jEffvihefjhh012drY5ffq0U/PAAw+Yu+66y+zcudO899575tZbbzWTJk1yxpuamkxGRoaZMmWK2b9/v3nzzTdNz549zc9+9jOn5te//rXp0aOH+ad/+idz8OBBs3DhQpOYmGj27dsXVS+Xw1NjAADceKL5/I4qCBljzPr1682dd95pXC6XGTp0qHnllVcixtvb281zzz1nMjIyjMvlMvfff7+pr6+PqPnyyy/NpEmTTO/evY3b7TZTp041J0+ejKj5z//8T3Pvvfcal8tlvve975lFixZd1Mu6devMbbfdZpKSksywYcNMVVVV1L1cDkEIAIAbTzSf33HGGBPba1LXr3A4LI/Ho1AodE3uFxo8vypi/bNFhVf9GAAA2Caaz29+awwAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtaIKQs8//7zi4uIilqFDhzrjZ86cUSAQUN++fdW7d28VFRWpsbExYo6GhgYVFhaqZ8+eSk9P19NPP61z585F1GzdulWjR4+Wy+XSkCFDVFFRcVEvy5cv1+DBg5WcnKz8/Hzt2rUrYrwrvQAAALtFfUVo2LBh+uKLL5zlvffec8bmzp2r9evXq7KyUtu2bdPRo0c1ceJEZ7ytrU2FhYVqbW3Vjh07tGbNGlVUVKisrMypOXz4sAoLCzVu3Djt3btXc+bM0WOPPabNmzc7NWvXrlVpaakWLlyoPXv2aOTIkfL7/Tp27FiXewEAAJCJwsKFC83IkSMvOdbU1GQSExNNZWWls+3gwYNGkqmtrTXGGLNx40YTHx9vgsGgU7Ny5UrjdrtNS0uLMcaYefPmmWHDhkXMXVxcbPx+v7Oel5dnAoGAs97W1mYyMzNNeXl5l3vpilAoZCSZUCjU5X2iMeiZDRELAAD49qL5/I76itChQ4eUmZmpW265RZMnT1ZDQ4Mkqa6uTmfPnlVBQYFTO3ToUA0cOFC1tbWSpNraWg0fPlwZGRlOjd/vVzgc1oEDB5ya8+foqOmYo7W1VXV1dRE18fHxKigocGq60sultLS0KBwORywAAKD7iioI5efnq6KiQps2bdLKlSt1+PBh3XfffTp58qSCwaCSkpKUmpoasU9GRoaCwaAkKRgMRoSgjvGOscvVhMNhnT59WidOnFBbW9sla86fo7NeLqW8vFwej8dZsrKyuvbCAACAG1KPaIoffPBB598jRoxQfn6+Bg0apHXr1iklJeWqN/ddW7BggUpLS531cDhMGAIAoBv7Vo/Pp6am6rbbbtMnn3wir9er1tZWNTU1RdQ0NjbK6/VKkrxe70VPbnWsd1bjdruVkpKifv36KSEh4ZI158/RWS+X4nK55Ha7IxYAANB9fasgdOrUKX366afq37+/cnNzlZiYqJqaGme8vr5eDQ0N8vl8kiSfz6d9+/ZFPN1VXV0tt9utnJwcp+b8OTpqOuZISkpSbm5uRE17e7tqamqcmq70AgAAENWfxp566ik99NBDGjRokI4ePaqFCxcqISFBkyZNksfj0fTp01VaWqq0tDS53W7Nnj1bPp9PY8eOlSSNHz9eOTk5mjJlihYvXqxgMKhnn31WgUBALpdLkjRjxgwtW7ZM8+bN07Rp07RlyxatW7dOVVVVTh+lpaUqKSnRmDFjlJeXpyVLlqi5uVlTp06VpC71AgAAEFUQ+vzzzzVp0iR9+eWXuvnmm3Xvvffq/fff18033yxJeumllxQfH6+ioiK1tLTI7/drxYoVzv4JCQnasGGDZs6cKZ/Pp169eqmkpEQvvPCCU5Odna2qqirNnTtXS5cu1YABA7R69Wr5/X6npri4WMePH1dZWZmCwaBGjRqlTZs2RdxA3VkvAAAAccYYE+smrlfhcFgej0ehUOia3C80eH5VxPpniwqv+jEAALBNNJ/f/NYYAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGt9qyC0aNEixcXFac6cOc62M2fOKBAIqG/fvurdu7eKiorU2NgYsV9DQ4MKCwvVs2dPpaen6+mnn9a5c+ciarZu3arRo0fL5XJpyJAhqqiouOj4y5cv1+DBg5WcnKz8/Hzt2rUrYrwrvQAAAHtdcRDavXu3fvazn2nEiBER2+fOnav169ersrJS27Zt09GjRzVx4kRnvK2tTYWFhWptbdWOHTu0Zs0aVVRUqKyszKk5fPiwCgsLNW7cOO3du1dz5szRY489ps2bNzs1a9euVWlpqRYuXKg9e/Zo5MiR8vv9OnbsWJd7AQAAljNX4OTJk+bWW2811dXV5vd///fNk08+aYwxpqmpySQmJprKykqn9uDBg0aSqa2tNcYYs3HjRhMfH2+CwaBTs3LlSuN2u01LS4sxxph58+aZYcOGRRyzuLjY+P1+Zz0vL88EAgFnva2tzWRmZpry8vIu99KZUChkJJlQKNSl+mgNemZDxAIAAL69aD6/r+iKUCAQUGFhoQoKCiK219XV6ezZsxHbhw4dqoEDB6q2tlaSVFtbq+HDhysjI8Op8fv9CofDOnDggFNz4dx+v9+Zo7W1VXV1dRE18fHxKigocGq60gsAALBbj2h3ePPNN7Vnzx7t3r37orFgMKikpCSlpqZGbM/IyFAwGHRqzg9BHeMdY5erCYfDOn36tL7++mu1tbVdsubjjz/uci8XamlpUUtLi7MeDocvWQcAALqHqK4IHTlyRE8++aRee+01JScnX6ueYqa8vFwej8dZsrKyYt0SAAC4hqIKQnV1dTp27JhGjx6tHj16qEePHtq2bZtefvll9ejRQxkZGWptbVVTU1PEfo2NjfJ6vZIkr9d70ZNbHeud1bjdbqWkpKhfv35KSEi4ZM35c3TWy4UWLFigUCjkLEeOHOn6iwMAAG44UQWh+++/X/v27dPevXudZcyYMZo8ebLz78TERNXU1Dj71NfXq6GhQT6fT5Lk8/m0b9++iKe7qqur5Xa7lZOT49ScP0dHTcccSUlJys3Njahpb29XTU2NU5Obm9tpLxdyuVxyu90RCwAA6L6iukeoT58+uvPOOyO29erVS3379nW2T58+XaWlpUpLS5Pb7dbs2bPl8/k0duxYSdL48eOVk5OjKVOmaPHixQoGg3r22WcVCATkcrkkSTNmzNCyZcs0b948TZs2TVu2bNG6detUVVXlHLe0tFQlJSUaM2aM8vLytGTJEjU3N2vq1KmSJI/H02kvAADAblHfLN2Zl156SfHx8SoqKlJLS4v8fr9WrFjhjCckJGjDhg2aOXOmfD6fevXqpZKSEr3wwgtOTXZ2tqqqqjR37lwtXbpUAwYM0OrVq+X3+52a4uJiHT9+XGVlZQoGgxo1apQ2bdoUcQN1Z70AAAC7xRljTKybuF6Fw2F5PB6FQqFr8meywfOrItY/W1R41Y8BAIBtovn85rfGAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKwVVRBauXKlRowYIbfbLbfbLZ/Pp3fffdcZP3PmjAKBgPr27avevXurqKhIjY2NEXM0NDSosLBQPXv2VHp6up5++mmdO3cuombr1q0aPXq0XC6XhgwZooqKiot6Wb58uQYPHqzk5GTl5+dr165dEeNd6QUAANgtqiA0YMAALVq0SHV1dfrggw/0h3/4h3r44Yd14MABSdLcuXO1fv16VVZWatu2bTp69KgmTpzo7N/W1qbCwkK1trZqx44dWrNmjSoqKlRWVubUHD58WIWFhRo3bpz27t2rOXPm6LHHHtPmzZudmrVr16q0tFQLFy7Unj17NHLkSPn9fh07dsyp6awXAAAAmW/ppptuMqtXrzZNTU0mMTHRVFZWOmMHDx40kkxtba0xxpiNGzea+Ph4EwwGnZqVK1cat9ttWlpajDHGzJs3zwwbNiziGMXFxcbv9zvreXl5JhAIOOttbW0mMzPTlJeXG2NMl3rpilAoZCSZUCjU5X2iMeiZDRELAAD49qL5/L7ie4Ta2tr05ptvqrm5WT6fT3V1dTp79qwKCgqcmqFDh2rgwIGqra2VJNXW1mr48OHKyMhwavx+v8LhsHNVqba2NmKOjpqOOVpbW1VXVxdREx8fr4KCAqemK71cSktLi8LhcMQCAAC6r6iD0L59+9S7d2+5XC7NmDFDb731lnJychQMBpWUlKTU1NSI+oyMDAWDQUlSMBiMCEEd4x1jl6sJh8M6ffq0Tpw4oba2tkvWnD9HZ71cSnl5uTwej7NkZWV17UUBAAA3pKiD0O233669e/dq586dmjlzpkpKSvTRRx9di96+cwsWLFAoFHKWI0eOxLolAABwDfWIdoekpCQNGTJEkpSbm6vdu3dr6dKlKi4uVmtrq5qamiKuxDQ2Nsrr9UqSvF7vRU93dTzJdX7NhU93NTY2yu12KyUlRQkJCUpISLhkzflzdNbLpbhcLrlcriheDQAAcCP71t8j1N7erpaWFuXm5ioxMVE1NTXOWH19vRoaGuTz+SRJPp9P+/bti3i6q7q6Wm63Wzk5OU7N+XN01HTMkZSUpNzc3Iia9vZ21dTUODVd6QUAACCqK0ILFizQgw8+qIEDB+rkyZN6/fXXtXXrVm3evFkej0fTp09XaWmp0tLS5Ha7NXv2bPl8Po0dO1aSNH78eOXk5GjKlClavHixgsGgnn32WQUCAedKzIwZM7Rs2TLNmzdP06ZN05YtW7Ru3TpVVVU5fZSWlqqkpERjxoxRXl6elixZoubmZk2dOlWSutQLAABAVEHo2LFj+tGPfqQvvvhCHo9HI0aM0ObNm/VHf/RHkqSXXnpJ8fHxKioqUktLi/x+v1asWOHsn5CQoA0bNmjmzJny+Xzq1auXSkpK9MILLzg12dnZqqqq0ty5c7V06VINGDBAq1evlt/vd2qKi4t1/PhxlZWVKRgMatSoUdq0aVPEDdSd9QIAABBnjDGxbuJ6FQ6H5fF4FAqF5Ha7r/r8g+dXRax/tqjwqh8DAADbRPP5zW+NAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLV6xLoB/J/B86su2vbZosIYdAIAgB24IgQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWiioIlZeX6+6771afPn2Unp6uCRMmqL6+PqLmzJkzCgQC6tu3r3r37q2ioiI1NjZG1DQ0NKiwsFA9e/ZUenq6nn76aZ07dy6iZuvWrRo9erRcLpeGDBmiioqKi/pZvny5Bg8erOTkZOXn52vXrl1R9wIAAOwVVRDatm2bAoGA3n//fVVXV+vs2bMaP368mpubnZq5c+dq/fr1qqys1LZt23T06FFNnDjRGW9ra1NhYaFaW1u1Y8cOrVmzRhUVFSorK3NqDh8+rMLCQo0bN0579+7VnDlz9Nhjj2nz5s1Ozdq1a1VaWqqFCxdqz549GjlypPx+v44dO9blXgAAgN3ijDHmSnc+fvy40tPTtW3bNv3gBz9QKBTSzTffrNdff11/9md/Jkn6+OOPdccdd6i2tlZjx47Vu+++qz/5kz/R0aNHlZGRIUlatWqVnnnmGR0/flxJSUl65plnVFVVpf379zvHeuSRR9TU1KRNmzZJkvLz83X33Xdr2bJlkqT29nZlZWVp9uzZmj9/fpd66Uw4HJbH41EoFJLb7b7Sl+kbDZ5f1WnNZ4sKr/pxAQDozqL5/P5W9wiFQiFJUlpamiSprq5OZ8+eVUFBgVMzdOhQDRw4ULW1tZKk2tpaDR8+3AlBkuT3+xUOh3XgwAGn5vw5Omo65mhtbVVdXV1ETXx8vAoKCpyarvRyoZaWFoXD4YgFAAB0X1cchNrb2zVnzhzdc889uvPOOyVJwWBQSUlJSk1NjajNyMhQMBh0as4PQR3jHWOXqwmHwzp9+rROnDihtra2S9acP0dnvVyovLxcHo/HWbKysrr4agAAgBvRFQehQCCg/fv3680337ya/cTUggULFAqFnOXIkSOxbgkAAFxDPa5kp1mzZmnDhg3avn27BgwY4Gz3er1qbW1VU1NTxJWYxsZGeb1ep+bCp7s6nuQ6v+bCp7saGxvldruVkpKihIQEJSQkXLLm/Dk66+VCLpdLLpcrilcCAADcyKK6ImSM0axZs/TWW29py5Ytys7OjhjPzc1VYmKiampqnG319fVqaGiQz+eTJPl8Pu3bty/i6a7q6mq53W7l5OQ4NefP0VHTMUdSUpJyc3Mjatrb21VTU+PUdKUXAABgt6iuCAUCAb3++ut655131KdPH+deG4/Ho5SUFHk8Hk2fPl2lpaVKS0uT2+3W7Nmz5fP5nKe0xo8fr5ycHE2ZMkWLFy9WMBjUs88+q0Ag4FyNmTFjhpYtW6Z58+Zp2rRp2rJli9atW6eqqv97yqq0tFQlJSUaM2aM8vLytGTJEjU3N2vq1KlOT531AgAA7BZVEFq5cqUk6Q/+4A8itr/66qt69NFHJUkvvfSS4uPjVVRUpJaWFvn9fq1YscKpTUhI0IYNGzRz5kz5fD716tVLJSUleuGFF5ya7OxsVVVVae7cuVq6dKkGDBig1atXy+/3OzXFxcU6fvy4ysrKFAwGNWrUKG3atCniBurOegEAAHb7Vt8j1N3xPUIAANx4vrPvEQIAALiREYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsFbUQWj79u166KGHlJmZqbi4OL399tsR48YYlZWVqX///kpJSVFBQYEOHToUUfPVV19p8uTJcrvdSk1N1fTp03Xq1KmImg8//FD33XefkpOTlZWVpcWLF1/US2VlpYYOHark5GQNHz5cGzdujLoXAABgr6iDUHNzs0aOHKnly5dfcnzx4sV6+eWXtWrVKu3cuVO9evWS3+/XmTNnnJrJkyfrwIEDqq6u1oYNG7R9+3Y98cQTzng4HNb48eM1aNAg1dXV6cUXX9Tzzz+vV155xanZsWOHJk2apOnTp+s3v/mNJkyYoAkTJmj//v1R9XK9Gzy/KmIBAABXT5wxxlzxznFxeuuttzRhwgRJv7sCk5mZqb/927/VU089JUkKhULKyMhQRUWFHnnkER08eFA5OTnavXu3xowZI0natGmT/viP/1iff/65MjMztXLlSv2///f/FAwGlZSUJEmaP3++3n77bX388ceSpOLiYjU3N2vDhg1OP2PHjtWoUaO0atWqLvXSmXA4LI/Ho1AoJLfbfaUv0ze6kmDz2aLCq94HAADdSTSf31f1HqHDhw8rGAyqoKDA2ebxeJSfn6/a2lpJUm1trVJTU50QJEkFBQWKj4/Xzp07nZof/OAHTgiSJL/fr/r6en399ddOzfnH6ajpOE5XerlQS0uLwuFwxAIAALqvqxqEgsGgJCkjIyNie0ZGhjMWDAaVnp4eMd6jRw+lpaVF1FxqjvOP8U0154931suFysvL5fF4nCUrK6sL/9UAAOBGxVNj51mwYIFCoZCzHDlyJNYtAQCAa+iqBiGv1ytJamxsjNje2NjojHm9Xh07dixi/Ny5c/rqq68iai41x/nH+Kaa88c76+VCLpdLbrc7YgEAAN3XVQ1C2dnZ8nq9qqmpcbaFw2Ht3LlTPp9PkuTz+dTU1KS6ujqnZsuWLWpvb1d+fr5Ts337dp09e9apqa6u1u23366bbrrJqTn/OB01HcfpSi8AAMBuUQehU6dOae/evdq7d6+k392UvHfvXjU0NCguLk5z5szRT37yE/3iF7/Qvn379KMf/UiZmZnOk2V33HGHHnjgAT3++OPatWuXfv3rX2vWrFl65JFHlJmZKUn6i7/4CyUlJWn69Ok6cOCA1q5dq6VLl6q0tNTp48knn9SmTZv005/+VB9//LGef/55ffDBB5o1a5YkdakXAABgtx7R7vDBBx9o3LhxznpHOCkpKVFFRYXmzZun5uZmPfHEE2pqatK9996rTZs2KTk52dnntdde06xZs3T//fcrPj5eRUVFevnll51xj8ej//iP/1AgEFBubq769eunsrKyiO8a+r3f+z29/vrrevbZZ/V3f/d3uvXWW/X222/rzjvvdGq60gsAALDXt/oeoe6O7xECAODGE7PvEQIAALiREIQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWKtHrBtAdAbPr7po22eLCmPQCQAANz6uCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtfiJjW7gwp/d4Cc3AADoGq4IAQAAaxGEAACAtQhCAADAWgQhAABgLW6W7oYuvHla4gZqAAAuhStCAADAWgQhAABgLYIQAACwFvcIWYIvXQQA4GJcEQIAANbiipCleLIMAABLrggtX75cgwcPVnJysvLz87Vr165Yt3RdGjy/KmIBAKC76/ZXhNauXavS0lKtWrVK+fn5WrJkifx+v+rr65Wenh7r9q5rXDUCAHR3ccYYE+smrqX8/HzdfffdWrZsmSSpvb1dWVlZmj17tubPn3/ZfcPhsDwej0KhkNxu91XvrbtedSEsAQBiKZrP7259Rai1tVV1dXVasGCBsy0+Pl4FBQWqra29qL6lpUUtLS3OeigUkvS7F/RaaG/532syb6wNnFt5Rfvt/7H/KncCALBRx+d2V671dOsgdOLECbW1tSkjIyNie0ZGhj7++OOL6svLy/XjH//4ou1ZWVnXrEf8H8+SWHcAAOhOTp48KY/Hc9mabh2EorVgwQKVlpY66+3t7frqq6/Ut29fxcXFXdVjhcNhZWVl6ciRI9fkz26IDu/H9YX34/rBe3F94f3oGmOMTp48qczMzE5ru3UQ6tevnxISEtTY2BixvbGxUV6v96J6l8sll8sVsS01NfVatii3283/zNcR3o/rC+/H9YP34vrC+9G5zq4EdejWj88nJSUpNzdXNTU1zrb29nbV1NTI5/PFsDMAAHA96NZXhCSptLRUJSUlGjNmjPLy8rRkyRI1Nzdr6tSpsW4NAADEWLcPQsXFxTp+/LjKysoUDAY1atQobdq06aIbqL9rLpdLCxcuvOhPcYgN3o/rC+/H9YP34vrC+3H1dfvvEQIAAPgm3foeIQAAgMshCAEAAGsRhAAAgLUIQgAAwFoEoRhYvny5Bg8erOTkZOXn52vXrl2xbslKzz//vOLi4iKWoUOHxrota2zfvl0PPfSQMjMzFRcXp7fffjti3BijsrIy9e/fXykpKSooKNChQ4di06wFOns/Hn300YvOlwceeCA2zXZz5eXluvvuu9WnTx+lp6drwoQJqq+vj6g5c+aMAoGA+vbtq969e6uoqOiiLw9G1xCEvmNr165VaWmpFi5cqD179mjkyJHy+/06duxYrFuz0rBhw/TFF184y3vvvRfrlqzR3NyskSNHavny5ZccX7x4sV5++WWtWrVKO3fuVK9eveT3+3XmzJnvuFM7dPZ+SNIDDzwQcb688cYb32GH9ti2bZsCgYDef/99VVdX6+zZsxo/fryam5udmrlz52r9+vWqrKzUtm3bdPToUU2cODGGXd/ADL5TeXl5JhAIOOttbW0mMzPTlJeXx7ArOy1cuNCMHDky1m3AGCPJvPXWW856e3u78Xq95sUXX3S2NTU1GZfLZd54440YdGiXC98PY4wpKSkxDz/8cEz6sd2xY8eMJLNt2zZjzO/OhcTERFNZWenUHDx40EgytbW1sWrzhsUVoe9Qa2ur6urqVFBQ4GyLj49XQUGBamtrY9iZvQ4dOqTMzEzdcsstmjx5shoaGmLdEiQdPnxYwWAw4lzxeDzKz8/nXImhrVu3Kj09XbfffrtmzpypL7/8MtYtWSEUCkmS0tLSJEl1dXU6e/ZsxPkxdOhQDRw4kPPjChCEvkMnTpxQW1vbRd9qnZGRoWAwGKOu7JWfn6+Kigpt2rRJK1eu1OHDh3Xffffp5MmTsW7Neh3nA+fK9eOBBx7Qv/3bv6mmpkb/+I//qG3btunBBx9UW1tbrFvr1trb2zVnzhzdc889uvPOOyX97vxISkq66EfBOT+uTLf/iQ3gmzz44IPOv0eMGKH8/HwNGjRI69at0/Tp02PYGXD9eeSRR5x/Dx8+XCNGjND3v/99bd26Vffff38MO+veAoGA9u/fz/2L1xBXhL5D/fr1U0JCwkV39jc2Nsrr9caoK3RITU3Vbbfdpk8++STWrViv43zgXLl+3XLLLerXrx/nyzU0a9YsbdiwQb/61a80YMAAZ7vX61Vra6uampoi6jk/rgxB6DuUlJSk3Nxc1dTUONva29tVU1Mjn88Xw84gSadOndKnn36q/v37x7oV62VnZ8vr9UacK+FwWDt37uRcuU58/vnn+vLLLzlfrgFjjGbNmqW33npLW7ZsUXZ2dsR4bm6uEhMTI86P+vp6NTQ0cH5cAf409h0rLS1VSUmJxowZo7y8PC1ZskTNzc2aOnVqrFuzzlNPPaWHHnpIgwYN0tGjR7Vw4UIlJCRo0qRJsW7NCqdOnYq4mnD48GHt3btXaWlpGjhwoObMmaOf/OQnuvXWW5Wdna3nnntOmZmZmjBhQuya7sYu936kpaXpxz/+sYqKiuT1evXpp59q3rx5GjJkiPx+fwy77p4CgYBef/11vfPOO+rTp49z34/H41FKSoo8Ho+mT5+u0tJSpaWlye12a/bs2fL5fBo7dmyMu78BxfqxNRv98z//sxk4cKBJSkoyeXl55v333491S1YqLi42/fv3N0lJSeZ73/ueKS4uNp988kms27LGr371KyPpoqWkpMQY87tH6J977jmTkZFhXC6Xuf/++019fX1sm+7GLvd+/O///q8ZP368ufnmm01iYqIZNGiQefzxx00wGIx1293Spd4HSebVV191ak6fPm3++q//2tx0002mZ8+e5k//9E/NF198Ebumb2Bxxhjz3ccvAACA2OMeIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACs9f8BXBVpAPzLg8kAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], "source": [ - "import matplotlib.pyplot as plt \n", - "aa = plt.hist(df.importance, bins=100)" + "### Baseline scores on donor 0. Normalized, non cell type specific" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
\n", - "\n", - "\n", + "
\n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - "
A1BGA1BG-AS1A2MA2M-AS1A2ML1A2ML1-AS1A3GALT2A4GALTAAASAACS...ZWILCHZWINTZXDAZXDBZXDCZYG11AZYG11BZYXZZEF1tracks
 ex(False)_tf(-1)ex(True)_tf(-1)static-theta-0.0static-theta-0.5
09322932293229322932293229322932293229322...932293229322932293229322932293229322ABI3_glue
18867886788678867886788678867886788678867...886788678867886788678867886788678867ALX3_glue
28738873887388738873887388738873887388738...873887388738873887388738873887388738ALX4_glue
38699869986998699869986998699869986998699...869986998699869986998699869986998699ARF4_glue
48935893589358935893589358935893589358935...893589358935893589358935893589358935ARGFX_glue
..................................................................
437932993299329932993299329932993292669329...932993299329932993299329932993299329ZNF85_gluepearson_corr0.2396200.5182170.5295020.524232
4389379937993799379937993799379937993799379...937993799379937993799379937993799379ZNF8_gluepearson_causal0.3646560.5924570.7413280.560490
43923512552132845713284132847153384655681343...331827481399590974752621287833186549ZNF93_glue
4408853885388538853885388538853885388538853...885388538853885388538853885388538853ZSCAN29_glue
4419185918591859185918591859185918591859185...91859185918591859185918591859185353ZSCAN31_gluepositive_control0.1973070.5792380.5308480.584694
\n", - "

442 rows × 17076 columns

\n", - "
" + "\n" ], "text/plain": [ - " A1BG A1BG-AS1 A2M A2M-AS1 A2ML1 A2ML1-AS1 A3GALT2 A4GALT AAAS \\\n", - "0 9322 9322 9322 9322 9322 9322 9322 9322 9322 \n", - "1 8867 8867 8867 8867 8867 8867 8867 8867 8867 \n", - "2 8738 8738 8738 8738 8738 8738 8738 8738 8738 \n", - "3 8699 8699 8699 8699 8699 8699 8699 8699 8699 \n", - "4 8935 8935 8935 8935 8935 8935 8935 8935 8935 \n", - ".. ... ... ... ... ... ... ... ... ... \n", - "437 9329 9329 9329 9329 9329 9329 9329 9329 266 \n", - "438 9379 9379 9379 9379 9379 9379 9379 9379 9379 \n", - "439 2351 2552 13284 57 13284 13284 7153 3846 5568 \n", - "440 8853 8853 8853 8853 8853 8853 8853 8853 8853 \n", - "441 9185 9185 9185 9185 9185 9185 9185 9185 9185 \n", - "\n", - " AACS ... ZWILCH ZWINT ZXDA ZXDB ZXDC ZYG11A ZYG11B ZYX ZZEF1 \\\n", - "0 9322 ... 9322 9322 9322 9322 9322 9322 9322 9322 9322 \n", - "1 8867 ... 8867 8867 8867 8867 8867 8867 8867 8867 8867 \n", - "2 8738 ... 8738 8738 8738 8738 8738 8738 8738 8738 8738 \n", - "3 8699 ... 8699 8699 8699 8699 8699 8699 8699 8699 8699 \n", - "4 8935 ... 8935 8935 8935 8935 8935 8935 8935 8935 8935 \n", - ".. ... ... ... ... ... ... ... ... ... ... ... \n", - "437 9329 ... 9329 9329 9329 9329 9329 9329 9329 9329 9329 \n", - "438 9379 ... 9379 9379 9379 9379 9379 9379 9379 9379 9379 \n", - "439 1343 ... 3318 2748 1399 5909 7475 2621 2878 3318 6549 \n", - "440 8853 ... 8853 8853 8853 8853 8853 8853 8853 8853 8853 \n", - "441 9185 ... 9185 9185 9185 9185 9185 9185 9185 9185 353 \n", - "\n", - " tracks \n", - "0 ABI3_glue \n", - "1 ALX3_glue \n", - "2 ALX4_glue \n", - "3 ARF4_glue \n", - "4 ARGFX_glue \n", - ".. ... \n", - "437 ZNF85_glue \n", - "438 ZNF8_glue \n", - "439 ZNF93_glue \n", - "440 ZSCAN29_glue \n", - "441 ZSCAN31_glue \n", - "\n", - "[442 rows x 17076 columns]" + "" ] }, - "execution_count": 19, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "import pandas as pd \n", - "df = pd.read_feather('output/scglue/glue.genes_vs_tracks.rankings.feather')\n", - "df" + "RUN_ID=\"benchmark_donor_0_baselines\" \n", + "models_all = ['pearson_corr', 'pearson_causal', 'positive_control']\n", + "df_all = process_data(RUN_ID, models_all)\n", + "df_all.style.background_gradient()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Baseline scores on donor 0. non normalized, non cell type specific" ] }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
#motif_idgene_namemotif_similarity_qvalueorthologous_identitydescription
0ZNF684_glueZNF6840.01.0placeholder
1TEAD1_glueTEAD10.01.0placeholder
2TEAD2_glueTEAD20.01.0placeholder
3KLF15_glueKLF150.01.0placeholder
4ZNF140_glueZNF1400.01.0placeholder
..................
879HOXA9_suppHOXA90.01.0placeholder
880ARF4_suppARF40.01.0placeholder
881GMEB1_suppGMEB10.01.0placeholder
882RUNX2_suppRUNX20.01.0placeholder
883MYBL2_suppMYBL20.01.0placeholder
\n", - "

884 rows × 5 columns

\n", - "
" - ], - "text/plain": [ - " #motif_id gene_name motif_similarity_qvalue orthologous_identity \\\n", - "0 ZNF684_glue ZNF684 0.0 1.0 \n", - "1 TEAD1_glue TEAD1 0.0 1.0 \n", - "2 TEAD2_glue TEAD2 0.0 1.0 \n", - "3 KLF15_glue KLF15 0.0 1.0 \n", - "4 ZNF140_glue ZNF140 0.0 1.0 \n", - ".. ... ... ... ... \n", - "879 HOXA9_supp HOXA9 0.0 1.0 \n", - "880 ARF4_supp ARF4 0.0 1.0 \n", - "881 GMEB1_supp GMEB1 0.0 1.0 \n", - "882 RUNX2_supp RUNX2 0.0 1.0 \n", - "883 MYBL2_supp MYBL2 0.0 1.0 \n", - "\n", - " description \n", - "0 placeholder \n", - "1 placeholder \n", - "2 placeholder \n", - "3 placeholder \n", - "4 placeholder \n", - ".. ... \n", - "879 placeholder \n", - "880 placeholder \n", - "881 placeholder \n", - "882 placeholder \n", - "883 placeholder \n", - "\n", - "[884 rows x 5 columns]" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "df = pd.read_csv('output/scglue/ctx_annotation.tsv', sep='\\t')\n", - "df" + "### Baseline scores on donor 0. Normalized, cell type specific" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 8, "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines_specific/trace.txt to resources/results/benchmark_donor_0_baselines_specific/trace.txt\n", + "download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines_specific/state.yaml to resources/results/benchmark_donor_0_baselines_specific/state.yaml\n", + "download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines_specific/scores.yaml to resources/results/benchmark_donor_0_baselines_specific/scores.yaml\n", + "download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines_specific/ridge.pearson_causal.pearson_causal.prediction.csv to resources/results/benchmark_donor_0_baselines_specific/ridge.pearson_causal.pearson_causal.prediction.csv\n", + "download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines_specific/ridge.pearson_corr.pearson_corr.prediction.csv to resources/results/benchmark_donor_0_baselines_specific/ridge.pearson_corr.pearson_corr.prediction.csv\n", + "download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines_specific/ridge.positive_control.positive_control.prediction.csv to resources/results/benchmark_donor_0_baselines_specific/ridge.positive_control.positive_control.prediction.csv\n" + ] + }, { "data": { "text/html": [ - "
\n", - "\n", - "\n", + "
\n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - "
A1BGA1BG-AS1A1CFA2MA2M-AS1A2ML1A2MP1A3GALT2A4GALTA4GNT...ZWINTZXDAZXDBZXDCZYG11AZYG11BZYXZZEF1ZZZ3motifs
 ex(False)_tf(-1)ex(True)_tf(-1)static-theta-0.0static-theta-0.5
01954822989145758239186432230222801964591611391...669727682748141721680421338151322151014349bergman__Su_H_
126042214171491723018960322322051618121020...14005105872468113584787515888142142224615789bergman__croc
2229442480072528955234814794147684001919610662...155821807918145931112693235605007156913856bergman__pho
35454195692260118342611676749324586464713525...84241224313551111610067210171548339568355bergman__tll
4734526106206061403225018144372252310084757417621...15401943621095187618552014314335219293261c2h2_zfs__M0369
..................................................................
5871209748928543864221745679113723786117469847...247701212224130199616776410726760209387794yetfasco__TBP-TFIIA_1328
587210316155196125529263555652130324243182617451...797923487212639603194001500826028131858314yetfasco__TBP-TFIIB_1329
58738291252483626194902595274721647712476656016048...619141385078271749081697413970183446172yetfasco__YFL044C_1166pearson_corr0.0256510.2026760.3580650.521077
587485792528070504530235243019127491189583999239...36201938420451564510062208607821256877395yetfasco__YGL192W_1000pearson_causal0.1080470.1975700.3370340.543963
587510932128731502252220345201441505219346780923358...209638528667027722216318393798213227848yetfasco__YPR086W_1327positive_control0.0858760.2353080.5965850.564612
\n", - "

5876 rows × 27016 columns

\n", - "
" + "\n" ], "text/plain": [ - " A1BG A1BG-AS1 A1CF A2M A2M-AS1 A2ML1 A2MP1 A3GALT2 A4GALT \\\n", - "0 19548 22989 14575 8239 18643 2230 22280 19645 916 \n", - "1 2604 22141 71 4917 23018 9603 2232 20516 181 \n", - "2 22944 24800 7252 8955 23481 4794 14768 4001 9196 \n", - "3 5454 19569 2260 11834 26116 7674 93 24586 4647 \n", - "4 7345 26106 20606 14032 25018 14437 22523 10084 7574 \n", - "... ... ... ... ... ... ... ... ... ... \n", - "5871 20974 8928 543 8642 2174 5679 1137 23786 11746 \n", - "5872 103 16155 19612 5529 26355 5652 1303 24243 1826 \n", - "5873 8291 25248 3626 19490 25952 7472 16477 12476 6560 \n", - "5874 8579 25280 7050 4530 23524 3019 12749 11895 8399 \n", - "5875 10932 12873 15022 522 20345 20144 15052 19346 7809 \n", - "\n", - " A4GNT ... ZWINT ZXDA ZXDB ZXDC ZYG11A ZYG11B ZYX ZZEF1 \\\n", - "0 11391 ... 6697 2768 2748 14172 16804 21338 15132 21510 \n", - "1 21020 ... 14005 10587 24681 13584 7875 15888 14214 22246 \n", - "2 10662 ... 15582 18079 18145 9311 12693 23560 5007 15691 \n", - "3 13525 ... 8424 12243 13551 1116 10067 21017 15483 3956 \n", - "4 17621 ... 1540 19436 21095 1876 1855 20143 14335 21929 \n", - "... ... ... ... ... ... ... ... ... ... ... \n", - "5871 9847 ... 24770 12122 24130 1996 16776 4107 26760 20938 \n", - "5872 17451 ... 7979 23487 21263 9603 19400 15008 26028 13185 \n", - "5873 16048 ... 619 14138 5078 2717 4908 16974 13970 18344 \n", - "5874 9239 ... 3620 19384 20451 5645 10062 20860 7821 25687 \n", - "5875 23358 ... 20963 8528 6670 2772 22163 18393 7982 13227 \n", - "\n", - " ZZZ3 motifs \n", - "0 14349 bergman__Su_H_ \n", - "1 15789 bergman__croc \n", - "2 3856 bergman__pho \n", - "3 8355 bergman__tll \n", - "4 3261 c2h2_zfs__M0369 \n", - "... ... ... \n", - "5871 7794 yetfasco__TBP-TFIIA_1328 \n", - "5872 8314 yetfasco__TBP-TFIIB_1329 \n", - "5873 6172 yetfasco__YFL044C_1166 \n", - "5874 7395 yetfasco__YGL192W_1000 \n", - "5875 848 yetfasco__YPR086W_1327 \n", - "\n", - "[5876 rows x 27016 columns]" + "" ] }, - "execution_count": 17, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df_2 = pd.read_feather('output/scenic/databases/hg38_500bp_up_100bp_down_full_tx_v10_clust.genes_vs_motifs.rankings.feather')\n", - "df_2" + "RUN_ID=\"benchmark_donor_0_baselines_specific\"\n", + "models_all = ['pearson_corr', 'pearson_causal', 'positive_control']\n", + "df_all = process_data(RUN_ID, models_all)\n", + "df_all.style.background_gradient()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Commands" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 8, "metadata": {}, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "/vol/tmp/users/jnourisa/ipykernel_2930878/200894551.py:1: DtypeWarning: Columns (4) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " df_2_annot = pd.read_csv('output/scenic/databases/motifs-v10nr_clust-nr.hgnc-m0.001-o0.0.tbl', sep='\\t')\n" + "upload: resources/grn-benchmark/multiomics_rna_qc.h5ad to s3://openproblems-data/resources/grn/grn-benchmark/multiomics_rna_qc.h5ad\n", + "upload: resources/grn-benchmark/multiomics_rna_0.h5ad to s3://openproblems-data/resources/grn/grn-benchmark/multiomics_rna_0.h5ad\n", + "upload: resources/grn-benchmark/multiomics_atac_0.h5ad to s3://openproblems-data/resources/grn/grn-benchmark/multiomics_atac_0.h5ad\n" ] - }, - { - "data": { - "text/plain": [ - "(27072, 30063, 1605)" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_2_annot = pd.read_csv('output/scenic/databases/motifs-v10nr_clust-nr.hgnc-m0.001-o0.0.tbl', sep='\\t')\n", - "df_2_annot['#motif_id'].nunique(), df_2_annot.motif_name.nunique(), df_2_annot.gene_name.nunique()" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1605" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ - "df_2['gene_name'].nunique()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# To be classified " + "!aws s3 sync resources/grn-benchmark s3://openproblems-data/resources/grn/grn-benchmark " ] }, { @@ -1210,135 +373,12 @@ "metadata": {}, "outputs": [], "source": [ - "import anndata as ad \n", - "import numpy as np \n", - "adata_rna = ad.read_h5ad('resources/grn-benchmark/multiomics_rna.h5ad')\n", - "X = adata_rna.X.todense()\n", - "\n", - "if True:\n", - " print('qc')\n", - " print('Data shape before QC: ', adata_rna.shape)\n", - " # Remove genes with >=90% of zeros\n", - " mask_gene = (np.mean(X == 0, axis=0) >= 0.9)\n", - " adata_rna = adata_rna[:, ~mask_gene]\n", - " # Remove samples with >=90% of zeros\n", - " mask_cells = (np.mean(X == 0, axis=1) >= 0.9)\n", - " adata_rna = adata_rna[~mask_cells, :]\n", - " print('Data shape after QC: ', adata_rna.shape)\n", - "adata_rna.write('resources/grn-benchmark/multiomics_rna_qc.h5ad')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## GRN inference: multiomics" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### scGLUE" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "download: s3://openproblems-data/resources/grn/results/scglue/state.yaml to resources/results/scglue/state.yaml\n", - "download: s3://openproblems-data/resources/grn/results/scglue/trace.txt to resources/results/scglue/trace.txt\n", - "download: s3://openproblems-data/resources/grn/results/scglue/output/grn.csv to resources/results/scglue/output/grn.csv\n" - ] - } - ], - "source": [ - "!aws s3 sync s3://openproblems-data/resources/grn/results/scglue ./resources/results/scglue" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[37mReading input files\u001b[0m\n", - "\u001b[37mCalculate basic stats\u001b[0m\n", - "\u001b[37mOutputting stats to : /viash_automount/mnt/c/Users/nourisa/Documents/testProjs/ongoing/task_grn_benchmark/output/stats.json\u001b[0m\n", - "\u001b[37mTopological analysis\u001b[0m\n", - "\u001b[37mPlotting tf-gene in degree, dir: /viash_automount/mnt/c/Users/nourisa/Documents/testProjs/ongoing/task_grn_benchmark/output/tf_gene_indegree.png\u001b[0m\n", - "\u001b[37mPlotting tf-gene out degree, dir: /viash_automount/mnt/c/Users/nourisa/Documents/testProjs/ongoing/task_grn_benchmark/output/tf_gene_outdegree.png\u001b[0m\n" - ] - } - ], - "source": [ - "!viash run src/exp_analysis/config.vsh.yaml -- --tf_gene_net resources/results/scglue/output/grn.csv" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## GRN evaluation for multiomics methods" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "temporaryFolder: /tmp/viash_hub_repo10489538114835231235 uri: https://github.com/openproblems-bio/openproblems-v2.git\n", - "Cloning into '.'...\n", - "checkout out: List(git, checkout, origin/main_build, --, .) 0 \n", - "\u001b[37mExporting run_grn_evaluation (workflows) =nextflow=> /mnt/c/Users/nourisa/Documents/testProjs/ongoing/task_grn_inference/target/nextflow/workflows/run_grn_evaluation\u001b[0m\n", - "\u001b[33mNot all configs built successfully\u001b[0m\n", - "\u001b[33m 39 configs were disabled\u001b[0m\n", - "\u001b[32m 1/1 configs built successfully\u001b[0m\n" - ] - } - ], - "source": [ - "!viash ns build --setup cb -q run_grn_evaluation" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "!bash scripts/run_grn_evaluation.sh \"ridge\"\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "download: s3://openproblems-data/resources/grn/results/grn_evaluation_ridge/scores.yaml to resources/results/grn_evaluation_ridge/scores.yaml\n", - "download: s3://openproblems-data/resources/grn/results/grn_evaluation_ridge/trace.txt to resources/results/grn_evaluation_ridge/trace.txt\n", - "download: s3://openproblems-data/resources/grn/results/grn_evaluation_ridge/metric_configs.yaml to resources/results/grn_evaluation_ridge/metric_configs.yaml\n" - ] - } - ], - "source": [ - "!aws s3 sync s3://openproblems-data/resources/grn/results/grn_evaluation_ridge ./resources/results/grn_evaluation_ridge" + "import yaml\n", + "import pandas as pd \n", + "import matplotlib.pyplot as plt\n", + "import anndata as ad\n", + "controls = ['negative_control','positive_control']\n", + "grn_models = ['collectri','granie', 'figr', 'celloracle', 'scglue', 'scenicplus']" ] }, { @@ -3043,139 +2083,6 @@ "!aws s3 sync s3://openproblems-data/resources/grn/results/robust_analy_causal_1 ./resources/results/robust_analy_causal_1" ] }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "reg1-corr-5\n", - "reg1-corr-3\n", - "reg1-corr-2\n", - "reg1-corr-10\n", - "reg1-corr-1\n", - "reg1-corr-9\n", - "reg1-corr-8\n", - "reg1-corr-6\n", - "reg1-corr-7\n", - "reg1-corr-4\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ex(False)_tf(-1)ex(True)_tf(-1)Mean
corr_50.1604330.1687580.164596
corr_30.1918520.2021960.197024
corr_20.2026650.2135070.208086
corr_100.1664900.1752330.170861
corr_10.2226730.2346880.228680
corr_90.1825400.1922680.187404
corr_80.1867100.1966920.191701
corr_60.2172030.2290860.223144
corr_70.2380840.2509730.244528
corr_40.1788510.1879960.183423
\n", - "
" - ], - "text/plain": [ - " ex(False)_tf(-1) ex(True)_tf(-1) Mean\n", - "corr_5 0.160433 0.168758 0.164596\n", - "corr_3 0.191852 0.202196 0.197024\n", - "corr_2 0.202665 0.213507 0.208086\n", - "corr_10 0.166490 0.175233 0.170861\n", - "corr_1 0.222673 0.234688 0.228680\n", - "corr_9 0.182540 0.192268 0.187404\n", - "corr_8 0.186710 0.196692 0.191701\n", - "corr_6 0.217203 0.229086 0.223144\n", - "corr_7 0.238084 0.250973 0.244528\n", - "corr_4 0.178851 0.187996 0.183423" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [] - }, { "cell_type": "code", "execution_count": 13, @@ -3241,572 +2148,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Corr grn from multiomics" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "metadata": {}, - "outputs": [], - "source": [ - "multiomics_rna = ad.read_h5ad('resources/grn-benchmark/multiomics_rna.h5ad')" - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "metadata": {}, - "outputs": [], - "source": [ - "import scanpy as sc \n", - "sc.pp.normalize_total(multiomics_rna)\n", - "sc.pp.log1p(multiomics_rna)\n", - "sc.pp.scale(multiomics_rna)\n", - "X = multiomics_rna.X" - ] - }, - { - "cell_type": "code", - "execution_count": 75, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Processing groups: 100%|██████████| 4/4 [00:43<00:00, 10.93s/it]\n" - ] - } - ], - "source": [ - "def corr_grn(X: np.ndarray, groups: np.ndarray):\n", - " grns = []\n", - " for group in tqdm(np.unique(groups), desc=\"Processing groups\"):\n", - " X_sub = X[groups == group, :]\n", - " X_sub = StandardScaler().fit_transform(X_sub)\n", - " grn = np.dot(X_sub.T, X_sub) / X_sub.shape[0]\n", - " grns.append(grn)\n", - " return np.mean(grns, axis=0)\n", - "groups = multiomics_rna.obs.cell_type\n", - "corr_net = corr_grn(X, groups)" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "metadata": {}, - "outputs": [], - "source": [ - "corr_net = pd.DataFrame(corr_net, index=multiomics_rna.var_names, columns=multiomics_rna.var_names)" - ] - }, - { - "cell_type": "code", - "execution_count": 99, - "metadata": {}, - "outputs": [], - "source": [ - "tfs = corr_net.abs().sum(axis=0).argsort()[::-1][:1000].index.to_numpy()\n", - "corr_net_sub = corr_net[tfs]" - ] - }, - { - "cell_type": "code", - "execution_count": 100, - "metadata": {}, - "outputs": [], - "source": [ - "corr_net_sub = corr_net_sub.reset_index().melt(id_vars='location', var_name='source', value_name='weight')\n", - "corr_net_sub.rename(columns={'location': 'target'}, inplace=True)\n", - "corr_net_sub.to_csv('output/causal/grns/corr_net_sub.csv')" - ] - }, - { - "cell_type": "code", - "execution_count": 101, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Reading input files\n", - "(549, 15215)\n", - "Compute metrics for layer: pearson\n", - "ex(False)_tf(-1)\n", - "\n", - "Processing groups: 0%| | 0/5 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "\n", - "import matplotlib.pyplot as plt\n", - "aa = plt.hist((df_==0).sum(axis=1)/df.shape[1], bins=100)" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/tmp/ipykernel_247612/3262953603.py:17: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n", - " groups = adata.obs.groupby([cell_type_col, donor_id_col])\n" - ] - }, - { - "ename": "ValueError", - "evalue": "invalid literal for int() with base 10: 'GTCTTTAGTTAGTGAT-3'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[34], line 53\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m meta_adata\n\u001b[1;32m 51\u001b[0m \u001b[38;5;66;03m# Usage\u001b[39;00m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;66;03m# Assuming you already have an AnnData object 'adata' with cell_type and donor_id in 'obs'\u001b[39;00m\n\u001b[0;32m---> 53\u001b[0m meta_adata \u001b[38;5;241m=\u001b[39m \u001b[43mcreate_meta_cells\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmultiomics_rna\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcell_type_col\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mcell_type\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdonor_id_col\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mdonor_id\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn_cells\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m15\u001b[39;49m\u001b[43m)\u001b[49m\n", - "Cell \u001b[0;32mIn[34], line 22\u001b[0m, in \u001b[0;36mcreate_meta_cells\u001b[0;34m(adata, cell_type_col, donor_id_col, n_cells)\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[38;5;66;03m# For each group (cell_type, donor_id combination)\u001b[39;00m\n\u001b[1;32m 20\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m (cell_type, donor_id), indices \u001b[38;5;129;01min\u001b[39;00m groups\u001b[38;5;241m.\u001b[39mgroups\u001b[38;5;241m.\u001b[39mitems():\n\u001b[1;32m 21\u001b[0m \u001b[38;5;66;03m# Convert indices to a numpy array of integers\u001b[39;00m\n\u001b[0;32m---> 22\u001b[0m cell_indices \u001b[38;5;241m=\u001b[39m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mindices\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mint\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 24\u001b[0m \u001b[38;5;66;03m# Filter indices to ensure they are within bounds\u001b[39;00m\n\u001b[1;32m 25\u001b[0m cell_indices \u001b[38;5;241m=\u001b[39m cell_indices[cell_indices \u001b[38;5;241m<\u001b[39m n_cells_in_adata]\n", - "\u001b[0;31mValueError\u001b[0m: invalid literal for int() with base 10: 'GTCTTTAGTTAGTGAT-3'" - ] - } - ], - "source": [ - "import numpy as np\n", - "import pandas as pd\n", - "import scanpy as sc\n", - "\n", - "def create_meta_cells(adata, cell_type_col='cell_type', donor_id_col='donor_id', n_cells=15):\n", - " # Initialize lists to store the meta cell data\n", - " meta_cell_data = []\n", - " meta_cell_obs = []\n", - "\n", - " # Get the raw counts matrix from the AnnData object\n", - " raw_counts = adata.X\n", - "\n", - " # Get the total number of rows in raw_counts\n", - " n_cells_in_adata = raw_counts.shape[0]\n", - "\n", - " # Group by cell_type and donor_id\n", - " groups = adata.obs.groupby([cell_type_col, donor_id_col])\n", - "\n", - " # For each group (cell_type, donor_id combination)\n", - " for (cell_type, donor_id), indices in groups.groups.items():\n", - " # Convert indices to a numpy array of integers\n", - " cell_indices = np.array(list(indices), dtype=int)\n", - "\n", - " # Filter indices to ensure they are within bounds\n", - " cell_indices = cell_indices[cell_indices < n_cells_in_adata]\n", - "\n", - " # Process in chunks of 'n_cells' (15)\n", - " for i in range(0, len(cell_indices), n_cells):\n", - " meta_cell_indices = cell_indices[i:i + n_cells]\n", - " \n", - " # Sum the raw counts for the meta cell\n", - " meta_cell_counts = raw_counts[meta_cell_indices].sum(axis=0)\n", - "\n", - " # Create observation data for the meta cell\n", - " meta_cell_obs.append({\n", - " cell_type_col: cell_type,\n", - " donor_id_col: donor_id\n", - " })\n", - " \n", - " # Append the meta cell data\n", - " meta_cell_data.append(meta_cell_counts)\n", - "\n", - " # Create a new AnnData object for the meta cells\n", - " meta_cell_data = np.array(meta_cell_data)\n", - " meta_cell_obs = pd.DataFrame(meta_cell_obs)\n", - "\n", - " meta_adata = sc.AnnData(X=meta_cell_data, obs=meta_cell_obs)\n", - "\n", - " return meta_adata\n", - "\n", - "# Usage\n", - "# Assuming you already have an AnnData object 'adata' with cell_type and donor_id in 'obs'\n", - "meta_adata = create_meta_cells(multiomics_rna, cell_type_col='cell_type', donor_id_col='donor_id', n_cells=15)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "AnnData object with n_obs × n_vars = 25551 × 22787\n", - " obs: 'cell_type', 'donor_id'\n", - " var: 'gene_ids', 'interval', 'mean', 'std'\n", - " uns: 'log1p'" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Read data\n", - "Noramlize data\n" - ] - } - ], - "source": [ - "import os\n", - "import pandas as pd\n", - "import numpy as np\n", - "import anndata as ad\n", - "import scanpy as sc\n", - "from tqdm import tqdm\n", - "from scipy.stats import spearmanr\n", - "\n", - "## VIASH START\n", - "par = {\n", - " \"multiomics_rna\": \"resources/grn-benchmark/multiomics_rna.h5ad\",\n", - " \"tf_all\": \"resources_test/prior/tf_all.csv\",\n", - " \"corr_method\": \"spearman\"\n", - "}\n", - "## VIASH END\n", - "print('Read data')\n", - "multiomics_rna = ad.read_h5ad(par[\"multiomics_rna\"])\n", - "gene_names = multiomics_rna.var_names.to_numpy()\n", - "tf_all = np.loadtxt(par['tf_all'], dtype=str)\n", - "tf_all = np.intersect1d(tf_all, gene_names)\n", - "groups = multiomics_rna.obs.cell_type\n", - "print('Noramlize data')\n", - "sc.pp.normalize_total(multiomics_rna)\n", - "sc.pp.log1p(multiomics_rna)\n", - "sc.pp.scale(multiomics_rna)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Processing groups: 0%| | 0/4 [00:00 15\u001b[0m \u001b[43maa\u001b[49m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;66;03m# grn = X_sub.T.dot(X_sub)\u001b[39;00m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;66;03m# grn = grn/stds\u001b[39;00m\n\u001b[1;32m 18\u001b[0m grn\n", - "\u001b[0;31mNameError\u001b[0m: name 'aa' is not defined" - ] - } - ], - "source": [ - "X = multiomics_rna.X\n", - "for group in tqdm(np.unique(groups), desc=\"Processing groups\"):\n", - " mask_group = groups == group\n", - " X_sub = X[mask_group, :]\n", - " # stds = np.std(X_sub, axis=0)\n", - " # mask_g = stds != 0\n", - " # stds = stds[mask_g]\n", - " # X_sub = X_sub[:, mask_g]\n", - " # g_names = gene_names[mask_g]\n", - " \n", - " # grn = X_sub.T.dot(X_sub)/np.std(X_sub, axis=0)\n", - " grn = np.corrcoef(X_sub, rowvar=False)\n", - " # grn = np.nan_to_num(grn, nan=0.0, posinf=0.0, neginf=0.0)\n", - " print(grn)\n", - " aa\n", - " # grn = X_sub.T.dot(X_sub)\n", - " # grn = grn/stds\n", - "grn" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Processing groups: 0%| | 0/4 [00:00 25\u001b[0m net \u001b[38;5;241m=\u001b[39m \u001b[43mcreate_corr_net\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmultiomics_rna\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgroups\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpar\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mcorr_method\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n", - "Cell \u001b[0;32mIn[34], line 22\u001b[0m, in \u001b[0;36mcreate_corr_net\u001b[0;34m(X, groups, method)\u001b[0m\n\u001b[1;32m 20\u001b[0m grn \u001b[38;5;241m=\u001b[39m spearmanr(X_sub)\u001b[38;5;241m.\u001b[39mstatistic\n\u001b[1;32m 21\u001b[0m \u001b[38;5;28mprint\u001b[39m(grn)\n\u001b[0;32m---> 22\u001b[0m \u001b[43maa\u001b[49m \n\u001b[1;32m 23\u001b[0m grns\u001b[38;5;241m.\u001b[39mappend(grn)\n\u001b[1;32m 24\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39mmean(grns, axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m)\n", - "\u001b[0;31mNameError\u001b[0m: name 'aa' is not defined" - ] - } - ], - "source": [ - "from scipy.stats import pearsonr\n", - "\n", - "def create_corr_net(X: np.ndarray, groups: np.ndarray, method=\"pearson\"):\n", - " grns = []\n", - " for group in tqdm(np.unique(groups), desc=\"Processing groups\"):\n", - " mask_group = groups == group\n", - " X_sub = X[mask_group, :]\n", - " stds = np.std(X_sub, axis=0)\n", - " mask_g = stds != 0\n", - " stds = stds[mask_g]\n", - " X_sub = X_sub[:, mask_g]\n", - " g_names = gene_names[mask_g]\n", - " if method == \"pearson\":\n", - " # grn = np.corrcoef(X_sub.T)\n", - " grn = X_sub.T.dot(X_sub)\n", - " grn = grn/stds\n", - "\n", - " elif method == \"spearman\":\n", - " grn = spearmanr(X_sub).statistic\n", - "\n", - " grns.append(grn)\n", - " return np.mean(grns, axis=0)\n", - "net = create_corr_net(multiomics_rna.X, groups, par['corr_method'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "\n", - "print('Read data')\n", - "\n", - "\n", - "print('Create corr net')\n", - "\n", - "net = pd.DataFrame(net, index=gene_names, columns=gene_names)\n", - "\n", - "if par['causal']:\n", - " net = net[tf_all]\n", - "else:\n", - " net = net.sample(len(tf_all), axis=1, random_state=par['seed'])\n", - " \n", - "net = net.reset_index().melt(id_vars='index', var_name='source', value_name='weight')\n", - "net.rename(columns={'index': 'target'}, inplace=True)\n", - "\n", - "\n", - "print('Output GRN')\n", - "net.to_csv(par['prediction'])\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Sinlge omics" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# !bash scripts/run_benchmark_single_omics.sh" + "# GRN Evaluation" ] }, { @@ -3830,42 +2172,6 @@ "!bash scripts/run_grn_evaluation.sh {tag} {reg_type}" ] }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "import scanpy as sc \n", - "import anndata as ad \n", - "multiomics_rna = ad.read_h5ad('resources/grn-benchmark/multiomics_rna.h5ad')\n", - "multiomics_rna.layers['counts'] = multiomics_rna.X.copy()\n", - "sc.pp.normalize_total(multiomics_rna)\n", - "sc.pp.log1p(multiomics_rna)\n", - "sc.pp.scale(multiomics_rna)\n", - "multiomics_rna.layers['lognorm'] = multiomics_rna.X.copy()\n", - "multiomics_rna.X = multiomics_rna.layers['counts']\n", - "del multiomics_rna.layers['counts']\n", - "multiomics_rna.write('resources/grn-benchmark/multiomics_rna.h5ad')" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "multiomics_rna = ad.read_h5ad('resources/grn-benchmark/multiomics_rna_0.h5ad')\n", - "multiomics_rna.layers['counts'] = multiomics_rna.X.copy()\n", - "sc.pp.normalize_total(multiomics_rna)\n", - "sc.pp.log1p(multiomics_rna)\n", - "sc.pp.scale(multiomics_rna)\n", - "multiomics_rna.layers['lognorm'] = multiomics_rna.X.copy()\n", - "multiomics_rna.X = multiomics_rna.layers['counts']\n", - "del multiomics_rna.layers['counts']\n", - "multiomics_rna.write('resources/grn-benchmark/multiomics_rna_0.h5ad')" - ] - }, { "cell_type": "code", "execution_count": 4, diff --git a/scripts/run_benchmark_single_omics.sh b/scripts/run_benchmark_all.sh similarity index 58% rename from scripts/run_benchmark_single_omics.sh rename to scripts/run_benchmark_all.sh index e2de6b4bb..5d6615f8f 100644 --- a/scripts/run_benchmark_single_omics.sh +++ b/scripts/run_benchmark_all.sh @@ -1,7 +1,7 @@ #!/bin/bash # RUN_ID="run_$(date +%Y-%m-%d_%H-%M-%S)" -RUN_ID="benchmark_donor_0_default" +RUN_ID="benchmark_donor_0_baselines_specific" # resources_dir="./resources_test/" resources_dir="s3://openproblems-data/resources/grn" publish_dir="${resources_dir}/results/${RUN_ID}" @@ -11,9 +11,9 @@ subsample=-2 max_workers=10 layer='scgen_pearson' metric_ids="[regression_1, regression_2]" -cell_type_specific=false #for controls +cell_type_specific=true #for controls # method_ids="[tigress, ennet, scsgl, pidc]" -method_ids="[pearson_corr, pearson_causal, positive_control, portia, grnboost2]" +method_ids="[pearson_corr, pearson_causal, positive_control]" param_file="./params/${RUN_ID}.yaml" @@ -45,21 +45,21 @@ HERE # -c src/common/nextflow_helpers/labels_ci.config \ # -params-file ${param_file} -./tw-windows-x86_64.exe launch ` - https://github.com/openproblems-bio/task_grn_inference.git ` - --revision build/main ` - --pull-latest ` - --main-script target/nextflow/workflows/run_benchmark/main.nf ` - --workspace 53907369739130 ` - --compute-env 6TeIFgV5OY4pJCk8I0bfOh ` - --params-file ./params/benchmark_donor_0_default.yaml ` - --config src/common/nextflow_helpers/labels_tw.config +# ./tw-windows-x86_64.exe launch ` +# https://github.com/openproblems-bio/task_grn_inference.git ` +# --revision build/main ` +# --pull-latest ` +# --main-script target/nextflow/workflows/run_benchmark/main.nf ` +# --workspace 53907369739130 ` +# --compute-env 6TeIFgV5OY4pJCk8I0bfOh ` +# --params-file ./params/benchmark_donor_0_default.yaml ` +# --config src/common/nextflow_helpers/labels_tw.config -# ./tw launch https://github.com/openproblems-bio/task_grn_inference \ -# --revision build/main \ -# --pull-latest \ -# --main-script target/nextflow/workflows/run_benchmark/main.nf \ -# --workspace 53907369739130 \ -# --compute-env 6TeIFgV5OY4pJCk8I0bfOh \ -# --params-file ${param_file} \ -# --config src/common/nextflow_helpers/labels_tw.config +./tw launch https://github.com/openproblems-bio/task_grn_inference \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 53907369739130 \ + --compute-env 6TeIFgV5OY4pJCk8I0bfOh \ + --params-file ${param_file} \ + --config src/common/nextflow_helpers/labels_tw.config diff --git a/scripts/sbatch/batch_grnboost2.sh b/scripts/sbatch/batch_grnboost2.sh deleted file mode 100644 index cee7e27bc..000000000 --- a/scripts/sbatch/batch_grnboost2.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -#SBATCH --time=12:00:00 -#SBATCH --job-name=grnboost2 -#SBATCH --output=logs/%j.out -#SBATCH --error=logs/%j.err -#SBATCH --mail-type=END -#SBATCH --mail-user=jalil.nourisa@gmail.com -#SBATCH --cpus-per-task=20 -#SBATCH --mem=64G - -singularity exec ../../images/scenic python src/methods/single_omics/grnboost2/script.py diff --git a/scripts/sbatch/batch_scenic.sh b/scripts/sbatch/batch_scenic.sh index e8da2a0ec..b1ab087bb 100644 --- a/scripts/sbatch/batch_scenic.sh +++ b/scripts/sbatch/batch_scenic.sh @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --time=12:00:00 -#SBATCH --job-name=scenic +#SBATCH --job-name=genie3 #SBATCH --output=logs/%j.out #SBATCH --error=logs/%j.err #SBATCH --mail-type=END @@ -8,4 +8,7 @@ #SBATCH --cpus-per-task=20 #SBATCH --mem=64G -singularity exec ../../images/scenic python src/methods/single_omics/scenic/script.py +# singularity exec ../../images/scenic python src/methods/single_omics/scenic/script.py +# singularity exec ../../images/scenic python src/methods/single_omics/grnboost2/script.py +singularity exec ../../images/scenic python src/methods/single_omics/genie3/script.py + diff --git a/src/api/comp_method.yaml b/src/api/comp_method.yaml index 8f47d029d..f156ccce6 100644 --- a/src/api/comp_method.yaml +++ b/src/api/comp_method.yaml @@ -47,6 +47,10 @@ functionality: type: boolean direction: input default: true + - name: --normalize + type: boolean + direction: input + default: false diff --git a/src/control_methods/pearson/script.py b/src/control_methods/pearson/script.py index 6a01094b6..045198a95 100644 --- a/src/control_methods/pearson/script.py +++ b/src/control_methods/pearson/script.py @@ -63,11 +63,11 @@ def create_corr_net(X, gene_names, groups, par): tf_all = np.loadtxt(par['tf_all'], dtype=str) groups = multiomics_rna.obs.cell_type tf_all = np.intersect1d(tf_all, gene_names) - -print('Noramlize data') -sc.pp.normalize_total(multiomics_rna) -sc.pp.log1p(multiomics_rna) -sc.pp.scale(multiomics_rna) +if par['normalize']: + print('Noramlize data') + sc.pp.normalize_total(multiomics_rna) + sc.pp.log1p(multiomics_rna) + sc.pp.scale(multiomics_rna) print('Create corr net') net = create_corr_net(multiomics_rna.X, multiomics_rna.var_names, groups, par) diff --git a/src/control_methods/pearson_causal/script.py b/src/control_methods/pearson_causal/script.py index 14fbc349a..36860fa09 100644 --- a/src/control_methods/pearson_causal/script.py +++ b/src/control_methods/pearson_causal/script.py @@ -65,10 +65,11 @@ def create_corr_net(X, gene_names, groups, par): groups = multiomics_rna.obs.cell_type tf_all = np.intersect1d(tf_all, gene_names) -print('Noramlize data') -sc.pp.normalize_total(multiomics_rna) -sc.pp.log1p(multiomics_rna) -sc.pp.scale(multiomics_rna) +if par['normalize']: + print('Noramlize data') + sc.pp.normalize_total(multiomics_rna) + sc.pp.log1p(multiomics_rna) + sc.pp.scale(multiomics_rna) print('Create corr net') net = create_corr_net(multiomics_rna.X, multiomics_rna.var_names, groups, par) diff --git a/src/control_methods/positive_control/script.py b/src/control_methods/positive_control/script.py index 2711cea83..46732a323 100644 --- a/src/control_methods/positive_control/script.py +++ b/src/control_methods/positive_control/script.py @@ -60,13 +60,18 @@ def create_corr_net(X, gene_names, groups, par): multiomics_rna = ad.read_h5ad(par["perturbation_data"]) +if par['normalize']: + print('Noramlize data') + sc.pp.normalize_total(multiomics_rna) + sc.pp.log1p(multiomics_rna) + sc.pp.scale(multiomics_rna) + gene_names = multiomics_rna.var_names.to_numpy() tf_all = np.loadtxt(par['tf_all'], dtype=str) groups = multiomics_rna.obs.cell_type tf_all = np.intersect1d(tf_all, gene_names) - print('Create corr net') net = create_corr_net(multiomics_rna.X, multiomics_rna.var_names, groups, par) diff --git a/src/methods/multi_omics/celloracle/script.py b/src/methods/multi_omics/celloracle/script.py index 7e7b5629b..b44dae210 100644 --- a/src/methods/multi_omics/celloracle/script.py +++ b/src/methods/multi_omics/celloracle/script.py @@ -5,16 +5,17 @@ ## VIASH START par = { - "multiomics_rna": "resources/grn-benchmark/multiomics_rna.h5ad", - "multiomics_atac": "resources/grn-benchmark/multiomics_atac.h5ad", + "multiomics_rna": "resources_test/grn-benchmark/multiomics_rna.h5ad", + "multiomics_atac": "resources_test/grn-benchmark/multiomics_atac.h5ad", + "base_grn": 'output/celloracle/base_grn.csv', "temp_dir": 'output/celloracle/', "num_workers": 4, - "prediction": "output/prediction.h5ad", + "prediction": "output/celloracle_test.h5ad", } ## VIASH END -# meta = { -# "resources_dir":'resources' -# } +meta = { + "resources_dir":'src/methods/multi_omics/celloracle' +} par['links'] = f"{par['temp_dir']}/links.celloracle.links" sys.path.append(meta["resources_dir"]) diff --git a/src/methods/multi_omics/celloracle_ns/config.novsh.yaml b/src/methods/multi_omics/celloracle_ns/config.novsh.yaml deleted file mode 100644 index d38d395e3..000000000 --- a/src/methods/multi_omics/celloracle_ns/config.novsh.yaml +++ /dev/null @@ -1,29 +0,0 @@ - -__merge__: ../../../api/comp_method.yaml - -functionality: - name: grn_inference_celloracle - namespace: "workflows" - info: - label: grn_inference_celloracle - summary: "Infers GRNs from multiomics data using celloracle." - arguments: - - name: --base_grn - type: file - direction: output - default: output/celloracle/base_grn.csv - - name: --links - type: file - direction: output - default: output/celloracle/links.celloracle.links - resources: - - type: nextflow_script - path: main.nf - entrypoint: run_wf - dependencies: - - name: grn_methods/celloracle - -platforms: - - type: nextflow - directives: - label: [ midtime, midmem, lowcpu ] diff --git a/src/methods/multi_omics/celloracle_ns/main.nf b/src/methods/multi_omics/celloracle_ns/main.nf deleted file mode 100644 index e224f438e..000000000 --- a/src/methods/multi_omics/celloracle_ns/main.nf +++ /dev/null @@ -1,21 +0,0 @@ -workflow run_wf { - take: - input_ch - - main: - output_ch = input_ch - - | celloracle.run( - fromState: [multiomics_rna: "multiomics_rna", - multiomics_atac: "multiomics_atac", - temp_dir: "temp_dir", - num_workers: "num_workers" - ], - toState: [prediction:"prediction", base_grn: "base_grn", links: "links"] - ) - - | setState(["prediction", "base_grn", "links"]) - - emit: - output_ch -} diff --git a/src/methods/multi_omics/celloracle_ns/run.sh b/src/methods/multi_omics/celloracle_ns/run.sh deleted file mode 100644 index bd5b3160a..000000000 --- a/src/methods/multi_omics/celloracle_ns/run.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -# RUN_ID="run_$(date +%Y-%m-%d_%H-%M-%S)" -RUN_ID="celloracle" -resources_dir="s3://openproblems-data/resources/grn" -publish_dir="s3://openproblems-data/resources/grn/results/${RUN_ID}" - -# resources_dir="./resources_test" -# publish_dir="./output/${RUN_ID}" - -num_workers=20 - -param_file="./params/${RUN_ID}.yaml" -# Start writing to the YAML file -cat > $param_file << HERE -param_list: - - id: ${RUN_ID} - multiomics_rna: ${resources_dir}/grn-benchmark/multiomics_rna.h5ad - multiomics_atac: ${resources_dir}/grn-benchmark/multiomics_atac.h5ad - num_workers: $num_workers - temp_dir: ./tmp/grn -output_state: "state.yaml" -publish_dir: "$publish_dir" -HERE - -if [ "$submit" = true ]; then - nextflow run . \ - -main-script target/nextflow/workflows/grn_inference_celloracle/main.nf \ - -profile docker \ - -with-trace \ - -c src/common/nextflow_helpers/labels_ci.config \ - -params-file params/${RUN_ID}.yaml - - - ./tw-windows-x86_64.exe launch ` - https://github.com/openproblems-bio/task_grn_inference.git ` - --revision build/main ` - --pull-latest ` - --main-script target/nextflow/workflows/run_grn_inference/main.nf ` - --workspace 53907369739130 ` - --compute-env 6TeIFgV5OY4pJCk8I0bfOh ` - --params-file ./params/celloracle_test.yaml ` - --config src/common/nextflow_helpers/labels_tw.config -fi - - - -if [ "$read_results" = true ]; then - aws s3 sync s3://openproblems-data/resources/grn/results/${RUN_ID} ./resources/results/${RUN_ID} -fi - - -./tw-windows-x86_64.exe launch ` - https://github.com/openproblems-bio/task_grn_inference.git ` - --revision build/main ` - --pull-latest ` - --main-script target/nextflow/workflows/grn_inference_celloracle/main.nf ` - --workspace 53907369739130 ` - --compute-env 6TeIFgV5OY4pJCk8I0bfOh ` - --params-file ./params/celloracle.yaml ` - --config src/common/nextflow_helpers/labels_tw.config \ No newline at end of file diff --git a/src/methods/multi_omics/granie_ns/config.novsh.yaml b/src/methods/multi_omics/granie_ns/config.novsh.yaml deleted file mode 100644 index 4e005399e..000000000 --- a/src/methods/multi_omics/granie_ns/config.novsh.yaml +++ /dev/null @@ -1,27 +0,0 @@ - -__merge__: ../../../api/comp_method_r.yaml - -functionality: - name: grn_inference_granie - namespace: "workflows" - info: - label: grn_inference_granie - summary: "Infers GRNs from multiomics data using granie." - arguments: - - name: --subset - type: boolean - required: false - default: false - direction: input - - resources: - - type: nextflow_script - path: main.nf - entrypoint: run_wf - dependencies: - - name: grn_methods/granie - -platforms: - - type: nextflow - directives: - label: [ hightime, midmem, highcpu ] diff --git a/src/methods/multi_omics/granie_ns/main.nf b/src/methods/multi_omics/granie_ns/main.nf deleted file mode 100644 index 9bef84115..000000000 --- a/src/methods/multi_omics/granie_ns/main.nf +++ /dev/null @@ -1,22 +0,0 @@ -workflow run_wf { - take: - input_ch - - main: - output_ch = input_ch - - | granie.run( - fromState: [ - multiomics_rna_r: "multiomics_rna_r", - multiomics_atac_r: "multiomics_atac_r", - num_workers: "num_workers", - subset: "subset" - ], - toState: [prediction:"prediction"] - ) - - | setState(["prediction"]) - - emit: - output_ch -} diff --git a/src/methods/multi_omics/granie_ns/run.sh b/src/methods/multi_omics/granie_ns/run.sh deleted file mode 100644 index 9ee2b676f..000000000 --- a/src/methods/multi_omics/granie_ns/run.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# RUN_ID="run_$(date +%Y-%m-%d_%H-%M-%S)" -RUN_ID="scglue" -resources_dir="s3://openproblems-data/resources/grn" -publish_dir="s3://openproblems-data/resources/grn/results/${RUN_ID}" - -num_workers=20 - -param_file="./params/${RUN_ID}.yaml" - -cat > $param_file << HERE -param_list: - - id: ${RUN_ID} - multiomics_rna: ${resources_dir}/grn-benchmark/multiomics_rna.h5ad - multiomics_atac: ${resources_dir}/grn-benchmark/multiomics_atac.h5ad - annotation_file: ${resources_dir}/supplementary/gencode.v45.annotation.gtf.gz - motif_file: ${resources_dir}/supplementary/JASPAR2022-hg38.bed.gz - num_workers: $num_workers - temp_dir: ./tmp/grn -output_state: "state.yaml" -publish_dir: "$publish_dir" -HERE - - - -./tw-windows-x86_64.exe launch ` - https://github.com/openproblems-bio/task_grn_inference.git ` - --revision build/main ` - --pull-latest ` - --main-script target/nextflow/workflows/grn_inference_scglue/main.nf ` - --workspace 53907369739130 ` - --compute-env 6TeIFgV5OY4pJCk8I0bfOh ` - --params-file ./params/scglue.yaml ` - --config src/common/nextflow_helpers/labels_tw.config \ No newline at end of file diff --git a/src/methods/multi_omics/scenicplus_ns/config.novsh.yaml b/src/methods/multi_omics/scenicplus_ns/config.novsh.yaml deleted file mode 100644 index ac7060938..000000000 --- a/src/methods/multi_omics/scenicplus_ns/config.novsh.yaml +++ /dev/null @@ -1,39 +0,0 @@ - -__merge__: ../../../api/comp_method.yaml - -functionality: - name: grn_inference_scenicplus - namespace: "workflows" - info: - label: grn_inference_scenicplus - summary: "Infers GRNs from multiomics data using scenicplus." - arguments: - - name: --scplus_mdata - type: file - required: false - direction: output - description: "Main output object." - default: "scplus_mdata.h5mu" - - name: --qc - type: boolean - default: false - description: "Whether to perform quality control." - - name: --cell_topic - type: file - default: output/cell_topic.csv - required: false - direction: output - description: "Cell-topics prob scores" - - - resources: - - type: nextflow_script - path: main.nf - entrypoint: run_wf - dependencies: - - name: grn_methods/scenicplus - -platforms: - - type: nextflow - directives: - label: [ midtime, highmem, highcpu ] diff --git a/src/methods/multi_omics/scenicplus_ns/main.nf b/src/methods/multi_omics/scenicplus_ns/main.nf deleted file mode 100644 index 4a27b82d0..000000000 --- a/src/methods/multi_omics/scenicplus_ns/main.nf +++ /dev/null @@ -1,23 +0,0 @@ -workflow run_wf { - take: - input_ch - - main: - output_ch = input_ch - - | scenicplus.run( - fromState: [ - multiomics_rna: "multiomics_rna", - multiomics_atac: "multiomics_atac", - temp_dir: "temp_dir", - num_workers: "num_workers" - - ], - toState: [prediction:"prediction", cell_topic:"cell_topic", scplus_mdata:"scplus_mdata"] - ) - - | setState(["prediction", "cell_topic", "scplus_mdata"]) - - emit: - output_ch -} diff --git a/src/methods/multi_omics/scenicplus_ns/run.sh b/src/methods/multi_omics/scenicplus_ns/run.sh deleted file mode 100644 index eef0d8c0a..000000000 --- a/src/methods/multi_omics/scenicplus_ns/run.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -RUN_ID="scenicplus" -resources_dir="s3://openproblems-data/resources_test/grn" -publish_dir="s3://openproblems-data/resources_test/grn/results/${RUN_ID}" - -# resources_dir="./resources_test" -# publish_dir="./output/${RUN_ID}" - -num_workers=10 - -param_file="./params/${RUN_ID}.yaml" -# Start writing to the YAML file -cat > $param_file << HERE -param_list: - - id: ${RUN_ID} - multiomics_rna: ${resources_dir}/grn-benchmark/multiomics_rna.h5ad - multiomics_atac: ${resources_dir}/grn-benchmark/multiomics_atac.h5ad - num_workers: $num_workers - temp_dir: tmp/grn -output_state: "state.yaml" -publish_dir: "$publish_dir" -HERE - - -nextflow run . \ - -main-script target/nextflow/workflows/grn_inference_scenicplus/main.nf \ - -profile docker \ - -with-trace \ - -c src/common/nextflow_helpers/labels_ci.config \ - -params-file params/${RUN_ID}.yaml - - -# ./tw-windows-x86_64.exe launch ` -# https://github.com/openproblems-bio/task_grn_inference.git ` -# --revision build/main ` -# --pull-latest ` -# --main-script target/nextflow/workflows/grn_inference_scenicplus/main.nf ` -# --workspace 53907369739130 ` -# --compute-env 6TeIFgV5OY4pJCk8I0bfOh ` -# --params-file ./params/scenicplus.yaml ` -# --config src/common/nextflow_helpers/labels_tw.config - - diff --git a/src/methods/multi_omics/scglue_ns/config.novsh.yaml b/src/methods/multi_omics/scglue_ns/config.novsh.yaml deleted file mode 100644 index 30c69fee8..000000000 --- a/src/methods/multi_omics/scglue_ns/config.novsh.yaml +++ /dev/null @@ -1,20 +0,0 @@ - -__merge__: ../../../api/comp_method.yaml - -functionality: - name: grn_inference_scglue - namespace: "workflows" - info: - label: grn_inference_scglue - summary: "Infers GRNs from multiomics data using scglue." - resources: - - type: nextflow_script - path: main.nf - entrypoint: run_wf - dependencies: - - name: grn_methods/scglue - -platforms: - - type: nextflow - directives: - label: [twodaytime, midmem, highcpu] diff --git a/src/methods/multi_omics/scglue_ns/main.nf b/src/methods/multi_omics/scglue_ns/main.nf deleted file mode 100644 index 9b9bc0e89..000000000 --- a/src/methods/multi_omics/scglue_ns/main.nf +++ /dev/null @@ -1,22 +0,0 @@ -workflow run_wf { - take: - input_ch - - main: - output_ch = input_ch - - | scglue.run( - fromState: [ - multiomics_rna: "multiomics_rna", - multiomics_atac: "multiomics_atac", - temp_dir: "temp_dir", - num_workers: "num_workers" - ], - toState: [prediction:"prediction"] - ) - - | setState(["prediction"]) - - emit: - output_ch -} diff --git a/src/methods/multi_omics/scglue_ns/run.sh b/src/methods/multi_omics/scglue_ns/run.sh deleted file mode 100644 index 1c58dead3..000000000 --- a/src/methods/multi_omics/scglue_ns/run.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# RUN_ID="run_$(date +%Y-%m-%d_%H-%M-%S)" -RUN_ID="scglue" -resources_dir="s3://openproblems-data/resources_test/grn" -publish_dir="s3://openproblems-data/resources/grn/results/${RUN_ID}" - - -param_file="./params/${RUN_ID}.yaml" - -cat > $param_file << HERE -param_list: - - id: ${RUN_ID} - multiomics_rna: ${resources_dir}/grn-benchmark/multiomics_rna.h5ad - multiomics_atac: ${resources_dir}/grn-benchmark/multiomics_atac.h5ad - num_workers: $num_workers - temp_dir: ./tmp/grn -output_state: "state.yaml" -publish_dir: "$publish_dir" -HERE - - - -# ./tw-windows-x86_64.exe launch ` -# https://github.com/openproblems-bio/task_grn_inference.git ` -# --revision build/main ` -# --pull-latest ` -# --main-script target/nextflow/workflows/grn_inference_scglue/main.nf ` -# --workspace 53907369739130 ` -# --compute-env 6TeIFgV5OY4pJCk8I0bfOh ` -# --params-file ./params/scglue.yaml ` -# --config src/common/nextflow_helpers/labels_tw.config \ No newline at end of file diff --git a/src/methods/single_omics/ennet/config.vsh.yaml b/src/methods/single_omics/ennet/config.novsh.yaml similarity index 100% rename from src/methods/single_omics/ennet/config.vsh.yaml rename to src/methods/single_omics/ennet/config.novsh.yaml diff --git a/src/methods/single_omics/genie3/script.py b/src/methods/single_omics/genie3/script.py index 9e632553e..c4a6f6af1 100644 --- a/src/methods/single_omics/genie3/script.py +++ b/src/methods/single_omics/genie3/script.py @@ -9,35 +9,25 @@ ## VIASH START par = { - 'multiomics_rna': 'resources/resources_test/grn-benchmark/multiomics_rna.h5ad', + 'multiomics_rna': 'resources/grn-benchmark/multiomics_rna_0.h5ad', "tf_all": 'resources/prior/tf_all.csv', - 'prediction': 'output/genie3/prediction.csv', + 'prediction': 'output//genie3_donor0.csv', 'max_n_links': 50000 } ## VIASH END -# Load scRNA-seq data +print('Load scRNA-seq data') adata_rna = anndata.read_h5ad(par['multiomics_rna']) gene_names = adata_rna.var.gene_ids.index.to_numpy() X = adata_rna.X.toarray() -if False: - # Remove genes with >=90% of zeros - mask = (np.mean(X == 0, axis=0) >= 0.9) - X = X[:, ~mask] - gene_names = gene_names[~mask] - - # Remove samples with >=90% of zeros - mask = (np.mean(X == 0, axis=1) >= 0.9) - adata_rna = X[~mask, :] - # Load list of putative TFs df = pd.read_csv(par["tf_all"], header=None, names=['gene_name']) tfs = set(list(df['gene_name'])) tf_names = [gene_name for gene_name in gene_names if (gene_name in tfs)] -# GRN inference +print('GRN inference') client = Client(processes=False) network = genie3(X, client_or_address=client, gene_names=gene_names, tf_names=tf_names) diff --git a/src/methods/single_omics/ppcor/script.R b/src/methods/single_omics/ppcor/script.R index c05516136..7ff9416ff 100644 --- a/src/methods/single_omics/ppcor/script.R +++ b/src/methods/single_omics/ppcor/script.R @@ -4,7 +4,7 @@ library(dplyr) ## VIASH START par <- list( - "multiomics_rna" = 'resources/resources_test/grn-benchmark/multiomics_rna.h5ad', + "multiomics_rna" = 'resources_test/grn-benchmark/multiomics_rna.h5ad', "prediction" = 'output/ppcor/prediction.csv', "temp_dir": 'output/ppcor', "max_n_links": 50000 @@ -18,16 +18,6 @@ geneNames <- colnames(inputExpr) colnames(inputExpr) <- c(geneNames) X <- as.matrix(inputExpr) - -# # Keep genes with less than 10% of zeros -# mask <- (apply(X, 2, function(x) mean(x != 0)) >= 0.1) -# X <- X[, mask] -# geneNames <- geneNames[mask] - -# # Keep samples with less than 10% of zeros -# mask <- (apply(X, 1, function(x) mean(x != 0)) >= 0.1) -# X <- X[mask,] - # Run GRN inference method pcorResults = pcor(x = X, method = "pearson") diff --git a/src/methods/single_omics/scgpt/config.vsh.yaml b/src/methods/single_omics/scgpt/config.vsh.yaml index 230d090a6..2ec20365b 100644 --- a/src/methods/single_omics/scgpt/config.vsh.yaml +++ b/src/methods/single_omics/scgpt/config.vsh.yaml @@ -32,7 +32,7 @@ functionality: - name: --batch_size type: integer direction: input - default: 16 + default: 10 - name: --condition type: string direction: input diff --git a/src/workflows/run_benchmark_all/config.vsh.yaml b/src/workflows/run_benchmark/config.vsh.yaml similarity index 92% rename from src/workflows/run_benchmark_all/config.vsh.yaml rename to src/workflows/run_benchmark/config.vsh.yaml index f1f13c087..3f0cd257f 100644 --- a/src/workflows/run_benchmark_all/config.vsh.yaml +++ b/src/workflows/run_benchmark/config.vsh.yaml @@ -90,21 +90,29 @@ functionality: repository: openproblems - name: metrics/regression_2 - name: metrics/regression_1 + # ---- single omics - name: grn_methods/portia - - name: grn_methods/ennet - - name: grn_methods/genie3 - name: grn_methods/grnboost2 - # - name: grn_methods/pidc - - name: grn_methods/ppcor - # - name: grn_methods/scsgl - # - name: grn_methods/tigress - - name: grn_methods/celloracle - - name: grn_methods/scgpt - name: grn_methods/scenic + - name: grn_methods/genie3 + + - name: grn_methods/ppcor #needs docker image + - name: grn_methods/scgpt + + # ---- multiomics + - name: grn_methods/celloracle + - name: grn_methods/scglue + # ---- baselines - name: control_methods/pearson_corr - name: control_methods/pearson_causal - name: control_methods/negative_control - name: control_methods/positive_control + # -- needs development + # - name: grn_methods/pidc + # - name: grn_methods/scsgl + # - name: grn_methods/tigress + # - name: grn_methods/ennet + repositories: - name: openproblems type: github diff --git a/src/workflows/run_benchmark_all/main.nf b/src/workflows/run_benchmark/main.nf similarity index 99% rename from src/workflows/run_benchmark_all/main.nf rename to src/workflows/run_benchmark/main.nf index d653e39ef..d3a54008f 100644 --- a/src/workflows/run_benchmark_all/main.nf +++ b/src/workflows/run_benchmark/main.nf @@ -26,7 +26,8 @@ workflow run_wf { negative_control, positive_control, - celloracle + celloracle, + scglue ] diff --git a/test.sh b/test.sh deleted file mode 100644 index fde2fae79..000000000 --- a/test.sh +++ /dev/null @@ -1,20 +0,0 @@ -# viash run src/control_methods/baseline_corr/config.vsh.yaml -- --multiomics_rna resources/grn-benchmark/multiomics_rna.h5ad \ -# --tf_all resources/prior/tf_all.csv \ -# --causal true \ -# --prediction output/baseline_causal.csv - -# viash run src/metrics/regression_1/config.vsh.yaml -- --perturbation_data resources/grn-benchmark/perturbation_data.h5ad \ -# --tf_all resources/prior/tf_all.csv \ -# --prediction output/baseline_causal.csv \ -# --score output/score_causal.h5ad - -viash run src/control_methods/baseline_corr/config.vsh.yaml -- --multiomics_rna resources/grn-benchmark/multiomics_rna.h5ad \ - --tf_all resources/prior/tf_all.csv \ - --causal false \ - --corr_method pearson \ - --prediction output/baseline_noncausal.csv - -viash run src/metrics/regression_1/config.vsh.yaml -- --perturbation_data resources/grn-benchmark/perturbation_data.h5ad \ - --tf_all resources/prior/tf_all.csv \ - --prediction output/baseline_noncausal.csv \ - --score output/score_noncausal.h5ad \ No newline at end of file