Skip to content

Commit

Permalink
bug in util fixed
Browse files Browse the repository at this point in the history
  • Loading branch information
@janursa
janursa committed Sep 19, 2024
1 parent 1f61770 commit f15751f
Show file tree
Hide file tree
Showing 16 changed files with 112 additions and 128 deletions.
140 changes: 72 additions & 68 deletions runs.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -153,9 +153,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
"download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines/state.yaml to resources/results/benchmark_donor_0_baselines/state.yaml\n",
"download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines/scores.yaml to resources/results/benchmark_donor_0_baselines/scores.yaml\n",
"download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines/trace.txt to resources/results/benchmark_donor_0_baselines/trace.txt\n",
"download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines/scores.yaml to resources/results/benchmark_donor_0_baselines/scores.yaml\n",
"download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines/ridge.positive_control.positive_control.prediction.csv to resources/results/benchmark_donor_0_baselines/ridge.positive_control.positive_control.prediction.csv\n",
"download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines/ridge.pearson_corr.pearson_corr.prediction.csv to resources/results/benchmark_donor_0_baselines/ridge.pearson_corr.pearson_corr.prediction.csv\n",
"download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines/ridge.pearson_causal.pearson_causal.prediction.csv to resources/results/benchmark_donor_0_baselines/ridge.pearson_causal.pearson_causal.prediction.csv\n"
Expand All @@ -165,68 +164,68 @@
"data": {
"text/html": [
"<style type=\"text/css\">\n",
"#T_b33f6_row0_col0 {\n",
" background-color: #d0d1e6;\n",
"#T_05831_row0_col0, #T_05831_row0_col1, #T_05831_row0_col2, #T_05831_row0_col3 {\n",
" background-color: #023858;\n",
" color: #f1f1f1;\n",
"}\n",
"#T_05831_row1_col0 {\n",
" background-color: #cacee5;\n",
" color: #000000;\n",
"}\n",
"#T_b33f6_row0_col1, #T_b33f6_row0_col2, #T_b33f6_row0_col3, #T_b33f6_row2_col0 {\n",
"#T_05831_row1_col1, #T_05831_row1_col3, #T_05831_row2_col0, #T_05831_row2_col2 {\n",
" background-color: #fff7fb;\n",
" color: #000000;\n",
"}\n",
"#T_b33f6_row1_col0, #T_b33f6_row1_col1, #T_b33f6_row1_col2, #T_b33f6_row2_col3 {\n",
" background-color: #023858;\n",
" color: #f1f1f1;\n",
"#T_05831_row1_col2 {\n",
" background-color: #b8c6e0;\n",
" color: #000000;\n",
"}\n",
"#T_b33f6_row1_col3 {\n",
" background-color: #4295c3;\n",
"#T_05831_row2_col1 {\n",
" background-color: #1c7fb8;\n",
" color: #f1f1f1;\n",
"}\n",
"#T_b33f6_row2_col1 {\n",
" background-color: #04639b;\n",
"#T_05831_row2_col3 {\n",
" background-color: #1379b5;\n",
" color: #f1f1f1;\n",
"}\n",
"#T_b33f6_row2_col2 {\n",
" background-color: #fef6fb;\n",
" color: #000000;\n",
"}\n",
"</style>\n",
"<table id=\"T_b33f6\">\n",
"<table id=\"T_05831\">\n",
" <thead>\n",
" <tr>\n",
" <th class=\"blank level0\" >&nbsp;</th>\n",
" <th id=\"T_b33f6_level0_col0\" class=\"col_heading level0 col0\" >ex(False)_tf(-1)</th>\n",
" <th id=\"T_b33f6_level0_col1\" class=\"col_heading level0 col1\" >ex(True)_tf(-1)</th>\n",
" <th id=\"T_b33f6_level0_col2\" class=\"col_heading level0 col2\" >static-theta-0.0</th>\n",
" <th id=\"T_b33f6_level0_col3\" class=\"col_heading level0 col3\" >static-theta-0.5</th>\n",
" <th id=\"T_05831_level0_col0\" class=\"col_heading level0 col0\" >ex(False)_tf(-1)</th>\n",
" <th id=\"T_05831_level0_col1\" class=\"col_heading level0 col1\" >ex(True)_tf(-1)</th>\n",
" <th id=\"T_05831_level0_col2\" class=\"col_heading level0 col2\" >static-theta-0.0</th>\n",
" <th id=\"T_05831_level0_col3\" class=\"col_heading level0 col3\" >static-theta-0.5</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th id=\"T_b33f6_level0_row0\" class=\"row_heading level0 row0\" >pearson_corr</th>\n",
" <td id=\"T_b33f6_row0_col0\" class=\"data row0 col0\" >0.239620</td>\n",
" <td id=\"T_b33f6_row0_col1\" class=\"data row0 col1\" >0.518217</td>\n",
" <td id=\"T_b33f6_row0_col2\" class=\"data row0 col2\" >0.529502</td>\n",
" <td id=\"T_b33f6_row0_col3\" class=\"data row0 col3\" >0.524232</td>\n",
" <th id=\"T_05831_level0_row0\" class=\"row_heading level0 row0\" >pearson_corr</th>\n",
" <td id=\"T_05831_row0_col0\" class=\"data row0 col0\" >0.286728</td>\n",
" <td id=\"T_05831_row0_col1\" class=\"data row0 col1\" >0.543611</td>\n",
" <td id=\"T_05831_row0_col2\" class=\"data row0 col2\" >0.739006</td>\n",
" <td id=\"T_05831_row0_col3\" class=\"data row0 col3\" >0.553125</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_b33f6_level0_row1\" class=\"row_heading level0 row1\" >pearson_causal</th>\n",
" <td id=\"T_b33f6_row1_col0\" class=\"data row1 col0\" >0.364656</td>\n",
" <td id=\"T_b33f6_row1_col1\" class=\"data row1 col1\" >0.592457</td>\n",
" <td id=\"T_b33f6_row1_col2\" class=\"data row1 col2\" >0.741328</td>\n",
" <td id=\"T_b33f6_row1_col3\" class=\"data row1 col3\" >0.560490</td>\n",
" <th id=\"T_05831_level0_row1\" class=\"row_heading level0 row1\" >pearson_causal</th>\n",
" <td id=\"T_05831_row1_col0\" class=\"data row1 col0\" >0.152208</td>\n",
" <td id=\"T_05831_row1_col1\" class=\"data row1 col1\" >0.436537</td>\n",
" <td id=\"T_05831_row1_col2\" class=\"data row1 col2\" >0.632154</td>\n",
" <td id=\"T_05831_row1_col3\" class=\"data row1 col3\" >0.520246</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_b33f6_level0_row2\" class=\"row_heading level0 row2\" >positive_control</th>\n",
" <td id=\"T_b33f6_row2_col0\" class=\"data row2 col0\" >0.197307</td>\n",
" <td id=\"T_b33f6_row2_col1\" class=\"data row2 col1\" >0.579238</td>\n",
" <td id=\"T_b33f6_row2_col2\" class=\"data row2 col2\" >0.530848</td>\n",
" <td id=\"T_b33f6_row2_col3\" class=\"data row2 col3\" >0.584694</td>\n",
" <th id=\"T_05831_level0_row2\" class=\"row_heading level0 row2\" >positive_control</th>\n",
" <td id=\"T_05831_row2_col0\" class=\"data row2 col0\" >0.103243</td>\n",
" <td id=\"T_05831_row2_col1\" class=\"data row2 col1\" >0.510231</td>\n",
" <td id=\"T_05831_row2_col2\" class=\"data row2 col2\" >0.581397</td>\n",
" <td id=\"T_05831_row2_col3\" class=\"data row2 col3\" >0.543740</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n"
],
"text/plain": [
"<pandas.io.formats.style.Styler at 0x7fe34e74ba00>"
"<pandas.io.formats.style.Styler at 0x14878cfcc430>"
]
},
"execution_count": 3,
Expand All @@ -250,85 +249,90 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines_nonspecific_notnormalized/trace.txt to resources/results/benchmark_donor_0_baselines_nonspecific_notnormalized/trace.txt\n"
"download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines_nonspecific_notnormalized/trace.txt to resources/results/benchmark_donor_0_baselines_nonspecific_notnormalized/trace.txt\n",
"download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines_nonspecific_notnormalized/scores.yaml to resources/results/benchmark_donor_0_baselines_nonspecific_notnormalized/scores.yaml\n",
"download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines_nonspecific_notnormalized/state.yaml to resources/results/benchmark_donor_0_baselines_nonspecific_notnormalized/state.yaml\n",
"download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines_nonspecific_notnormalized/ridge.pearson_causal.pearson_causal.prediction.csv to resources/results/benchmark_donor_0_baselines_nonspecific_notnormalized/ridge.pearson_causal.pearson_causal.prediction.csv\n",
"download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines_nonspecific_notnormalized/ridge.positive_control.positive_control.prediction.csv to resources/results/benchmark_donor_0_baselines_nonspecific_notnormalized/ridge.positive_control.positive_control.prediction.csv\n",
"download: s3://openproblems-data/resources/grn/results/benchmark_donor_0_baselines_nonspecific_notnormalized/ridge.pearson_corr.pearson_corr.prediction.csv to resources/results/benchmark_donor_0_baselines_nonspecific_notnormalized/ridge.pearson_corr.pearson_corr.prediction.csv\n"
]
},
{
"data": {
"text/html": [
"<style type=\"text/css\">\n",
"#T_32bb3_row0_col0, #T_32bb3_row0_col1, #T_32bb3_row0_col2, #T_32bb3_row0_col3 {\n",
"#T_29efd_row0_col0, #T_29efd_row0_col1, #T_29efd_row0_col2, #T_29efd_row0_col3 {\n",
" background-color: #023858;\n",
" color: #f1f1f1;\n",
"}\n",
"#T_32bb3_row1_col0 {\n",
"#T_29efd_row1_col0 {\n",
" background-color: #cacee5;\n",
" color: #000000;\n",
"}\n",
"#T_32bb3_row1_col1, #T_32bb3_row1_col3, #T_32bb3_row2_col0, #T_32bb3_row2_col2 {\n",
"#T_29efd_row1_col1, #T_29efd_row1_col3, #T_29efd_row2_col0, #T_29efd_row2_col2 {\n",
" background-color: #fff7fb;\n",
" color: #000000;\n",
"}\n",
"#T_32bb3_row1_col2 {\n",
"#T_29efd_row1_col2 {\n",
" background-color: #b8c6e0;\n",
" color: #000000;\n",
"}\n",
"#T_32bb3_row2_col1 {\n",
"#T_29efd_row2_col1 {\n",
" background-color: #1c7fb8;\n",
" color: #f1f1f1;\n",
"}\n",
"#T_32bb3_row2_col3 {\n",
"#T_29efd_row2_col3 {\n",
" background-color: #1379b5;\n",
" color: #f1f1f1;\n",
"}\n",
"</style>\n",
"<table id=\"T_32bb3\">\n",
"<table id=\"T_29efd\">\n",
" <thead>\n",
" <tr>\n",
" <th class=\"blank level0\" >&nbsp;</th>\n",
" <th id=\"T_32bb3_level0_col0\" class=\"col_heading level0 col0\" >ex(False)_tf(-1)</th>\n",
" <th id=\"T_32bb3_level0_col1\" class=\"col_heading level0 col1\" >ex(True)_tf(-1)</th>\n",
" <th id=\"T_32bb3_level0_col2\" class=\"col_heading level0 col2\" >static-theta-0.0</th>\n",
" <th id=\"T_32bb3_level0_col3\" class=\"col_heading level0 col3\" >static-theta-0.5</th>\n",
" <th id=\"T_29efd_level0_col0\" class=\"col_heading level0 col0\" >ex(False)_tf(-1)</th>\n",
" <th id=\"T_29efd_level0_col1\" class=\"col_heading level0 col1\" >ex(True)_tf(-1)</th>\n",
" <th id=\"T_29efd_level0_col2\" class=\"col_heading level0 col2\" >static-theta-0.0</th>\n",
" <th id=\"T_29efd_level0_col3\" class=\"col_heading level0 col3\" >static-theta-0.5</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th id=\"T_32bb3_level0_row0\" class=\"row_heading level0 row0\" >pearson_corr</th>\n",
" <td id=\"T_32bb3_row0_col0\" class=\"data row0 col0\" >0.286728</td>\n",
" <td id=\"T_32bb3_row0_col1\" class=\"data row0 col1\" >0.543611</td>\n",
" <td id=\"T_32bb3_row0_col2\" class=\"data row0 col2\" >0.739006</td>\n",
" <td id=\"T_32bb3_row0_col3\" class=\"data row0 col3\" >0.553125</td>\n",
" <th id=\"T_29efd_level0_row0\" class=\"row_heading level0 row0\" >pearson_corr</th>\n",
" <td id=\"T_29efd_row0_col0\" class=\"data row0 col0\" >0.286728</td>\n",
" <td id=\"T_29efd_row0_col1\" class=\"data row0 col1\" >0.543611</td>\n",
" <td id=\"T_29efd_row0_col2\" class=\"data row0 col2\" >0.739006</td>\n",
" <td id=\"T_29efd_row0_col3\" class=\"data row0 col3\" >0.553125</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_32bb3_level0_row1\" class=\"row_heading level0 row1\" >pearson_causal</th>\n",
" <td id=\"T_32bb3_row1_col0\" class=\"data row1 col0\" >0.152208</td>\n",
" <td id=\"T_32bb3_row1_col1\" class=\"data row1 col1\" >0.436537</td>\n",
" <td id=\"T_32bb3_row1_col2\" class=\"data row1 col2\" >0.632154</td>\n",
" <td id=\"T_32bb3_row1_col3\" class=\"data row1 col3\" >0.520246</td>\n",
" <th id=\"T_29efd_level0_row1\" class=\"row_heading level0 row1\" >pearson_causal</th>\n",
" <td id=\"T_29efd_row1_col0\" class=\"data row1 col0\" >0.152208</td>\n",
" <td id=\"T_29efd_row1_col1\" class=\"data row1 col1\" >0.436537</td>\n",
" <td id=\"T_29efd_row1_col2\" class=\"data row1 col2\" >0.632154</td>\n",
" <td id=\"T_29efd_row1_col3\" class=\"data row1 col3\" >0.520246</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_32bb3_level0_row2\" class=\"row_heading level0 row2\" >positive_control</th>\n",
" <td id=\"T_32bb3_row2_col0\" class=\"data row2 col0\" >0.103243</td>\n",
" <td id=\"T_32bb3_row2_col1\" class=\"data row2 col1\" >0.510231</td>\n",
" <td id=\"T_32bb3_row2_col2\" class=\"data row2 col2\" >0.581397</td>\n",
" <td id=\"T_32bb3_row2_col3\" class=\"data row2 col3\" >0.543740</td>\n",
" <th id=\"T_29efd_level0_row2\" class=\"row_heading level0 row2\" >positive_control</th>\n",
" <td id=\"T_29efd_row2_col0\" class=\"data row2 col0\" >0.103243</td>\n",
" <td id=\"T_29efd_row2_col1\" class=\"data row2 col1\" >0.510231</td>\n",
" <td id=\"T_29efd_row2_col2\" class=\"data row2 col2\" >0.581397</td>\n",
" <td id=\"T_29efd_row2_col3\" class=\"data row2 col3\" >0.543740</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n"
],
"text/plain": [
"<pandas.io.formats.style.Styler at 0x7fe34e7a9840>"
"<pandas.io.formats.style.Styler at 0x14878cd6a3e0>"
]
},
"execution_count": 2,
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
Expand Down
36 changes: 18 additions & 18 deletions scripts/run_benchmark_all.sh
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
#!/bin/bash

# RUN_ID="run_$(date +%Y-%m-%d_%H-%M-%S)"
RUN_ID="benchmark_donor_0_baselines"
resources_dir="./resources/"
# resources_dir="s3://openproblems-data/resources/grn"
RUN_ID="benchmark_donor_0_baselines_nonspecific_notnormalized"
# resources_dir="./resources/"
resources_dir="s3://openproblems-data/resources/grn"
publish_dir="${resources_dir}/results/${RUN_ID}"

reg_type=ridge
Expand All @@ -12,7 +12,7 @@ max_workers=10
layer='scgen_pearson'
metric_ids="[regression_1, regression_2]"
cell_type_specific=false #for controls
normalize=true
normalize=false
only_hvgs=false
# method_ids="[tigress, ennet, scsgl, pidc]"
method_ids="[pearson_corr, pearson_causal, positive_control]"
Expand Down Expand Up @@ -42,12 +42,12 @@ output_state: "state.yaml"
publish_dir: "$publish_dir"
HERE

nextflow run . \
-main-script target/nextflow/workflows/run_benchmark/main.nf \
-profile docker \
-with-trace \
-c src/common/nextflow_helpers/labels_ci.config \
-params-file ${param_file}
# nextflow run . \
# -main-script target/nextflow/workflows/run_benchmark/main.nf \
# -profile docker \
# -with-trace \
# -c src/common/nextflow_helpers/labels_ci.config \
# -params-file ${param_file}

# ./tw-windows-x86_64.exe launch `
# https://github.com/openproblems-bio/task_grn_inference.git `
Expand All @@ -59,11 +59,11 @@ nextflow run . \
# --params-file ./params/benchmark_donor_0_default.yaml `
# --config src/common/nextflow_helpers/labels_tw.config

# ./tw launch https://github.com/openproblems-bio/task_grn_inference \
# --revision build/main \
# --pull-latest \
# --main-script target/nextflow/workflows/run_benchmark/main.nf \
# --workspace 53907369739130 \
# --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
# --params-file ${param_file} \
# --config src/common/nextflow_helpers/labels_tw.config
./tw launch https://github.com/openproblems-bio/task_grn_inference \
--revision build/main \
--pull-latest \
--main-script target/nextflow/workflows/run_benchmark/main.nf \
--workspace 53907369739130 \
--compute-env 6TeIFgV5OY4pJCk8I0bfOh \
--params-file ${param_file} \
--config src/common/nextflow_helpers/labels_tw.config
2 changes: 1 addition & 1 deletion src/api/comp_metric.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ functionality:
direction: input
default: -2
description: number of samples randomly drawn from perturbation data
- name: --max_workers
- name: --num_workers
type: integer
direction: input
default: 4
Expand Down
2 changes: 1 addition & 1 deletion src/metrics/regression_1/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ def main(par):
# net = net.groupby(['source', 'target']).mean().reset_index()

subsample = par['subsample']
max_workers = par['max_workers']
max_workers = par['num_workers']
layer = par["layer"]
if subsample == -1:
pass
Expand Down
2 changes: 1 addition & 1 deletion src/metrics/regression_1/script.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
'reg_type': 'ridge',
'layer': 'scgen_pearson',
'subsample': -2,
'max_workers': 4,
'num_workers': 4,
}
## VIASH END
# meta = {
Expand Down
6 changes: 3 additions & 3 deletions src/metrics/regression_2/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -316,11 +316,11 @@ def main(par: Dict[str, Any]) -> pd.DataFrame:
print(f'Compute metrics for layer: {layer}', flush=True)
# print(f'Dynamic approach:', flush=True)
print(f'Static approach (theta=0):', flush=True)
score_static_min = static_approach(grn, n_features_theta_min, X, groups, gene_names, tf_names, par['reg_type'], n_jobs=par['max_workers'], n_features_dict=n_features_dict, clip_scores=clip_scores)
score_static_min = static_approach(grn, n_features_theta_min, X, groups, gene_names, tf_names, par['reg_type'], n_jobs=par['num_workers'], n_features_dict=n_features_dict, clip_scores=clip_scores)
print(f'Static approach (theta=0.5):', flush=True)
score_static_median = static_approach(grn, n_features_theta_median, X, groups, gene_names, tf_names, par['reg_type'], n_jobs=par['max_workers'], n_features_dict=n_features_dict, clip_scores=clip_scores)
score_static_median = static_approach(grn, n_features_theta_median, X, groups, gene_names, tf_names, par['reg_type'], n_jobs=par['num_workers'], n_features_dict=n_features_dict, clip_scores=clip_scores)
# print(f'Static approach (theta=1):', flush=True)
# score_static_max = static_approach(grn, n_features_theta_max, X, groups, gene_names, tf_names, par['reg_type'], n_jobs=par['max_workers'])
# score_static_max = static_approach(grn, n_features_theta_max, X, groups, gene_names, tf_names, par['reg_type'], n_jobs=par['num_workers'])
# TODO: find a mathematically sound way to combine Z-scores and r2 scores

results = {
Expand Down
2 changes: 1 addition & 1 deletion src/metrics/regression_2/script.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
'static_only': True,
'layer': 'scgen_pearson',
'subsample': -2,
'max_workers': 4,
'num_workers': 4,
'apply_tf': True,
'clip_scores': True,
'method_id': 'grnboost'
Expand Down
Loading

0 comments on commit f15751f

Please sign in to comment.