diff --git a/runs.ipynb b/runs.ipynb index e433b03ee..988d4d6ae 100644 --- a/runs.ipynb +++ b/runs.ipynb @@ -16,140 +16,9 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "upload: resources/datasets_raw/adamson_bulked.h5ad to s3://openproblems-data/resources/grn/datasets_raw/adamson_bulked.h5ad\n", - "upload: resources/datasets_raw/norman_bulked.h5ad to s3://openproblems-data/resources/grn/datasets_raw/norman_bulked.h5ad\n", - "delete: s3://openproblems-data/resources/grn/datasets_raw/op_perturbation_counts.h5ad \n", - "upload: resources/datasets_raw/nakatake_bulked.h5ad to s3://openproblems-data/resources/grn/datasets_raw/nakatake_bulked.h5ad\n", - "upload: resources/datasets_raw/op_bulked.h5ad to s3://openproblems-data/resources/grn/datasets_raw/op_bulked.h5ad\n", - "delete: s3://openproblems-data/resources/grn/datasets_raw/replogle2.h5ad \n", - "upload: resources/datasets_raw/adamson_sc_counts.h5ad to s3://openproblems-data/resources/grn/datasets_raw/adamson_sc_counts.h5ad\n", - "upload: resources/evaluation_datasets/adamson_perturbation.h5ad to s3://openproblems-data/resources/grn/evaluation_datasets/adamson_perturbation.h5ad\n", - "upload: resources/evaluation_datasets/nakatake_perturbation.h5ad to s3://openproblems-data/resources/grn/evaluation_datasets/nakatake_perturbation.h5ad\n", - "upload: resources/evaluation_datasets/norman_perturbation.h5ad to s3://openproblems-data/resources/grn/evaluation_datasets/norman_perturbation.h5ad\n", - "upload: resources/evaluation_datasets/replogle2_perturbation.h5ad to s3://openproblems-data/resources/grn/evaluation_datasets/replogle2_perturbation.h5ad\n", - "upload: resources/grn_models/adamson/grnboost2_0.5.csv to s3://openproblems-data/resources/grn/grn_models/adamson/grnboost2_0.5.csv\n", - "upload: resources/grn_models/adamson/grnboost2_1.0.csv to s3://openproblems-data/resources/grn/grn_models/adamson/grnboost2_1.0.csv\n", - "upload: resources/grn_models/adamson/pearson_corr_0.2.csv to s3://openproblems-data/resources/grn/grn_models/adamson/pearson_corr_0.2.csv\n", - "upload: resources/grn_models/adamson/pearson_corr_0.5.csv to s3://openproblems-data/resources/grn/grn_models/adamson/pearson_corr_0.5.csv\n", - "upload: resources/grn_models/adamson/pearson_corr_1.0.csv to s3://openproblems-data/resources/grn/grn_models/adamson/pearson_corr_1.0.csv\n", - "upload: resources/grn_models/adamson/portia_0.2.csv to s3://openproblems-data/resources/grn/grn_models/adamson/portia_0.2.csv\n", - "upload: resources/grn_models/adamson/portia_0.5.csv to s3://openproblems-data/resources/grn/grn_models/adamson/portia_0.5.csv\n", - "upload: resources/grn_models/adamson/portia_1.0.csv to s3://openproblems-data/resources/grn/grn_models/adamson/portia_1.0.csv\n", - "upload: resources/grn_models/global/Ananse:Bone marrow.csv to s3://openproblems-data/resources/grn/grn_models/global/Ananse:Bone marrow.csv\n", - "upload: resources/grn_models/global/Ananse:Heart.csv to s3://openproblems-data/resources/grn/grn_models/global/Ananse:Heart.csv\n", - "upload: resources/grn_models/global/Ananse:Lung.csv to s3://openproblems-data/resources/grn/grn_models/global/Ananse:Lung.csv\n", - "upload: resources/grn_models/global/Ananse:Stomach.csv to s3://openproblems-data/resources/grn/grn_models/global/Ananse:Stomach.csv\n", - "upload: resources/grn_models/global/Cellnet:Bcell.csv to s3://openproblems-data/resources/grn/grn_models/global/Cellnet:Bcell.csv\n", - "upload: resources/grn_models/global/Cellnet:Heart.csv to s3://openproblems-data/resources/grn/grn_models/global/Cellnet:Heart.csv\n", - "upload: resources/grn_models/global/Cellnet:Neuron.csv to s3://openproblems-data/resources/grn/grn_models/global/Cellnet:Neuron.csv\n", - "upload: resources/grn_models/global/Cellnet:Skin.csv to s3://openproblems-data/resources/grn/grn_models/global/Cellnet:Skin.csv\n", - "upload: resources/grn_models/global/Cellnet:Tcell.csv to s3://openproblems-data/resources/grn/grn_models/global/Cellnet:Tcell.csv\n", - "upload: resources/grn_models/global/Gtex:Brain amygdala.csv to s3://openproblems-data/resources/grn/grn_models/global/Gtex:Brain amygdala.csv\n", - "upload: resources/grn_models/global/Gtex:Breast mammary tissue.csv to s3://openproblems-data/resources/grn/grn_models/global/Gtex:Breast mammary tissue.csv\n", - "upload: resources/datasets_raw/replogle2_bulked.h5ad to s3://openproblems-data/resources/grn/datasets_raw/replogle2_bulked.h5ad\n", - "upload: resources/grn_models/global/Gtex:Lung.csv to s3://openproblems-data/resources/grn/grn_models/global/Gtex:Lung.csv\n", - "upload: resources/grn_models/global/collectri.csv to s3://openproblems-data/resources/grn/grn_models/global/collectri.csv\n", - "upload: resources/grn_models/global/Gtex:Stomach.csv to s3://openproblems-data/resources/grn/grn_models/global/Gtex:Stomach.csv\n", - "upload: resources/grn_models/global/Gtex:Whole blood.csv to s3://openproblems-data/resources/grn/grn_models/global/Gtex:Whole blood.csv\n", - "upload: resources/grn_models/nakatake/grnboost2_0.2.csv to s3://openproblems-data/resources/grn/grn_models/nakatake/grnboost2_0.2.csv\n", - "upload: resources/grn_models/nakatake/grnboost2_0.5.csv to s3://openproblems-data/resources/grn/grn_models/nakatake/grnboost2_0.5.csv\n", - "upload: resources/grn_models/nakatake/grnboost2_1.0.csv to s3://openproblems-data/resources/grn/grn_models/nakatake/grnboost2_1.0.csv\n", - "upload: resources/grn_models/nakatake/pearson_corr_0.2.csv to s3://openproblems-data/resources/grn/grn_models/nakatake/pearson_corr_0.2.csv\n", - "upload: resources/grn_models/nakatake/pearson_corr_0.5.csv to s3://openproblems-data/resources/grn/grn_models/nakatake/pearson_corr_0.5.csv\n", - "upload: resources/grn_models/nakatake/pearson_corr_1.0.csv to s3://openproblems-data/resources/grn/grn_models/nakatake/pearson_corr_1.0.csv\n", - "upload: resources/grn_models/nakatake/portia_0.2.csv to s3://openproblems-data/resources/grn/grn_models/nakatake/portia_0.2.csv\n", - "upload: resources/grn_models/nakatake/portia_0.5.csv to s3://openproblems-data/resources/grn/grn_models/nakatake/portia_0.5.csv\n", - "upload: resources/grn_models/nakatake/portia_1.0.csv to s3://openproblems-data/resources/grn/grn_models/nakatake/portia_1.0.csv\n", - "upload: resources/grn_models/norman/pearson_corr.csv to s3://openproblems-data/resources/grn/grn_models/norman/pearson_corr.csv\n", - "upload: resources/grn_models/norman/pearson_corr_0.2.csv to s3://openproblems-data/resources/grn/grn_models/norman/pearson_corr_0.2.csv\n", - "upload: resources/grn_models/norman/pearson_corr_0.5.csv to s3://openproblems-data/resources/grn/grn_models/norman/pearson_corr_0.5.csv\n", - "upload: resources/grn_models/norman/portia_0.2.csv to s3://openproblems-data/resources/grn/grn_models/norman/portia_0.2.csv\n", - "upload: resources/grn_models/norman/pearson_corr_1.0.csv to s3://openproblems-data/resources/grn/grn_models/norman/pearson_corr_1.0.csv\n", - "upload: resources/grn_models/norman/portia_1.0.csv to s3://openproblems-data/resources/grn/grn_models/norman/portia_1.0.csv\n", - "upload: resources/grn_models/norman/portia_0.5.csv to s3://openproblems-data/resources/grn/grn_models/norman/portia_0.5.csv\n", - "upload: resources/grn_models/norman/scenic.csv to s3://openproblems-data/resources/grn/grn_models/norman/scenic.csv\n", - "delete: s3://openproblems-data/resources/grn/grn_models/op/collectri.csv\n", - "upload: resources/grn_models/op/pearson_corr_1.csv to s3://openproblems-data/resources/grn/grn_models/op/pearson_corr_1.csv\n", - "upload: resources/grn_models/op/pearson_corr_2.csv to s3://openproblems-data/resources/grn/grn_models/op/pearson_corr_2.csv\n", - "upload: resources/grn_models/op/portia_1.csv to s3://openproblems-data/resources/grn/grn_models/op/portia_1.csv\n", - "upload: resources/grn_models/op/portia_2.csv to s3://openproblems-data/resources/grn/grn_models/op/portia_2.csv\n", - "upload: resources/grn_models/replogle2/grnboost2_0.2.csv to s3://openproblems-data/resources/grn/grn_models/replogle2/grnboost2_0.2.csv\n", - "upload: resources/grn_models/replogle2/grnboost2_0.5.csv to s3://openproblems-data/resources/grn/grn_models/replogle2/grnboost2_0.5.csv\n", - "upload: resources/grn_models/replogle2/grnboost2_1.0.csv to s3://openproblems-data/resources/grn/grn_models/replogle2/grnboost2_1.0.csv\n", - "upload: resources/grn_models/replogle2/pearson_corr_0.5.csv to s3://openproblems-data/resources/grn/grn_models/replogle2/pearson_corr_0.5.csv\n", - "upload: resources/grn_models/replogle2/pearson_corr_0.2.csv to s3://openproblems-data/resources/grn/grn_models/replogle2/pearson_corr_0.2.csv\n", - "upload: resources/grn_models/replogle2/pearson_corr_1.0.csv to s3://openproblems-data/resources/grn/grn_models/replogle2/pearson_corr_1.0.csv\n", - "upload: resources/grn_models/replogle2/portia_0.2.csv to s3://openproblems-data/resources/grn/grn_models/replogle2/portia_0.2.csv\n", - "upload: resources/grn_models/replogle2/portia_0.5.csv to s3://openproblems-data/resources/grn/grn_models/replogle2/portia_0.5.csv\n", - "upload: resources/grn_models/replogle2/portia_1.0.csv to s3://openproblems-data/resources/grn/grn_models/replogle2/portia_1.0.csv\n", - "upload: resources/datasets_raw/norman_sc_counts.h5ad to s3://openproblems-data/resources/grn/datasets_raw/norman_sc_counts.h5ad\n", - "upload: resources/inference_datasets/nakatake_rna.h5ad to s3://openproblems-data/resources/grn/inference_datasets/nakatake_rna.h5ad\n", - "upload: resources/inference_datasets/adamson_rna.h5ad to s3://openproblems-data/resources/grn/inference_datasets/adamson_rna.h5ad\n", - "delete: s3://openproblems-data/resources/grn/prior/adamson_consensus-num-regulators.json\n", - "delete: s3://openproblems-data/resources/grn/prior/frangieh_IFNg_v2_consensus-num-regulators.json\n", - "delete: s3://openproblems-data/resources/grn/prior/nakatake_consensus-num-regulators.json\n", - "delete: s3://openproblems-data/resources/grn/prior/norman_consensus-num-regulators.json\n", - "delete: s3://openproblems-data/resources/grn/prior/op_consensus-num-regulators.json\n", - "delete: s3://openproblems-data/resources/grn/prior/op_consensus-num-regulators_global.json\n", - "upload: resources/prior/regulators_consensus_adamson.json to s3://openproblems-data/resources/grn/prior/regulators_consensus_adamson.json\n", - "upload: resources/prior/regulators_consensus_nakatake.json to s3://openproblems-data/resources/grn/prior/regulators_consensus_nakatake.json\n", - "upload: resources/prior/regulators_consensus_norman.json to s3://openproblems-data/resources/grn/prior/regulators_consensus_norman.json\n", - "upload: resources/prior/regulators_consensus_op.json to s3://openproblems-data/resources/grn/prior/regulators_consensus_op.json\n", - "upload: resources/prior/regulators_consensus_replogle2.json to s3://openproblems-data/resources/grn/prior/regulators_consensus_replogle2.json\n", - "delete: s3://openproblems-data/resources/grn/prior/replogle2_consensus-num-regulators.json\n", - "delete: s3://openproblems-data/resources/grn/prior/skeleton_promotor.csv\n", - "upload: resources/prior/ws_consensus_adamson.csv to s3://openproblems-data/resources/grn/prior/ws_consensus_adamson.csv\n", - "upload: resources/prior/ws_consensus_norman.csv to s3://openproblems-data/resources/grn/prior/ws_consensus_norman.csv\n", - "upload: resources/prior/ws_distance_background_adamson.csv to s3://openproblems-data/resources/grn/prior/ws_distance_background_adamson.csv\n", - "upload: resources/results/figs/centrality_in_short.png to s3://openproblems-data/resources/grn/results/figs/centrality_in_short.png\n", - "upload: resources/results/figs/consensus_ws.png to s3://openproblems-data/resources/grn/results/figs/consensus_ws.png\n", - "upload: resources/prior/ws_distance_background_norman.csv to s3://openproblems-data/resources/grn/prior/ws_distance_background_norman.csv\n", - "upload: resources/results/figs/evaluation_scores_global_models.png to s3://openproblems-data/resources/grn/results/figs/evaluation_scores_global_models.png\n", - "upload: resources/results/figs/evaluation_scores_norman_adamson.png to s3://openproblems-data/resources/grn/results/figs/evaluation_scores_norman_adamson.png\n", - "upload: resources/datasets_raw/op_multiome_sc_counts.h5ad to s3://openproblems-data/resources/grn/datasets_raw/op_multiome_sc_counts.h5ad\n", - "upload: resources/results/figs/evaluation_scores_opsca.png to s3://openproblems-data/resources/grn/results/figs/evaluation_scores_opsca.png\n", - "upload: resources/results/figs/evaluation_scores_other_datasets.png to s3://openproblems-data/resources/grn/results/figs/evaluation_scores_other_datasets.png\n", - "upload: resources/results/figs/gb_vs_ridge.png to s3://openproblems-data/resources/grn/results/figs/gb_vs_ridge.png\n", - "upload: resources/results/figs/perturbation_strength_datasets.png to s3://openproblems-data/resources/grn/results/figs/perturbation_strength_datasets.png\n", - "upload: resources/results/figs/raw_scores_norman_adamson.png to s3://openproblems-data/resources/grn/results/figs/raw_scores_norman_adamson.png\n", - "upload: resources/results/figs/raw_scores_others.png to s3://openproblems-data/resources/grn/results/figs/raw_scores_others.png\n", - "upload: resources/results/figs/relative_scores_all.png to s3://openproblems-data/resources/grn/results/figs/relative_scores_all.png\n", - "upload: resources/results/figs/robustnes_analysis.png to s3://openproblems-data/resources/grn/results/figs/robustnes_analysis.png\n", - "upload: resources/results/figs/robustnes_analysis_reg1_1.png to s3://openproblems-data/resources/grn/results/figs/robustnes_analysis_reg1_1.png\n", - "upload: resources/results/figs/robustness_reg1.png to s3://openproblems-data/resources/grn/results/figs/robustness_reg1.png\n", - "upload: resources/results/figs/robustness_reg2.png to s3://openproblems-data/resources/grn/results/figs/robustness_reg2.png\n", - "upload: resources/results/figs/skelton_guided_barplot.png to s3://openproblems-data/resources/grn/results/figs/skelton_guided_barplot.png\n", - "upload: resources/results/figs/summary_figure.pdf to s3://openproblems-data/resources/grn/results/figs/summary_figure.pdf\n", - "upload: resources/results/figs/summary_figure.png to s3://openproblems-data/resources/grn/results/figs/summary_figure.png\n", - "upload: resources/results/figs/tf_guided_barplot.png to s3://openproblems-data/resources/grn/results/figs/tf_guided_barplot.png\n", - "upload: resources/results/figs/topology_stats.png to s3://openproblems-data/resources/grn/results/figs/topology_stats.png\n", - "upload: resources/results/figs/topology_stats_short.png to s3://openproblems-data/resources/grn/results/figs/topology_stats_short.png\n", - "upload: resources/results/summary.tsv to s3://openproblems-data/resources/grn/results/summary.tsv\n", - "upload: resources/results/summary_figure.pdf to s3://openproblems-data/resources/grn/results/summary_figure.pdf\n", - "upload: resources/results/summary_figure.png to s3://openproblems-data/resources/grn/results/summary_figure.png\n", - "upload: resources/scores/adamson/subsampled.csv to s3://openproblems-data/resources/grn/scores/adamson/subsampled.csv\n", - "upload: resources/scores/default_scores.csv to s3://openproblems-data/resources/grn/scores/default_scores.csv\n", - "upload: resources/scores/default_scores_short.csv to s3://openproblems-data/resources/grn/scores/default_scores_short.csv\n", - "upload: resources/scores/nakatake/subsampled.csv to s3://openproblems-data/resources/grn/scores/nakatake/subsampled.csv\n", - "upload: resources/scores/norman/subsampled.csv to s3://openproblems-data/resources/grn/scores/norman/subsampled.csv\n", - "upload: resources/scores/op/subsampled.csv to s3://openproblems-data/resources/grn/scores/op/subsampled.csv\n", - "upload: resources/scores/replogle2/X_norm-50000-skeleton_False-binarize_False-ridge-global-False.csv to s3://openproblems-data/resources/grn/scores/replogle2/X_norm-50000-skeleton_False-binarize_False-ridge-global-False.csv\n", - "upload: resources/scores/replogle2/subsampled.csv to s3://openproblems-data/resources/grn/scores/replogle2/subsampled.csv\n", - "upload: resources/scores/ws_distance_mean.csv to s3://openproblems-data/resources/grn/scores/ws_distance_mean.csv\n", - "upload: resources/scores/ws_distance.csv to s3://openproblems-data/resources/grn/scores/ws_distance.csv\n", - "upload: resources/inference_datasets/replogle2_rna.h5ad to s3://openproblems-data/resources/grn/inference_datasets/replogle2_rna.h5ad\n", - "upload: resources/inference_datasets/norman_rna.h5ad to s3://openproblems-data/resources/grn/inference_datasets/norman_rna.h5ad\n", - "upload: resources/datasets_raw/op_perturbation_sc_counts.h5ad to s3://openproblems-data/resources/grn/datasets_raw/op_perturbation_sc_counts.h5ad\n" - ] - } - ], + "outputs": [], "source": [ - "!aws s3 sync resources/ s3://openproblems-data/resources/grn/ --delete\n", + "# !aws s3 sync resources/ s3://openproblems-data/resources/grn/ --delete\n", "# !aws s3 sync resources/grn_models/ s3://openproblems-data/resources/grn/grn_models --delete\n", "# !aws s3 sync resources/prior/ s3://openproblems-data/resources/grn/prior --delete\n", "# !aws s3 sync resources/results/ s3://openproblems-data/resources/grn/results --delete\n", @@ -160,19 +29,140 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "delete: resources/datasets_raw/op_bulked.h5ad.dde839C8\n", + "delete: resources/datasets_raw/op_multiome_sc_counts.h5ad.DDCAB9FD \n", + "delete: resources/results/.DS_Store \n", + "delete: resources/results/figs/.DS_Store \n", + "delete: resources/results/figs/consensus_regulators.png \n", + "delete: resources/results/figs/raw_scores_one.png \n", + "delete: resources/results/figs/raw_scores_three.png \n", + "delete: resources/results/figs/similaritys_donors.png \n", + "delete: resources/results/figs/topology_stats_1.png \n", + "delete: resources/results/figs/topology_stats_2.png \n", + "download: s3://openproblems-data/resources/grn/datasets_raw/nakatake_bulked.h5ad to resources/datasets_raw/nakatake_bulked.h5ad\n", + "download: s3://openproblems-data/resources/grn/datasets_raw/adamson_sc_counts.h5ad to resources/datasets_raw/adamson_sc_counts.h5ad\n", + "download: s3://openproblems-data/resources/grn/datasets_raw/op_multiome_sc_counts.h5ad to resources/datasets_raw/op_multiome_sc_counts.h5ad\n", + "download: s3://openproblems-data/resources/grn/evaluation_datasets/norman_perturbation.h5ad to resources/evaluation_datasets/norman_perturbation.h5ad\n", + "download: s3://openproblems-data/resources/grn/grn_models/adamson/grnboost2_0.5.csv to resources/grn_models/adamson/grnboost2_0.5.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/adamson/grnboost2_1.0.csv to resources/grn_models/adamson/grnboost2_1.0.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/adamson/pearson_corr_0.2.csv to resources/grn_models/adamson/pearson_corr_0.2.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/adamson/pearson_corr_0.5.csv to resources/grn_models/adamson/pearson_corr_0.5.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/adamson/pearson_corr_1.0.csv to resources/grn_models/adamson/pearson_corr_1.0.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/adamson/portia_0.2.csv to resources/grn_models/adamson/portia_0.2.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/adamson/portia_0.5.csv to resources/grn_models/adamson/portia_0.5.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/adamson/portia_1.0.csv to resources/grn_models/adamson/portia_1.0.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/global/Ananse:Bone marrow.csv to resources/grn_models/global/Ananse:Bone marrow.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/global/Ananse:Heart.csv to resources/grn_models/global/Ananse:Heart.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/global/Ananse:Lung.csv to resources/grn_models/global/Ananse:Lung.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/global/Ananse:Stomach.csv to resources/grn_models/global/Ananse:Stomach.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/global/Cellnet:Bcell.csv to resources/grn_models/global/Cellnet:Bcell.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/global/Cellnet:Heart.csv to resources/grn_models/global/Cellnet:Heart.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/global/Cellnet:Neuron.csv to resources/grn_models/global/Cellnet:Neuron.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/global/Cellnet:Skin.csv to resources/grn_models/global/Cellnet:Skin.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/global/Cellnet:Tcell.csv to resources/grn_models/global/Cellnet:Tcell.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/global/Gtex:Brain amygdala.csv to resources/grn_models/global/Gtex:Brain amygdala.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/global/Gtex:Breast mammary tissue.csv to resources/grn_models/global/Gtex:Breast mammary tissue.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/global/Gtex:Lung.csv to resources/grn_models/global/Gtex:Lung.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/global/Gtex:Stomach.csv to resources/grn_models/global/Gtex:Stomach.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/global/Gtex:Whole blood.csv to resources/grn_models/global/Gtex:Whole blood.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/global/collectri.csv to resources/grn_models/global/collectri.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/nakatake/grnboost2_0.2.csv to resources/grn_models/nakatake/grnboost2_0.2.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/nakatake/grnboost2_0.5.csv to resources/grn_models/nakatake/grnboost2_0.5.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/nakatake/grnboost2_1.0.csv to resources/grn_models/nakatake/grnboost2_1.0.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/nakatake/pearson_corr_0.2.csv to resources/grn_models/nakatake/pearson_corr_0.2.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/nakatake/pearson_corr_0.5.csv to resources/grn_models/nakatake/pearson_corr_0.5.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/nakatake/pearson_corr_1.0.csv to resources/grn_models/nakatake/pearson_corr_1.0.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/nakatake/portia_0.2.csv to resources/grn_models/nakatake/portia_0.2.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/nakatake/portia_0.5.csv to resources/grn_models/nakatake/portia_0.5.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/nakatake/portia_1.0.csv to resources/grn_models/nakatake/portia_1.0.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/norman/pearson_corr.csv to resources/grn_models/norman/pearson_corr.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/norman/pearson_corr_0.2.csv to resources/grn_models/norman/pearson_corr_0.2.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/norman/pearson_corr_0.5.csv to resources/grn_models/norman/pearson_corr_0.5.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/norman/pearson_corr_1.0.csv to resources/grn_models/norman/pearson_corr_1.0.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/norman/portia_0.2.csv to resources/grn_models/norman/portia_0.2.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/norman/portia_0.5.csv to resources/grn_models/norman/portia_0.5.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/norman/portia_1.0.csv to resources/grn_models/norman/portia_1.0.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/norman/scenic.csv to resources/grn_models/norman/scenic.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/op/pearson_corr_1.csv to resources/grn_models/op/pearson_corr_1.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/op/pearson_corr_2.csv to resources/grn_models/op/pearson_corr_2.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/op/portia_1.csv to resources/grn_models/op/portia_1.csv\n", + "download: s3://openproblems-data/resources/grn/datasets_raw/op_bulked.h5ad to resources/datasets_raw/op_bulked.h5ad\n", + "download: s3://openproblems-data/resources/grn/grn_models/op/portia_2.csv to resources/grn_models/op/portia_2.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/replogle2/grnboost2_0.5.csv to resources/grn_models/replogle2/grnboost2_0.5.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/replogle2/grnboost2_1.0.csv to resources/grn_models/replogle2/grnboost2_1.0.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/replogle2/grnboost2_0.2.csv to resources/grn_models/replogle2/grnboost2_0.2.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/replogle2/pearson_corr_0.2.csv to resources/grn_models/replogle2/pearson_corr_0.2.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/replogle2/pearson_corr_1.0.csv to resources/grn_models/replogle2/pearson_corr_1.0.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/replogle2/pearson_corr_0.5.csv to resources/grn_models/replogle2/pearson_corr_0.5.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/replogle2/portia_0.2.csv to resources/grn_models/replogle2/portia_0.2.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/replogle2/portia_0.5.csv to resources/grn_models/replogle2/portia_0.5.csv\n", + "download: s3://openproblems-data/resources/grn/grn_models/replogle2/portia_1.0.csv to resources/grn_models/replogle2/portia_1.0.csv\n", + "download: s3://openproblems-data/resources/grn/results/figs/boxplot_r2scores_tfactivation.png to resources/results/figs/boxplot_r2scores_tfactivation.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/centrality_in.png to resources/results/figs/centrality_in.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/centrality_in_short.png to resources/results/figs/centrality_in_short.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/consensus_all.png to resources/results/figs/consensus_all.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/consensus_ws.png to resources/results/figs/consensus_ws.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/evaluation_scores_global_models.png to resources/results/figs/evaluation_scores_global_models.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/evaluation_scores_norman_adamson.png to resources/results/figs/evaluation_scores_norman_adamson.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/evaluation_scores_opsca.png to resources/results/figs/evaluation_scores_opsca.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/evaluation_scores_other_datasets.png to resources/results/figs/evaluation_scores_other_datasets.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/gb_vs_ridge.png to resources/results/figs/gb_vs_ridge.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/jaccard_similarity_op.png to resources/results/figs/jaccard_similarity_op.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/nregulators_opsca.png to resources/results/figs/nregulators_opsca.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/opsca_perturbation_strength.png to resources/results/figs/opsca_perturbation_strength.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/perturbation_strength_celltypes.png to resources/results/figs/perturbation_strength_celltypes.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/perturbation_strength_datasets.png to resources/results/figs/perturbation_strength_datasets.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/perturbation_strength_lowscored_compounds.png to resources/results/figs/perturbation_strength_lowscored_compounds.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/raw_scores_norman_adamson.png to resources/results/figs/raw_scores_norman_adamson.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/raw_scores_others.png to resources/results/figs/raw_scores_others.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/reg1_feature_importannce_model_corr.png to resources/results/figs/reg1_feature_importannce_model_corr.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/reg1_sample_wise_cell_type.png to resources/results/figs/reg1_sample_wise_cell_type.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/reg1_sample_wise_compound.png to resources/results/figs/reg1_sample_wise_compound.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/reg1_sample_wise_compounds.png to resources/results/figs/reg1_sample_wise_compounds.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/reg1_sample_wise_model_corr.png to resources/results/figs/reg1_sample_wise_model_corr.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/reg1_sample_wise_scores_dist.png to resources/results/figs/reg1_sample_wise_scores_dist.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/reg1_tf_activation_stability.png to resources/results/figs/reg1_tf_activation_stability.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/reg2_feature_importance_stability.png to resources/results/figs/reg2_feature_importance_stability.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/reg2_grnderived_stability.png to resources/results/figs/reg2_grnderived_stability.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/reg2_joint_r2scores.png to resources/results/figs/reg2_joint_r2scores.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/reg2_models_corr.png to resources/results/figs/reg2_models_corr.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/reg2_nregulators_vs_r2scoes.png to resources/results/figs/reg2_nregulators_vs_r2scoes.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/reg2_tf_activation.png to resources/results/figs/reg2_tf_activation.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/relative_scores_all.png to resources/results/figs/relative_scores_all.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/robustnes_analysis.png to resources/results/figs/robustnes_analysis.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/robustnes_analysis_reg1_1.png to resources/results/figs/robustnes_analysis_reg1_1.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/robustness_reg1.png to resources/results/figs/robustness_reg1.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/robustness_reg2.png to resources/results/figs/robustness_reg2.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/similaritys_donors_reg2.png to resources/results/figs/similaritys_donors_reg2.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/skelton_guided_barplot.png to resources/results/figs/skelton_guided_barplot.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/summary_figure.pdf to resources/results/figs/summary_figure.pdf\n", + "download: s3://openproblems-data/resources/grn/results/figs/summary_figure.png to resources/results/figs/summary_figure.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/tf_guided_barplot.png to resources/results/figs/tf_guided_barplot.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/topology_stats.png to resources/results/figs/topology_stats.png\n", + "download: s3://openproblems-data/resources/grn/results/figs/topology_stats_short.png to resources/results/figs/topology_stats_short.png\n", + "download: s3://openproblems-data/resources/grn/results/summary.tsv to resources/results/summary.tsv\n", + "download: s3://openproblems-data/resources/grn/results/summary_figure.pdf to resources/results/summary_figure.pdf\n", + "download: s3://openproblems-data/resources/grn/results/summary_figure.png to resources/results/summary_figure.png\n", + "download: s3://openproblems-data/resources/grn/datasets_raw/replogle2_bulked.h5ad to resources/datasets_raw/replogle2_bulked.h5ad\n", + "download: s3://openproblems-data/resources/grn/inference_datasets/norman_rna.h5ad to resources/inference_datasets/norman_rna.h5ad\n", + "download: s3://openproblems-data/resources/grn/datasets_raw/norman_sc_counts.h5ad to resources/datasets_raw/norman_sc_counts.h5ad\n", + "download: s3://openproblems-data/resources/grn/datasets_raw/op_perturbation_sc_counts.h5ad to resources/datasets_raw/op_perturbation_sc_counts.h5ad\n" + ] + } + ], "source": [ "!aws s3 sync s3://openproblems-data/resources/grn/ resources/ --delete\n", - "\n", - "aws s3 sync s3://openproblems-data/resources/grn/results resources/results/ --delete\n", - "\n", - "aws s3 sync s3://openproblems-data/resources/grn/grn_models resources/grn_models/\n", - "\n", - "aws s3 sync s3://openproblems-data/resources/grn/inference_datasets/ resources/inference_datasets/\n", - "\n", - "aws s3 sync s3://openproblems-data/resources/grn/evaluation_datasets/ resources/evaluation_datasets/" + "# aws s3 sync s3://openproblems-data/resources/grn/results resources/results/ --delete\n", + "# aws s3 sync s3://openproblems-data/resources/grn/grn_models resources/grn_models/\n", + "# aws s3 sync s3://openproblems-data/resources/grn/inference_datasets/ resources/inference_datasets/\n", + "# aws s3 sync s3://openproblems-data/resources/grn/evaluation_datasets/ resources/evaluation_datasets/" ] }, { @@ -3611,7 +3601,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.15" + "version": "3.10.14" } }, "nbformat": 4, diff --git a/scripts/calculate_score.sh b/scripts/calculate_score.sh new file mode 100644 index 000000000..0be7f4bf1 --- /dev/null +++ b/scripts/calculate_score.sh @@ -0,0 +1,15 @@ +# bash src/metrics/all_metrics/run.sh resources/grn_models/norman/grnboost2.csv norman + +prediction=${1} +dataset_id=${2} + +viash run src/metrics/all_metrics/config.novsh.yaml -- \ + --prediction ${prediction} \ + --dataset_id ${dataset_id} \ + --score output/score.h5ad \ + --tf_all resources/prior/tf_all.csv \ + --regulators_consensus resources/prior/regulators_consensus_${dataset_id}.json \ + --ws_consensus resources/prior/ws_consensus_${dataset_id}.csv \ + --ws_distance_background resources/prior/ws_distance_background_${dataset_id}.csv \ + --evaluation_data_sc resources/evaluation_datasets/${dataset_id}_sc_counts.h5ad \ + --evaluation_data resources/evaluation_datasets/${dataset_id}_perturbation.h5ad \ No newline at end of file diff --git a/scripts/download_resources.sh b/scripts/download_resources.sh index 5181cf555..455fcfdeb 100644 --- a/scripts/download_resources.sh +++ b/scripts/download_resources.sh @@ -5,33 +5,23 @@ set -e echo ">> Downloading resources" viash run src/common/sync_test_resources/config.vsh.yaml -- \ - --input "s3://openproblems-data/resources/grn/grn-benchmark" \ - --output "resources/grn-benchmark" \ + --input "s3://openproblems-data/resources/grn/inference_datasets/" \ + --output "resources/inference_datasets/" \ --delete viash run src/common/sync_test_resources/config.vsh.yaml -- \ - --input "s3://openproblems-data/resources/grn/prior" \ - --output "resources/prior" \ + --input "s3://openproblems-data/resources/grn/evaluation_datasets/" \ + --output "resources/evaluation_datasets/" \ --delete viash run src/common/sync_test_resources/config.vsh.yaml -- \ - --input "s3://openproblems-data/resources/grn/grn_models" \ - --output "resources/grn_models" \ - --delete -echo ">> Downloading resources test" -viash run src/common/sync_test_resources/config.vsh.yaml -- \ - --input "s3://openproblems-data/resources_test/grn/grn-benchmark" \ - --output "resources_test/grn-benchmark" \ - --delete - -viash run src/common/sync_test_resources/config.vsh.yaml -- \ - --input "s3://openproblems-data/resources_test/grn/prior" \ - --output "resources_test/prior" \ + --input "s3://openproblems-data/resources/grn/prior" \ + --output "resources/prior" \ --delete viash run src/common/sync_test_resources/config.vsh.yaml -- \ - --input "s3://openproblems-data/resources_test/grn/grn_models" \ - --output "resources_tests/grn_models" \ + --input "s3://openproblems-data/resources/grn/grn_models/" \ + --output "resources/grn_models/" \ --delete diff --git a/scripts/download_resources_all.sh b/scripts/download_resources_all.sh new file mode 100644 index 000000000..f1e341ede --- /dev/null +++ b/scripts/download_resources_all.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +echo ">> Downloading resources" + +viash run src/common/sync_test_resources/config.vsh.yaml -- \ + --input "s3://openproblems-data/resources/grn/" \ + --output "resources/" \ + --delete + + + diff --git a/scripts/render_readme.sh b/scripts/render_readme.sh index d72583a94..bf63b0a2b 100644 --- a/scripts/render_readme.sh +++ b/scripts/render_readme.sh @@ -3,7 +3,7 @@ set -e viash run src/common/create_task_readme/config.vsh.yaml -- \ - --task "grn_benchmark" \ + --task "grn_inference" \ --task_dir "src" \ --github_url "https://github.com/openproblems-bio/task_grn_inference/tree/main/" \ --output "README.md" diff --git a/scripts/upload_resources.sh b/scripts/upload_resources.sh deleted file mode 100644 index 4a65abe06..000000000 --- a/scripts/upload_resources.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -e - -echo ">> Uploading resources" - -aws s3 sync ./resources s3://openproblems-data/resources/grn/ - - diff --git a/src/api/comp_metric.yaml b/src/api/comp_metric.yaml index f5f497c68..20693f4bf 100644 --- a/src/api/comp_metric.yaml +++ b/src/api/comp_metric.yaml @@ -8,48 +8,19 @@ functionality: description: | A metric to evaluate the performance of the inferred GRN arguments: - - name: --evaluation_data - __merge__: file_evaluation_h5ad.yaml - required: false - direction: input - name: --prediction __merge__: file_prediction.yaml required: true direction: input - name: --score __merge__: file_score.yaml - required: false - direction: output - - name: --tf_all - type: file - direction: input required: true - example: resources_test/prior/tf_all.csv - - name: --reg_type - type: string - direction: input - default: ridge - description: name of regretion to use - multiple: false - - name: --subsample - type: integer - direction: input - default: -1 - description: number of samples randomly drawn from perturbation data - - name: --num_workers - type: integer - direction: input - default: 4 + direction: output - name: --method_id type: string direction: input required: false - example: collectri - - name: --apply_tf - type: boolean - required: false - default: true - + example: grnboost2 - name: --layer type: string direction: input @@ -62,14 +33,6 @@ functionality: type: integer default: 2 direction: input - - name: --skeleton - type: file - direction: input - example: resources_test/prior/skeleton.csv - - name: --apply_skeleton - type: boolean - direction: input - default: false - name: --dataset_id type: string direction: input diff --git a/src/api/comp_metric_regression.yaml b/src/api/comp_metric_regression.yaml new file mode 100644 index 000000000..e1fc4c1d0 --- /dev/null +++ b/src/api/comp_metric_regression.yaml @@ -0,0 +1,42 @@ +__merge__: comp_metric.yaml +functionality: + name: metrics_regression + namespace: "metrics" + info: + label: metrics_regression + summary: Calculates regression scores + arguments: + - name: --evaluation_data + __merge__: file_evaluation_h5ad.yaml + required: false + direction: input + - name: --tf_all + type: file + direction: input + required: true + example: resources_test/prior/tf_all.csv + - name: --reg_type + type: string + direction: input + default: ridge + description: name of regretion to use + multiple: false + - name: --subsample + type: integer + direction: input + default: -1 + description: number of samples randomly drawn from perturbation data + - name: --num_workers + type: integer + direction: input + default: 4 + - name: --apply_tf + type: boolean + required: false + default: true + - name: --apply_skeleton + type: boolean + required: false + default: false + + diff --git a/src/api/comp_metric_ws.yaml b/src/api/comp_metric_ws.yaml new file mode 100644 index 000000000..736fd8c27 --- /dev/null +++ b/src/api/comp_metric_ws.yaml @@ -0,0 +1,26 @@ +__merge__: comp_metric.yaml +functionality: + name: ws_distance + namespace: "metrics" + info: + label: ws_distance + summary: Calculates Wasserstein distance for a given GRN and dataset + arguments: + - name: --ws_consensus + type: file + direction: input + must_exist: false + required: true + example: resources_test/prior/ws_consensus_norman.csv + - name: --ws_distance_background + type: file + direction: input + must_exist: false + required: true + example: resources_test/prior/ws_distance_background_norman.csv + - name: --evaluation_data_sc + type: file + required: true + direction: input + example: 'resources_test/datasets_raw/adamson_sc_counts.h5ad' + diff --git a/src/api/task_info.yaml b/src/api/task_info.yaml index 2b928f650..e5968c2f3 100644 --- a/src/api/task_info.yaml +++ b/src/api/task_info.yaml @@ -1,13 +1,20 @@ name: GRN inference benchmark -label: A dynamic benchmark for gene regulatory network (GRN) inference +label: Living benchmark for gene regulatory network (GRN) inference motivation: | GRNs are essential for understanding cellular identity and behavior. They are simplified models of gene expression regulated by complex processes involving multiple layers of control, from transcription to post-transcriptional modifications, incorporating various regulatory elements and non-coding RNAs. Gene transcription is controlled by a regulatory complex that includes transcription factors (TFs), cis-regulatory elements (CREs) like promoters and enhancers, and essential co-factors. High-throughput datasets, covering thousands of genes, facilitate the use of machine learning approaches to decipher GRNs. The advent of single-cell sequencing technologies, such as scRNA-seq, has made it possible to infer GRNs from a single experiment due to the abundance of samples. This allows researchers to infer condition-specific GRNs, such as for different cell types or diseases, and study potential regulatory factors associated with these conditions. Combining chromatin accessibility data with gene expression measurements has led to the development of enhancer-driven GRN (eGRN) inference pipelines, which offer significantly improved accuracy over single-modality methods. description: | - Here, we present a dynamic benchmark platform for GRN inference. This platform provides curated datasets for GRN inference and evaluation, standardized evaluation protocols and metrics, computational infrastructure, and a dynamically updated leaderboard to track state-of-the-art methods. It runs novel GRNs in the cloud, offers competition scores, and stores them for future comparisons, reflecting new developments over time. + Here, we present geneRNIB as a living benchmark platform for GRN inference. This platform provides curated datasets for GRN inference and evaluation, standardized evaluation protocols and metrics, computational infrastructure, and a dynamically updated leaderboard to track state-of-the-art methods. It runs novel GRNs in the cloud, offers competition scores, and stores them for future comparisons, reflecting new developments over time. - The platform supports the integration of new datasets and protocols. When a new feature is added, previously evaluated GRNs are re-assessed, and the leaderboard is updated accordingly. The aim is to evaluate both the accuracy and completeness of inferred GRNs. It is designed for both single-modality and multi-omics GRN inference. Ultimately, it is a community-driven platform. So far, six eGRN inference methods have been integrated: Scenic+, CellOracle, FigR, scGLUE, GRaNIE, and ANANSE. + The platform supports the integration of new datasets and protocols. When a new feature is added, previously evaluated GRNs are re-assessed, and the leaderboard is updated accordingly. The aim is to evaluate both the accuracy and completeness of inferred GRNs. It is designed for both single-modality and multi-omics GRN inference. Ultimately, it is a community-driven platform. + + So far, ten GRN inference methods have been integrated: five sinlge-omics methods of GRNBoost2, GENIE3, Portia, PPCOR, and Scenic; and five eGRN inference methods of Scenic+, CellOracle, FigR, scGLUE, and GRaNIE. + + Due to its flexible nature, the platform can incorporate various benchmark datasets and evaluation methods, using either prior knowledge or feature-based approaches. + In the current version, due to the absence of standardized prior knowledge, we use indirect approaches to benchmark GRNs. Employing interventional data as evaluation datasets, we have developed 8 metrics using feature-based approach and Wasserstein distance, accounting for both accuracy and comprehensiveness. + + Five datasets have been integrated so far, namely OPSCA, Nakatake, Norman, Adamson, and Replogle. For each dataset, standardized inference datasets are provided to be used for GRN inference and evaluation datasets are employed to benchmark. + See our publication for the details of methods. - Due to its flexible nature, the platform can incorporate various benchmark datasets and evaluation methods, using either prior knowledge or feature-based approaches. In the current version, due to the absence of standardized prior knowledge, we use a feature-based approach to benchmark GRNs. Our evaluation utilizes standardized datasets for GRN inference and evaluation, employing multiple regression analysis approaches to assess both accuracy and comprehensiveness. summary: | Benchmarking GRN inference methods @@ -28,21 +35,16 @@ readme: | # download resources scripts/download_resources.sh ``` + The datasets for GRN inference are located in `resources/inference_datasets`. ## Infer a GRN - - ```bash - viash run src/methods/dummy/config.vsh.yaml -- --multiomics_rna resources/grn-benchmark/multiomics_rna.h5ad --multiomics_atac resources/grn-benchmark/multiomics_atac.h5ad --prediction output/dummy.csv - - ``` - Similarly, run the command for other methods. + One GRN should be inferred for each inference dataset (op, norman, replogle2, adamson, and nakatake). The inferred GRN should have three columns of `source, target, weight`. See `resources/grn_models/op/grnboost2.csv` as an example. ## Evaluate a GRN + Once a GRN is inferred (e.g. located in `output/your_GRN.csv`) for a given dataset (e.g. `norman`), use the following code to obtain evaluation scores. ```bash - scripts/benchmark_grn.sh --grn resources/grn-benchmark/models/collectri.csv - + scripts/calculate_score.sh output/your_GRN.csv norman ``` - - Similarly, run the command for other GRN models. + This will calculate and print the scores as well as output the scores into `output/score.h5ad` ## Add a method @@ -64,14 +66,15 @@ authors: roles: [ contributor ] info: github: AntoinePassemiers - - name: Christian Arnold - roles: [ contributor ] - info: - github: chrarnold - name: Marco Stock roles: [ contributor ] info: github: stkmrc + - name: Christian Arnold + roles: [ contributor ] + info: + github: chrarnold + diff --git a/src/metrics/all_metrics/config.novsh.yaml b/src/metrics/all_metrics/config.novsh.yaml new file mode 100644 index 000000000..f776eb44e --- /dev/null +++ b/src/metrics/all_metrics/config.novsh.yaml @@ -0,0 +1,97 @@ + +__merge__: ../../api/comp_metric.yaml + +functionality: + name: metrics_all + info: + label: metrics_all + summary: Calculates all metrics for a given GRN and dataset + arguments: + - name: --evaluation_data + type: file + required: true + direction: input + - name: --tf_all + type: file + direction: input + required: true + example: resources_test/prior/tf_all.csv + - name: --reg_type + type: string + direction: input + default: ridge + description: name of regretion to use + multiple: false + - name: --subsample + type: integer + direction: input + default: -1 + description: number of samples randomly drawn from perturbation data + - name: --num_workers + type: integer + direction: input + default: 4 + - name: --apply_tf + type: boolean + required: false + default: true + - name: --apply_skeleton + type: boolean + required: false + default: false + - name: --regulators_consensus + type: file + direction: input + must_exist: false + required: true + example: resources_test/prior/regulators_consensus_norman.json + - name: --static_only + direction: input + type: boolean + default: true + - name: --binarize + type: boolean + direction: input + description: whether to binarize the weight + default: true + - name: --ws_consensus + type: file + direction: input + must_exist: false + required: true + example: resources_test/prior/ws_consensus_norman.csv + - name: --ws_distance_background + type: file + direction: input + must_exist: false + required: true + example: resources_test/prior/ws_distance_background_norman.csv + - name: --evaluation_data_sc + type: file + required: true + direction: input + example: 'resources_test/datasets_raw/adamson_sc_counts.h5ad' + + + resources: + - type: python_script + path: script.py + - path: /src/utils/util.py + dest: util.py + - path: /src/metrics/regression_1/main.py + dest: reg1_main.py + - path: /src/metrics/regression_2/main.py + dest: reg2_main.py + - path: /src/metrics/wasserstein/main.py + dest: ws_main.py + + +platforms: + - type: docker + image: ghcr.io/openproblems-bio/base_python:1.0.4 + setup: + - type: python + packages: [ lightgbm==4.3.0, numpy==1.26.4 ] + - type: nextflow + directives: + label: [ midtime, midmem, midcpu ] \ No newline at end of file diff --git a/src/metrics/all_metrics/script.py b/src/metrics/all_metrics/script.py index 440175fc9..fe4412782 100644 --- a/src/metrics/all_metrics/script.py +++ b/src/metrics/all_metrics/script.py @@ -25,16 +25,27 @@ 'score': 'output/score.h5ad' } ## VIASH END - -meta = { - "resources_dir": 'src/metrics/', - "util": 'src/utils' -} sys.path.append(meta["resources_dir"]) -sys.path.append(meta["util"]) -from regression_1.main import main as main_reg1 -from regression_2.main import main as main_reg2 -from wasserstein.script import main as main_ws +from reg1_main import main as main_reg1 +from reg2_main import main as main_reg2 +from ws_main import main as main_ws + +# try: +# sys.path.append(meta["resources_dir"]) +# from reg1_main.main import main as main_reg1 +# from reg2_main.main import main as main_reg2 +# from ws_main.main import main as main_ws + +# except: +# meta = { +# "resources_dir": 'src/metrics/', +# "util": 'src/utils' +# } +# sys.path.append(meta["resources_dir"]) +# sys.path.append(meta["util"]) +# from regression_1.main import main as main_reg1 +# from regression_2.main import main as main_reg2 +# from wasserstein.script import main as main_ws @@ -45,11 +56,11 @@ def main(par): assert par['dataset_id'] dataset = par['dataset_id'] - par['evaluation_data'] = f'resources/evaluation_datasets/{dataset}_perturbation.h5ad' - par['evaluation_data_sc'] = f'resources/datasets_raw/{dataset}_sc_counts.h5ad' - par['regulators_consensus'] = f'resources/prior/regulators_consensus_{dataset}.json' - par['ws_consensus'] = f'resources/prior/ws_consensus_{dataset}.csv' - par['ws_distance_background'] = f'resources/prior/ws_distance_background_{dataset}.csv' + # par['evaluation_data'] = f'resources/evaluation_datasets/{dataset}_perturbation.h5ad' + # par['evaluation_data_sc'] = f'resources/datasets_raw/{dataset}_sc_counts.h5ad' + # par['regulators_consensus'] = f'resources/prior/regulators_consensus_{dataset}.json' + # par['ws_consensus'] = f'resources/prior/ws_consensus_{dataset}.csv' + # par['ws_distance_background'] = f'resources/prior/ws_distance_background_{dataset}.csv' scores_all = [] @@ -68,6 +79,8 @@ def main(par): if __name__ == '__main__': scores_all = main(par) + print(scores_all) + output = ad.AnnData( X=np.empty((0, 0)), uns={ @@ -77,6 +90,5 @@ def main(par): "metric_values": scores_all.values[0] } ) - print(output) output.write_h5ad(par['score'], compression='gzip') print('Completed', flush=True) diff --git a/src/metrics/regression_1/config.vsh.yaml b/src/metrics/regression_1/config.vsh.yaml index a2c8c937a..75aa6c296 100644 --- a/src/metrics/regression_1/config.vsh.yaml +++ b/src/metrics/regression_1/config.vsh.yaml @@ -1,15 +1,13 @@ -__merge__: ../../api/comp_metric.yaml +__merge__: ../../api/comp_metric_regression.yaml functionality: - name: regression_1 + name: metrics_regression_1 namespace: "metrics" info: - label: Regression 1 - summary: Calculates R2 score for regression 1 - description: | - Calculates R2 score using regression approach 1. - - arguments: + label: metrics_regression_1 + summary: Calculates regression scores 1 + + arguments: - name: --binarize type: boolean direction: input @@ -22,6 +20,7 @@ functionality: - path: main.py - path: /src/utils/util.py dest: util.py + platforms: - type: docker image: ghcr.io/openproblems-bio/base_python:1.0.4 diff --git a/src/metrics/regression_1/run.sh b/src/metrics/regression_1/run.sh new file mode 100644 index 000000000..9fd3211a5 --- /dev/null +++ b/src/metrics/regression_1/run.sh @@ -0,0 +1,5 @@ +viash run src/metrics/regression_1/config.vsh.yaml -- \ + --prediction resources/grn_models/norman/grnboost2.csv \ + --dataset_id norman --evaluation_data resources/evaluation_datasets/norman_perturbation.h5ad \ + --score output/score.h5ad \ + --tf_all resources/prior/tf_all.csv \ No newline at end of file diff --git a/src/metrics/regression_2/config.vsh.yaml b/src/metrics/regression_2/config.vsh.yaml index 9f7465fb4..575b79306 100644 --- a/src/metrics/regression_2/config.vsh.yaml +++ b/src/metrics/regression_2/config.vsh.yaml @@ -1,29 +1,30 @@ -__merge__: ../../api/comp_metric.yaml +__merge__: ../../api/comp_metric_regression.yaml + functionality: - name: regression_2 + name: metrics_regression_2 + namespace: "metrics" info: - label: regression_2 - summary: Calculates R2 score for regression 2 - description: | - Calculates R2 score using regression approach 2. - resources: - - type: python_script - path: script.py - - path: main.py - - path: /src/utils/util.py - dest: util.py - arguments: - - name: --consensus + label: metrics_regression_2 + summary: Calculates regression scores 2 + + arguments: + - name: --regulators_consensus type: file direction: input must_exist: false required: true - example: resources_test/prior/op_consensus-num-regulators.json + example: resources_test/prior/regulators_consensus_norman.json - name: --static_only direction: input type: boolean default: true - + resources: + - type: python_script + path: script.py + - path: main.py + - path: /src/utils/util.py + dest: util.py + platforms: - type: docker image: ghcr.io/openproblems-bio/base_python:1.0.4 diff --git a/src/metrics/regression_2/run.sh b/src/metrics/regression_2/run.sh new file mode 100644 index 000000000..a4cd3e6ad --- /dev/null +++ b/src/metrics/regression_2/run.sh @@ -0,0 +1,6 @@ +viash run src/metrics/regression_2/config.vsh.yaml -- \ + --prediction resources/grn_models/norman/grnboost2.csv \ + --dataset_id norman --evaluation_data resources/evaluation_datasets/norman_perturbation.h5ad \ + --score output/score.h5ad \ + --tf_all resources/prior/tf_all.csv \ + --regulators_consensus resources/prior/regulators_consensus_norman.json \ No newline at end of file diff --git a/src/metrics/wasserstein/config.vsh.yaml b/src/metrics/wasserstein/config.vsh.yaml new file mode 100644 index 000000000..1a207912c --- /dev/null +++ b/src/metrics/wasserstein/config.vsh.yaml @@ -0,0 +1,41 @@ +__merge__: comp_metric.yaml +functionality: + name: ws_distance + namespace: "metrics" + info: + label: ws_distance + summary: Calculates Wasserstein distance for a given GRN and dataset + arguments: + - name: --ws_consensus + type: file + direction: input + must_exist: false + required: true + example: resources_test/prior/ws_consensus_norman.csv + - name: --ws_distance_background + type: file + direction: input + must_exist: false + required: true + example: resources_test/prior/ws_distance_background_norman.csv + - name: --evaluation_data_sc + type: file + required: true + direction: input + example: 'resources_test/datasets_raw/adamson_sc_counts.h5ad' + resources: + - type: python_script + path: script.py + - path: /src/utils/util.py + dest: util.py + - path: main.py + +platforms: + - type: docker + image: ghcr.io/openproblems-bio/base_python:1.0.4 + setup: + - type: python + packages: [numpy==1.26.4 ] + - type: nextflow + directives: + label: [ midtime, midmem, midcpu ] \ No newline at end of file diff --git a/src/metrics/wasserstein/main.py b/src/metrics/wasserstein/main.py new file mode 100644 index 000000000..d74ce848d --- /dev/null +++ b/src/metrics/wasserstein/main.py @@ -0,0 +1,42 @@ +import pandas as pd +import anndata as ad +import sys +import numpy as np + +def main(par): + prediction = pd.read_csv(par['prediction'], index_col=0) + consensus = pd.read_csv(par['ws_consensus'], index_col=0) + background_distance = pd.read_csv(par['ws_distance_background'], index_col=0) + evaluation_data = ad.read_h5ad(par['evaluation_data_sc']) + evaluation_data.X = evaluation_data.layers[par['layer']] + + # - for each theta, and each tf: + scores_model = [] + for theta in consensus['theta'].unique(): + consensus_theta = consensus[consensus['theta'] == theta] + for tf in consensus_theta['source'].unique(): + if tf not in prediction['source'].unique(): # skip the evaluation if tf is not given in the predictions + continue + # - get the prior + background_distance_tf = background_distance[background_distance['source']==tf] + n_edges = consensus_theta[consensus_theta['source'] == tf]['value'].values[0] + # - subset the prediction to the given tf: choose the top edges based on n_edges + prediction_tf = prediction[prediction['source']==tf] + prediction_tf = prediction_tf.nlargest(n_edges, 'weight') + # - get the ws distance + ws_distance = background_distance_tf[background_distance_tf['target'].isin(prediction_tf['target'])].copy() + # - fill the missing links with random scores + n_missing = n_edges - len(ws_distance) + ws_distance_missing = background_distance_tf.sample(n_missing, replace=True) + ws_distance = pd.concat([ws_distance, ws_distance_missing]) + ws_distance['present_edges_n'] = len(ws_distance) + # - normalize to the background dist -> percentile rank + background_distance_random = np.random.choice(background_distance_tf['ws_distance'].values, 1000) + ws_distance_pc = ws_distance['ws_distance'].apply(lambda val: (val>background_distance_random).sum())/len(background_distance_random) + ws_distance['ws_distance_pc'] = ws_distance_pc + + ws_distance['theta'] = theta + scores_model.append(ws_distance) + scores_model = pd.concat(scores_model).reset_index(drop=True) + mean_scores = scores_model.groupby('theta')['ws_distance_pc'].mean().to_frame().T.reset_index(drop=True) + return scores_model, mean_scores \ No newline at end of file diff --git a/src/metrics/wasserstein/run.sh b/src/metrics/wasserstein/run.sh new file mode 100644 index 000000000..44802fac5 --- /dev/null +++ b/src/metrics/wasserstein/run.sh @@ -0,0 +1,6 @@ +viash run src/metrics/wasserstein/config.vsh.yaml -- --prediction resources/grn_models/norman/grnboost2.csv \ + --dataset_id norman \ + --ws_consensus resources/prior/ws_consensus_norman.csv \ + --ws_distance_background resources/prior/ws_distance_background_norman.csv \ + --evaluation_data_sc resources/datasets_raw/norman_sc_counts.h5ad \ + --score output/score.h5ad \ No newline at end of file diff --git a/src/metrics/wasserstein/script.py b/src/metrics/wasserstein/script.py index f4693768b..4fc783063 100644 --- a/src/metrics/wasserstein/script.py +++ b/src/metrics/wasserstein/script.py @@ -8,7 +8,6 @@ par = { 'prediction': 'resources/grn_models/adamson/pearson_corr.csv', 'evaluation_data_sc': 'resources/datasets_raw/adamson_sc_counts.h5ad', - 'output_file': 'resources/prior/ws_distance_background.csv', 'score': 'output/score.h5ad', 'ws_consensus': 'resources/prior/consensus_ws_distance_adamson.csv', 'ws_distance_background':'resources/prior/ws_distance_background_adamson.csv', @@ -18,43 +17,18 @@ } ## VIASH END -def main(par): - prediction = pd.read_csv(par['prediction'], index_col=0) - consensus = pd.read_csv(par['ws_consensus'], index_col=0) - background_distance = pd.read_csv(par['ws_distance_background'], index_col=0) - evaluation_data = ad.read_h5ad(par['evaluation_data_sc']) - evaluation_data.X = evaluation_data.layers[par['layer']] +try: + sys.path.append(meta["resources_dir"]) + from main import main +except: + meta = { + "resources_dir": 'src/metrics/wasserstein', + "util": 'src/utils' + } + sys.path.append(meta["resources_dir"]) + sys.path.append(meta["util"]) + from main import main - # - for each theta, and each tf: - scores_model = [] - for theta in consensus['theta'].unique(): - consensus_theta = consensus[consensus['theta'] == theta] - for tf in consensus_theta['source'].unique(): - if tf not in prediction['source'].unique(): # skip the evaluation if tf is not given in the predictions - continue - # - get the prior - background_distance_tf = background_distance[background_distance['source']==tf] - n_edges = consensus_theta[consensus_theta['source'] == tf]['value'].values[0] - # - subset the prediction to the given tf: choose the top edges based on n_edges - prediction_tf = prediction[prediction['source']==tf] - prediction_tf = prediction_tf.nlargest(n_edges, 'weight') - # - get the ws distance - ws_distance = background_distance_tf[background_distance_tf['target'].isin(prediction_tf['target'])].copy() - # - fill the missing links with random scores - n_missing = n_edges - len(ws_distance) - ws_distance_missing = background_distance_tf.sample(n_missing, replace=True) - ws_distance = pd.concat([ws_distance, ws_distance_missing]) - ws_distance['present_edges_n'] = len(ws_distance) - # - normalize to the background dist -> percentile rank - background_distance_random = np.random.choice(background_distance_tf['ws_distance'].values, 1000) - ws_distance_pc = ws_distance['ws_distance'].apply(lambda val: (val>background_distance_random).sum())/len(background_distance_random) - ws_distance['ws_distance_pc'] = ws_distance_pc - - ws_distance['theta'] = theta - scores_model.append(ws_distance) - scores_model = pd.concat(scores_model).reset_index(drop=True) - mean_scores = scores_model.groupby('theta')['ws_distance_pc'].mean().to_frame().T.reset_index(drop=True) - return scores_model, mean_scores if __name__ == '__main__': _, mean_scores = main(par) @@ -63,7 +37,7 @@ def main(par): uns={ "dataset_id": str(par["dataset_id"]), "method_id": f"reg2-{par['method_id']}", - "metric_ids": mean_scores.columns, + "metric_ids": mean_scores.columns.values, "metric_values": mean_scores.values[0] } )