From 25905ec4d635cd2003e7b121223384b6946ca728 Mon Sep 17 00:00:00 2001
From: Robrecht Cannoodt <rcannood@gmail.com>
Date: Tue, 26 Nov 2024 14:16:03 +0100
Subject: [PATCH] Reprocess the perturbation prediction results (#364)

* reprocess the results

* update metric info
---
 .../data/metric_execution_info.json           | 348 +++++++++---------
 .../data/metric_info.json                     |   5 +
 .../perturbation_prediction/data/results.json |  33 +-
 3 files changed, 199 insertions(+), 187 deletions(-)

diff --git a/results/perturbation_prediction/data/metric_execution_info.json b/results/perturbation_prediction/data/metric_execution_info.json
index 76773f81..28df7aa5 100644
--- a/results/perturbation_prediction/data/metric_execution_info.json
+++ b/results/perturbation_prediction/data/metric_execution_info.json
@@ -1,338 +1,338 @@
 [
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
     "method_id": "ground_truth",
-    "metric_id": "mean_rowwise_correlation",
+    "metric_component_name": "mean_rowwise_correlation",
     "resources": {
+      "submit": "2024-10-31 05:25:38",
       "exit_code": 0,
-      "duration_sec": 7.6,
+      "duration_sec": 22.8,
       "cpu_pct": 226.9,
       "peak_memory_mb": 3482,
-      "disk_read_mb": 138,
-      "disk_write_mb": 1
+      "disk_read_mb": 414,
+      "disk_write_mb": 3
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "jn_ap_op2",
-    "metric_id": "mean_rowwise_correlation",
+    "method_id": "ground_truth",
+    "metric_component_name": "mean_rowwise_error",
     "resources": {
+      "submit": "2024-10-31 05:25:39",
       "exit_code": 0,
-      "duration_sec": 5.9,
-      "cpu_pct": 377.6,
-      "peak_memory_mb": 6144,
-      "disk_read_mb": 135,
-      "disk_write_mb": 1
+      "duration_sec": 12.2,
+      "cpu_pct": 262.4,
+      "peak_memory_mb": 3380,
+      "disk_read_mb": 276,
+      "disk_write_mb": 2
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "lgc_ensemble",
-    "metric_id": "mean_rowwise_correlation",
+    "method_id": "jn_ap_op2",
+    "metric_component_name": "mean_rowwise_correlation",
     "resources": {
+      "submit": "2024-10-31 06:20:29",
       "exit_code": 0,
-      "duration_sec": 4.8,
-      "cpu_pct": 450,
+      "duration_sec": 17.7,
+      "cpu_pct": 377.6,
       "peak_memory_mb": 6144,
-      "disk_read_mb": 138,
-      "disk_write_mb": 1
+      "disk_read_mb": 405,
+      "disk_write_mb": 3
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "mean_across_celltypes",
-    "metric_id": "mean_rowwise_correlation",
+    "method_id": "jn_ap_op2",
+    "metric_component_name": "mean_rowwise_error",
     "resources": {
+      "submit": "2024-10-31 06:20:28",
       "exit_code": 0,
-      "duration_sec": 5.8,
-      "cpu_pct": 368.8,
-      "peak_memory_mb": 6144,
-      "disk_read_mb": 134,
-      "disk_write_mb": 1
+      "duration_sec": 8.4,
+      "cpu_pct": 511.1,
+      "peak_memory_mb": 5940,
+      "disk_read_mb": 270,
+      "disk_write_mb": 2
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "mean_across_compounds",
-    "metric_id": "mean_rowwise_correlation",
+    "method_id": "lgc_ensemble",
+    "metric_component_name": "mean_rowwise_correlation",
     "resources": {
+      "submit": "2024-10-31 05:51:49",
       "exit_code": 0,
-      "duration_sec": 5,
-      "cpu_pct": 439.6,
+      "duration_sec": 14.4,
+      "cpu_pct": 450,
       "peak_memory_mb": 6144,
-      "disk_read_mb": 136,
-      "disk_write_mb": 1
+      "disk_read_mb": 414,
+      "disk_write_mb": 3
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "mean_outcome",
-    "metric_id": "mean_rowwise_correlation",
+    "method_id": "lgc_ensemble",
+    "metric_component_name": "mean_rowwise_error",
     "resources": {
+      "submit": "2024-10-31 05:51:49",
       "exit_code": 0,
-      "duration_sec": 7.5,
-      "cpu_pct": 231.1,
-      "peak_memory_mb": 3584,
-      "disk_read_mb": 133,
-      "disk_write_mb": 1
+      "duration_sec": 13.2,
+      "cpu_pct": 247.5,
+      "peak_memory_mb": 6144,
+      "disk_read_mb": 276,
+      "disk_write_mb": 2
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "nn_retraining_with_pseudolabels",
-    "metric_id": "mean_rowwise_correlation",
+    "method_id": "mean_across_celltypes",
+    "metric_component_name": "mean_rowwise_correlation",
     "resources": {
+      "submit": "2024-10-31 05:26:19",
       "exit_code": 0,
-      "duration_sec": 4.9,
-      "cpu_pct": 442.1,
+      "duration_sec": 17.4,
+      "cpu_pct": 368.8,
       "peak_memory_mb": 6144,
-      "disk_read_mb": 138,
-      "disk_write_mb": 1
+      "disk_read_mb": 402,
+      "disk_write_mb": 3
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "pyboost",
-    "metric_id": "mean_rowwise_correlation",
+    "method_id": "mean_across_celltypes",
+    "metric_component_name": "mean_rowwise_error",
     "resources": {
+      "submit": "2024-10-31 05:26:19",
       "exit_code": 0,
-      "duration_sec": 5.9,
-      "cpu_pct": 386.2,
+      "duration_sec": 15.2,
+      "cpu_pct": 283.3,
       "peak_memory_mb": 6144,
-      "disk_read_mb": 138,
-      "disk_write_mb": 1
+      "disk_read_mb": 268,
+      "disk_write_mb": 2
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "sample",
-    "metric_id": "mean_rowwise_correlation",
+    "method_id": "mean_across_compounds",
+    "metric_component_name": "mean_rowwise_correlation",
     "resources": {
+      "submit": "2024-10-31 05:26:19",
       "exit_code": 0,
-      "duration_sec": 10.8,
-      "cpu_pct": 207.3,
+      "duration_sec": 15,
+      "cpu_pct": 439.6,
       "peak_memory_mb": 6144,
-      "disk_read_mb": 138,
-      "disk_write_mb": 1
+      "disk_read_mb": 408,
+      "disk_write_mb": 3
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "scape",
-    "metric_id": "mean_rowwise_correlation",
+    "method_id": "mean_across_compounds",
+    "metric_component_name": "mean_rowwise_error",
     "resources": {
+      "submit": "2024-10-31 05:26:19",
       "exit_code": 0,
-      "duration_sec": 5.1,
-      "cpu_pct": 363.1,
+      "duration_sec": 17.4,
+      "cpu_pct": 249.4,
       "peak_memory_mb": 6144,
-      "disk_read_mb": 135,
-      "disk_write_mb": 1
+      "disk_read_mb": 272,
+      "disk_write_mb": 2
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "transformer_ensemble",
-    "metric_id": "mean_rowwise_correlation",
+    "method_id": "mean_outcome",
+    "metric_component_name": "mean_rowwise_correlation",
     "resources": {
+      "submit": "2024-10-31 05:25:19",
       "exit_code": 0,
-      "duration_sec": 5.8,
-      "cpu_pct": 393.2,
-      "peak_memory_mb": 6144,
-      "disk_read_mb": 133,
-      "disk_write_mb": 1
+      "duration_sec": 22.5,
+      "cpu_pct": 231.1,
+      "peak_memory_mb": 3584,
+      "disk_read_mb": 399,
+      "disk_write_mb": 3
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "zeros",
-    "metric_id": "mean_rowwise_correlation",
+    "method_id": "mean_outcome",
+    "metric_component_name": "mean_rowwise_error",
     "resources": {
+      "submit": "2024-10-31 05:25:19",
       "exit_code": 0,
-      "duration_sec": 11.1,
-      "cpu_pct": 192.7,
-      "peak_memory_mb": 6144,
-      "disk_read_mb": 133,
-      "disk_write_mb": 1
+      "duration_sec": 12.4,
+      "cpu_pct": 259.8,
+      "peak_memory_mb": 3380,
+      "disk_read_mb": 266,
+      "disk_write_mb": 2
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "ground_truth",
-    "metric_id": "mean_rowwise_error",
+    "method_id": "nn_retraining_with_pseudolabels",
+    "metric_component_name": "mean_rowwise_correlation",
     "resources": {
+      "submit": "2024-10-31 06:10:29",
       "exit_code": 0,
-      "duration_sec": 6.1,
-      "cpu_pct": 262.4,
-      "peak_memory_mb": 3380,
-      "disk_read_mb": 138,
-      "disk_write_mb": 1
+      "duration_sec": 14.7,
+      "cpu_pct": 442.1,
+      "peak_memory_mb": 6144,
+      "disk_read_mb": 414,
+      "disk_write_mb": 3
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "jn_ap_op2",
-    "metric_id": "mean_rowwise_error",
+    "method_id": "nn_retraining_with_pseudolabels",
+    "metric_component_name": "mean_rowwise_error",
     "resources": {
+      "submit": "2024-10-31 06:10:29",
       "exit_code": 0,
-      "duration_sec": 4.2,
-      "cpu_pct": 511.1,
-      "peak_memory_mb": 5940,
-      "disk_read_mb": 135,
-      "disk_write_mb": 1
+      "duration_sec": 8.8,
+      "cpu_pct": 455,
+      "peak_memory_mb": 6144,
+      "disk_read_mb": 276,
+      "disk_write_mb": 2
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "lgc_ensemble",
-    "metric_id": "mean_rowwise_error",
+    "method_id": "pyboost",
+    "metric_component_name": "mean_rowwise_correlation",
     "resources": {
+      "submit": "2024-10-31 05:31:39",
       "exit_code": 0,
-      "duration_sec": 6.6,
-      "cpu_pct": 247.5,
+      "duration_sec": 17.7,
+      "cpu_pct": 386.2,
       "peak_memory_mb": 6144,
-      "disk_read_mb": 138,
-      "disk_write_mb": 1
+      "disk_read_mb": 414,
+      "disk_write_mb": 3
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "mean_across_celltypes",
-    "metric_id": "mean_rowwise_error",
+    "method_id": "pyboost",
+    "metric_component_name": "mean_rowwise_error",
     "resources": {
+      "submit": "2024-10-31 05:31:39",
       "exit_code": 0,
-      "duration_sec": 7.6,
-      "cpu_pct": 283.3,
-      "peak_memory_mb": 6144,
-      "disk_read_mb": 134,
-      "disk_write_mb": 1
+      "duration_sec": 8,
+      "cpu_pct": 525,
+      "peak_memory_mb": 5940,
+      "disk_read_mb": 276,
+      "disk_write_mb": 2
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "mean_across_compounds",
-    "metric_id": "mean_rowwise_error",
+    "method_id": "sample",
+    "metric_component_name": "mean_rowwise_correlation",
     "resources": {
+      "submit": "2024-10-31 05:25:09",
       "exit_code": 0,
-      "duration_sec": 8.7,
-      "cpu_pct": 249.4,
+      "duration_sec": 32.4,
+      "cpu_pct": 207.3,
       "peak_memory_mb": 6144,
-      "disk_read_mb": 136,
-      "disk_write_mb": 1
+      "disk_read_mb": 414,
+      "disk_write_mb": 3
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "mean_outcome",
-    "metric_id": "mean_rowwise_error",
+    "method_id": "sample",
+    "metric_component_name": "mean_rowwise_error",
     "resources": {
+      "submit": "2024-10-31 05:25:10",
       "exit_code": 0,
-      "duration_sec": 6.2,
-      "cpu_pct": 259.8,
+      "duration_sec": 12.2,
+      "cpu_pct": 261.6,
       "peak_memory_mb": 3380,
-      "disk_read_mb": 133,
-      "disk_write_mb": 1
+      "disk_read_mb": 276,
+      "disk_write_mb": 2
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "nn_retraining_with_pseudolabels",
-    "metric_id": "mean_rowwise_error",
+    "method_id": "scape",
+    "metric_component_name": "mean_rowwise_correlation",
     "resources": {
+      "submit": "2024-10-31 07:48:19",
       "exit_code": 0,
-      "duration_sec": 4.4,
-      "cpu_pct": 455,
+      "duration_sec": 15.3,
+      "cpu_pct": 363.1,
       "peak_memory_mb": 6144,
-      "disk_read_mb": 138,
-      "disk_write_mb": 1
+      "disk_read_mb": 405,
+      "disk_write_mb": 3
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "pyboost",
-    "metric_id": "mean_rowwise_error",
+    "method_id": "scape",
+    "metric_component_name": "mean_rowwise_error",
     "resources": {
+      "submit": "2024-10-31 07:48:19",
       "exit_code": 0,
-      "duration_sec": 4,
-      "cpu_pct": 525,
+      "duration_sec": 8.2,
+      "cpu_pct": 432.8,
       "peak_memory_mb": 5940,
-      "disk_read_mb": 138,
-      "disk_write_mb": 1
+      "disk_read_mb": 270,
+      "disk_write_mb": 2
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "sample",
-    "metric_id": "mean_rowwise_error",
+    "method_id": "transformer_ensemble",
+    "metric_component_name": "mean_rowwise_correlation",
     "resources": {
+      "submit": "2024-10-31 06:33:49",
       "exit_code": 0,
-      "duration_sec": 6.1,
-      "cpu_pct": 261.6,
-      "peak_memory_mb": 3380,
-      "disk_read_mb": 138,
-      "disk_write_mb": 1
+      "duration_sec": 17.4,
+      "cpu_pct": 393.2,
+      "peak_memory_mb": 6144,
+      "disk_read_mb": 399,
+      "disk_write_mb": 3
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "scape",
-    "metric_id": "mean_rowwise_error",
+    "method_id": "transformer_ensemble",
+    "metric_component_name": "mean_rowwise_error",
     "resources": {
+      "submit": "2024-10-31 06:33:49",
       "exit_code": 0,
-      "duration_sec": 4.1,
-      "cpu_pct": 432.8,
-      "peak_memory_mb": 5940,
-      "disk_read_mb": 135,
-      "disk_write_mb": 1
+      "duration_sec": 15.8,
+      "cpu_pct": 245.5,
+      "peak_memory_mb": 6144,
+      "disk_read_mb": 266,
+      "disk_write_mb": 2
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
-    "method_id": "transformer_ensemble",
-    "metric_id": "mean_rowwise_error",
+    "method_id": "zeros",
+    "metric_component_name": "mean_rowwise_correlation",
     "resources": {
+      "submit": "2024-10-31 05:26:29",
       "exit_code": 0,
-      "duration_sec": 7.9,
-      "cpu_pct": 245.5,
+      "duration_sec": 33.3,
+      "cpu_pct": 192.7,
       "peak_memory_mb": 6144,
-      "disk_read_mb": 133,
-      "disk_write_mb": 1
+      "disk_read_mb": 399,
+      "disk_write_mb": 3
     }
   },
   {
     "dataset_id": "neurips-2023-data",
-    "normalization_id": null,
     "method_id": "zeros",
-    "metric_id": "mean_rowwise_error",
+    "metric_component_name": "mean_rowwise_error",
     "resources": {
+      "submit": "2024-10-31 05:26:29",
       "exit_code": 0,
-      "duration_sec": 6.4,
+      "duration_sec": 12.8,
       "cpu_pct": 249.6,
       "peak_memory_mb": 3380,
-      "disk_read_mb": 133,
-      "disk_write_mb": 1
+      "disk_read_mb": 266,
+      "disk_write_mb": 2
     }
   }
 ]
diff --git a/results/perturbation_prediction/data/metric_info.json b/results/perturbation_prediction/data/metric_info.json
index 6007d893..6aec5e87 100644
--- a/results/perturbation_prediction/data/metric_info.json
+++ b/results/perturbation_prediction/data/metric_info.json
@@ -1,6 +1,7 @@
 [
   {
     "task_id": "metrics",
+    "component_name": "mean_rowwise_error",
     "metric_id": "mean_rowwise_rmse",
     "metric_name": "Mean Rowwise RMSE",
     "metric_summary": "The mean of the root mean squared error (RMSE) of each row in the matrix.",
@@ -15,6 +16,7 @@
   },
   {
     "task_id": "metrics",
+    "component_name": "mean_rowwise_error",
     "metric_id": "mean_rowwise_mae",
     "metric_name": "Mean Rowwise MAE",
     "metric_summary": "The mean of the absolute error (MAE) of each row in the matrix.",
@@ -29,6 +31,7 @@
   },
   {
     "task_id": "metrics",
+    "component_name": "mean_rowwise_correlation",
     "metric_id": "mean_rowwise_pearson",
     "metric_name": "Mean Rowwise Pearson",
     "metric_summary": "The mean of Pearson correlations per row (perturbation).",
@@ -43,6 +46,7 @@
   },
   {
     "task_id": "metrics",
+    "component_name": "mean_rowwise_correlation",
     "metric_id": "mean_rowwise_spearman",
     "metric_name": "Mean Rowwise Spearman",
     "metric_summary": "The mean of Spearman correlations per row (perturbation).",
@@ -57,6 +61,7 @@
   },
   {
     "task_id": "metrics",
+    "component_name": "mean_rowwise_correlation",
     "metric_id": "mean_rowwise_cosine",
     "metric_name": "Mean Rowwise Cosine",
     "metric_summary": "The mean of cosine similarities per row (perturbation).",
diff --git a/results/perturbation_prediction/data/results.json b/results/perturbation_prediction/data/results.json
index b5fb6346..246742ba 100644
--- a/results/perturbation_prediction/data/results.json
+++ b/results/perturbation_prediction/data/results.json
@@ -17,8 +17,8 @@
       "mean_rowwise_spearman": 1
     },
     "mean_score": 1,
-    "normalization_id": null,
     "resources": {
+      "submit": "2024-10-31 05:21:23",
       "exit_code": 0,
       "duration_sec": 9,
       "cpu_pct": 206.8,
@@ -45,8 +45,8 @@
       "mean_rowwise_spearman": 0.3054
     },
     "mean_score": 0.3254,
-    "normalization_id": null,
     "resources": {
+      "submit": "2024-10-31 05:21:22",
       "exit_code": 0,
       "duration_sec": 3061,
       "cpu_pct": 100,
@@ -73,8 +73,15 @@
       "mean_rowwise_spearman": 0.4247
     },
     "mean_score": 0.4306,
-    "normalization_id": null,
-    "resources": {}
+    "resources": {
+      "submit": "2024-10-31 05:21:24",
+      "exit_code": 0,
+      "duration_sec": 6026,
+      "cpu_pct": 134.5855,
+      "peak_memory_mb": 50279,
+      "disk_read_mb": 12449,
+      "disk_write_mb": 3592
+    }
   },
   {
     "dataset_id": "neurips-2023-data",
@@ -94,8 +101,8 @@
       "mean_rowwise_spearman": 0.2806
     },
     "mean_score": 0.3113,
-    "normalization_id": null,
     "resources": {
+      "submit": "2024-10-31 05:21:22",
       "exit_code": 0,
       "duration_sec": 3,
       "cpu_pct": 472.1,
@@ -122,8 +129,8 @@
       "mean_rowwise_spearman": 0.2425
     },
     "mean_score": 0.2698,
-    "normalization_id": null,
     "resources": {
+      "submit": "2024-10-31 05:21:23",
       "exit_code": 0,
       "duration_sec": 6.4,
       "cpu_pct": 213.1,
@@ -150,8 +157,8 @@
       "mean_rowwise_spearman": 0.2117
     },
     "mean_score": 0.2677,
-    "normalization_id": null,
     "resources": {
+      "submit": "2024-10-31 05:21:21",
       "exit_code": 0,
       "duration_sec": 5.3,
       "cpu_pct": 265.6,
@@ -178,8 +185,8 @@
       "mean_rowwise_spearman": 0.4627
     },
     "mean_score": 0.4652,
-    "normalization_id": null,
     "resources": {
+      "submit": "2024-10-31 05:21:21",
       "exit_code": 0,
       "duration_sec": 2546,
       "cpu_pct": 1293.7,
@@ -206,8 +213,8 @@
       "mean_rowwise_spearman": 0.439
     },
     "mean_score": 0.4395,
-    "normalization_id": null,
     "resources": {
+      "submit": "2024-10-31 05:21:21",
       "exit_code": 0,
       "duration_sec": 151,
       "cpu_pct": 154.1,
@@ -234,8 +241,8 @@
       "mean_rowwise_spearman": 0.0562
     },
     "mean_score": 0.0326,
-    "normalization_id": null,
     "resources": {
+      "submit": "2024-10-31 05:21:22",
       "exit_code": 0,
       "duration_sec": 8.9,
       "cpu_pct": 216.6,
@@ -262,8 +269,8 @@
       "mean_rowwise_spearman": 0.4417
     },
     "mean_score": 0.4456,
-    "normalization_id": null,
     "resources": {
+      "submit": "2024-10-31 05:21:22",
       "exit_code": 0,
       "duration_sec": 8392,
       "cpu_pct": 737.4,
@@ -290,8 +297,8 @@
       "mean_rowwise_spearman": 0.2164
     },
     "mean_score": 0.2694,
-    "normalization_id": null,
     "resources": {
+      "submit": "2024-10-31 05:21:21",
       "exit_code": 0,
       "duration_sec": 3933,
       "cpu_pct": 216.4,
@@ -318,8 +325,8 @@
       "mean_rowwise_spearman": 0
     },
     "mean_score": 0.1335,
-    "normalization_id": null,
     "resources": {
+      "submit": "2024-10-31 05:21:23",
       "exit_code": 0,
       "duration_sec": 5.8,
       "cpu_pct": 234.9,