From 107c865862972950332ec60fb3071b421e2f6409 Mon Sep 17 00:00:00 2001
From: Ragnar Groot Koerkamp <ragnar.grootkoerkamp@gmail.com>
Date: Mon, 25 Mar 2024 12:53:43 +0100
Subject: [PATCH] A*PA2 readme

---
 README.md                |  2 +-
 evals/astarpa2/README.md | 19 +++++++++++++++++++
 evals/astarpa2/justfile  | 14 ++++++++++++++
 3 files changed, 34 insertions(+), 1 deletion(-)
 create mode 100644 evals/astarpa2/README.md

diff --git a/README.md b/README.md
index 1e825e5..052e287 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@ This repository contains a few things:
 - [`pa-bench`](#pa-bench-benchmarking): a tool to benchmark aligners against each other;
 - [`evals/astarpa`](./evals/astarpa): experiments and analysis for the
   [A\*PA](https://github.com/ragnargrootkoerkamp/astar-pairwise-aligner) paper.
-- [`evals/astarpa-next`](./evals/astarpa-next): the latest experiments and results.
+- [`evals/astarpa2`](./evals/astarpa2): experiments and analysis for the [A\*PA2](https://github.com/ragnargrootkoerkamp/astar-pairwise-aligner) paper.
 
 ## `pa-wrapper`: Wrapper API
 
diff --git a/evals/astarpa2/README.md b/evals/astarpa2/README.md
new file mode 100644
index 0000000..8897e53
--- /dev/null
+++ b/evals/astarpa2/README.md
@@ -0,0 +1,19 @@
+# A\*PA evals
+
+This directory contains the evals for the A\*PA paper.
+
+- `experiments/` contains `.yaml` files with experiment configurations, roughly
+  corresponding to the plots in the paper. (Some `.yaml` files are the source of
+  multiple plots.)
+- Use `make download_results` to download and extract the `results.zip` from the
+  [`astarpa-evals` release](https://github.com/pairwise-alignment/pa-bench/releases/tag/astarpa-evals) on github.
+- Alternatively, `make run` reruns all experiments locally, but be aware that
+  this takes a long time (1-2 days). You could make things faster by running
+  more in parallel (add e.g. `-j 5` to the `run` rule in `makefile`), or by
+  lowering the number of sequences tested (e.g. lower `total_size` in `experiments/*.yaml`).
+- `evals.ipynb` contains all plots and other statistics presented in the paper
+  from the `results/*.json` files generated by one of the two steps above. The
+  version in the repo should correspond to the latest executed version and include all plots.
+
+note: the git history of the `evals.ipynb` file has been squashed to reduce the
+size of the repo.
diff --git a/evals/astarpa2/justfile b/evals/astarpa2/justfile
index 4f51222..9dd3953 100644
--- a/evals/astarpa2/justfile
+++ b/evals/astarpa2/justfile
@@ -1,9 +1,12 @@
+# Main rule to rerun all experiments.
 release: cpufreq
     mkdir -p results
     mv results results-$(date +%Y%m%d-%H%M%S)
     cd ../.. && cargo run -r -- \
         bench --release evals/astarpa2/experiments/*.yaml --cache evals/astarpa2/results/cache.json
 
+# Below here is stuff used for development/debugging.
+
 bug *args='':
     cd ../.. && cargo run -r -- run evals/astarpa2/experiments/bug-pre-pruning.yaml {{args}}
 
@@ -38,7 +41,18 @@ record *args='':
 report *args='':
     cd ../.. && perf report
 
+# Set the CPU frequency 3.6GHz.
 cpufreq:
     sudo cpupower frequency-set --governor performance -d 3.6GHz -u 3.6GHz
 powersave:
     sudo cpupower frequency-set --governor powersave -d 0.8GHz -u 2.6GHz
+
+zip_results:
+    zip results.zip results/*.json
+
+# Download the results from the github release and unzip them.
+# See readme for more.
+download_results:
+    wget 'https://github.com/pairwise-alignment/pa-bench/releases/download/astarpa2-evals/results.zip'
+    unzip results.zip
+    rm results.zip