From 449a2929d34b72f158e37b274348071d8a8dd8b3 Mon Sep 17 00:00:00 2001
From: John Lees <lees.john6@gmail.com>
Date: Sat, 18 Feb 2023 11:34:20 +0000
Subject: [PATCH 1/5] Memory optimisation: don't store whole build array

---
 Cargo.toml            |   2 +-
 src/merge_ska_dict.rs | 157 ++++++++++++++++++++++++++++--------------
 tests/fastq_input.rs  |  34 ++++++++-
 3 files changed, 136 insertions(+), 57 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index cbd81f3..442d9a6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "ska"
-version = "0.2.1"
+version = "0.2.2"
 authors = [
     "John Lees <jlees@ebi.ac.uk>",
     "Simon Harris <simon.harris@gatesfoundation.org>",
diff --git a/src/merge_ska_dict.rs b/src/merge_ska_dict.rs
index 3b66e07..93415ad 100644
--- a/src/merge_ska_dict.rs
+++ b/src/merge_ska_dict.rs
@@ -11,8 +11,7 @@ use core::panic;
 use std::mem;
 
 use hashbrown::HashMap;
-use indicatif::{ParallelProgressIterator, ProgressIterator};
-use rayon::prelude::*;
+use indicatif::ProgressIterator;
 
 use crate::cli::QualFilter;
 use crate::ska_dict::bit_encoding::UInt;
@@ -224,17 +223,31 @@ where
 
 /// Serial `MergeSkaDict::append()` into a [`MergeSkaDict`]
 fn multi_append<IntT>(
-    input_dicts: &mut [SkaDict<IntT>],
+    input_files: &[InputFastx],
+    offset: usize,
     total_size: usize,
     k: usize,
     rc: bool,
+    min_count: u16,
+    min_qual: u8,
+    qual_filter: &QualFilter,
 ) -> MergeSkaDict<IntT>
 where
     IntT: for<'a> UInt<'a>,
 {
     let mut merged_dict = MergeSkaDict::new(k, total_size, rc);
-    for ska_dict in &mut input_dicts.iter() {
-        merged_dict.append(ska_dict);
+    for (idx, (name, filename, second_file)) in input_files.iter().enumerate() {
+        let ska_dict = SkaDict::new(
+            k,
+            idx + offset,
+            (filename, second_file.as_ref()),
+            name,
+            rc,
+            min_count,
+            qual_filter,
+            min_qual,
+        );
+        merged_dict.append(&ska_dict);
     }
     merged_dict
 }
@@ -245,26 +258,77 @@ where
 /// i.e. depth 1 splits in 2, depth 2 splits in 4
 fn parallel_append<IntT>(
     depth: usize,
-    dict_list: &mut [SkaDict<IntT>],
+    offset: usize,
+    file_list: &[InputFastx],
     total_size: usize,
     k: usize,
     rc: bool,
+    min_count: u16,
+    min_qual: u8,
+    qual_filter: &QualFilter,
 ) -> MergeSkaDict<IntT>
 where
     IntT: for<'a> UInt<'a>,
 {
-    let (bottom, top) = dict_list.split_at_mut(dict_list.len() / 2);
+    let split_point = file_list.len() / 2;
+    let (bottom, top) = file_list.split_at(split_point);
     if depth == 1 {
         let (mut bottom_merge, mut top_merge) = rayon::join(
-            || multi_append(bottom, total_size, k, rc),
-            || multi_append(top, total_size, k, rc),
+            || {
+                multi_append(
+                    bottom,
+                    offset,
+                    total_size,
+                    k,
+                    rc,
+                    min_count,
+                    min_qual,
+                    qual_filter,
+                )
+            },
+            || {
+                multi_append(
+                    top,
+                    offset + split_point,
+                    total_size,
+                    k,
+                    rc,
+                    min_count,
+                    min_qual,
+                    qual_filter,
+                )
+            },
         );
         bottom_merge.merge(&mut top_merge);
         bottom_merge
     } else {
         let (mut bottom_merge, mut top_merge) = rayon::join(
-            || parallel_append(depth - 1, bottom, total_size, k, rc),
-            || parallel_append(depth - 1, top, total_size, k, rc),
+            || {
+                parallel_append(
+                    depth - 1,
+                    offset,
+                    bottom,
+                    total_size,
+                    k,
+                    rc,
+                    min_count,
+                    min_qual,
+                    qual_filter,
+                )
+            },
+            || {
+                parallel_append(
+                    depth - 1,
+                    offset + split_point,
+                    top,
+                    total_size,
+                    k,
+                    rc,
+                    min_count,
+                    min_qual,
+                    qual_filter,
+                )
+            },
         );
         bottom_merge.merge(&mut top_merge);
         bottom_merge
@@ -310,67 +374,54 @@ where
     // Build indexes
     log::info!("Building skf dicts from sequence input");
     log::info!(
-        "Read quality filtering criteria: minimum quality {min_qual}/'{}'; filter: {qual_filter}",
+        "FASTQ (may not be used) quality filtering criteria: minimum quality {min_qual}/'{}'; filter: {qual_filter}",
         (min_qual + 33) as char
     );
-    let mut ska_dicts: Vec<SkaDict<IntT>> = Vec::new();
-    ska_dicts.reserve(input_files.len());
     if threads > 1 {
         rayon::ThreadPoolBuilder::new()
             .num_threads(threads)
             .build_global()
             .unwrap();
-        ska_dicts = input_files
-            .par_iter()
-            .progress_count(input_files.len() as u64)
-            .enumerate()
-            .map(|(idx, (name, filename, second_file))| {
-                SkaDict::new(
-                    k,
-                    idx,
-                    (filename, second_file.as_ref()),
-                    name,
-                    rc,
-                    min_count,
-                    qual_filter,
-                    min_qual,
-                )
-            })
-            .collect();
-    } else {
-        for file_it in input_files.iter().progress().enumerate() {
-            let (idx, (name, filename, second_file)) = file_it;
-            ska_dicts.push(SkaDict::new(
-                k,
-                idx,
-                (filename, second_file.as_ref()),
-                name,
-                rc,
-                min_count,
-                qual_filter,
-                min_qual,
-            ))
-        }
     }
 
     // Merge indexes, ensuring at least 10 samples per thread
-    let mut merged_dict = MergeSkaDict::new(k, ska_dicts.len(), rc);
-    let max_threads = usize::max(1, usize::min(threads, 1 + ska_dicts.len() / 10));
+    let total_size = input_files.len();
+    let mut merged_dict = MergeSkaDict::new(k, total_size, rc);
+    let max_threads = usize::max(1, usize::min(threads, 1 + total_size / 10));
     let max_depth = f64::floor(f64::log2(max_threads as f64)) as usize;
     if max_depth > 0 {
         log::info!(
             "{}",
             format!(
-                "Merging skf dicts in parallel using {} threads",
+                "Build and merge skf dicts in parallel using {} threads",
                 1 << max_depth
             )
         );
-        let total_size = ska_dicts.len();
-        merged_dict = parallel_append(max_depth, &mut ska_dicts, total_size, k, rc);
+        merged_dict = parallel_append(
+            max_depth,
+            0,
+            &input_files,
+            total_size,
+            k,
+            rc,
+            min_count,
+            min_qual,
+            qual_filter,
+        );
     } else {
-        log::info!("Merging skf dicts serially");
-        for ska_dict in ska_dicts.iter_mut().progress() {
-            merged_dict.append(ska_dict);
+        log::info!("Build and merge serially");
+        for (idx, (name, filename, second_file)) in input_files.iter().progress().enumerate() {
+            let ska_dict = SkaDict::new(
+                k,
+                idx,
+                (filename, second_file.as_ref()),
+                name,
+                rc,
+                min_count,
+                qual_filter,
+                min_qual,
+            );
+            merged_dict.append(&ska_dict);
         }
     }
     merged_dict
diff --git a/tests/fastq_input.rs b/tests/fastq_input.rs
index 461fc24..ddcb038 100644
--- a/tests/fastq_input.rs
+++ b/tests/fastq_input.rs
@@ -308,7 +308,15 @@ fn error_fastq() {
         .arg(rfile_name)
         .arg("-o")
         .arg("reads")
-        .args(&["--min-count", "5", "-v", "-k", "9", "--qual-filter", "no-filter"])
+        .args(&[
+            "--min-count",
+            "5",
+            "-v",
+            "-k",
+            "9",
+            "--qual-filter",
+            "no-filter",
+        ])
         .assert()
         .success();
 
@@ -331,7 +339,17 @@ fn error_fastq() {
         .arg(rfile_name)
         .arg("-o")
         .arg("reads")
-        .args(&["--min-count", "5", "-v", "-k", "9", "--qual-filter", "middle", "--min-qual", "5"])
+        .args(&[
+            "--min-count",
+            "5",
+            "-v",
+            "-k",
+            "9",
+            "--qual-filter",
+            "middle",
+            "--min-qual",
+            "5",
+        ])
         .assert()
         .success();
 
@@ -401,7 +419,17 @@ fn error_fastq() {
         .arg(rfile_name)
         .arg("-o")
         .arg("reads")
-        .args(&["--min-count", "5", "-v", "-k", "9", "--min-qual", "5", "--qual-filter", "strict"])
+        .args(&[
+            "--min-count",
+            "5",
+            "-v",
+            "-k",
+            "9",
+            "--min-qual",
+            "5",
+            "--qual-filter",
+            "strict",
+        ])
         .assert()
         .success();
 

From bebd7812cca829e5a7d68e95a1015756f7432dd5 Mon Sep 17 00:00:00 2001
From: John Lees <lees.john6@gmail.com>
Date: Sat, 18 Feb 2023 16:40:12 +0000
Subject: [PATCH 2/5] Extend parallel test

---
 tests/align.rs                             |   4 ++--
 tests/test_files_in/par_test/test_1.fa.gz  | Bin 0 -> 84 bytes
 tests/test_files_in/par_test/test_10.fa.gz | Bin 84 -> 85 bytes
 tests/test_files_in/par_test/test_11.fa.gz | Bin 84 -> 85 bytes
 tests/test_files_in/par_test/test_12.fa.gz | Bin 84 -> 85 bytes
 tests/test_files_in/par_test/test_13.fa.gz | Bin 84 -> 85 bytes
 tests/test_files_in/par_test/test_14.fa.gz | Bin 84 -> 85 bytes
 tests/test_files_in/par_test/test_15.fa.gz | Bin 84 -> 85 bytes
 tests/test_files_in/par_test/test_16.fa.gz | Bin 84 -> 85 bytes
 tests/test_files_in/par_test/test_17.fa.gz | Bin 84 -> 85 bytes
 tests/test_files_in/par_test/test_18.fa.gz | Bin 84 -> 85 bytes
 tests/test_files_in/par_test/test_19.fa.gz | Bin 84 -> 85 bytes
 tests/test_files_in/par_test/test_2.fa.gz  | Bin 84 -> 84 bytes
 tests/test_files_in/par_test/test_20.fa.gz | Bin 84 -> 85 bytes
 tests/test_files_in/par_test/test_21.fa.gz | Bin 84 -> 85 bytes
 tests/test_files_in/par_test/test_22.fa.gz | Bin 84 -> 85 bytes
 tests/test_files_in/par_test/test_23.fa.gz | Bin 84 -> 85 bytes
 tests/test_files_in/par_test/test_24.fa.gz | Bin 84 -> 85 bytes
 tests/test_files_in/par_test/test_25.fa.gz | Bin 84 -> 85 bytes
 tests/test_files_in/par_test/test_26.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_27.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_28.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_29.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_3.fa.gz  | Bin 84 -> 84 bytes
 tests/test_files_in/par_test/test_30.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_31.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_32.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_33.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_34.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_35.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_36.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_37.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_38.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_39.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_4.fa.gz  | Bin 84 -> 84 bytes
 tests/test_files_in/par_test/test_40.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_41.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_42.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_43.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_44.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_45.fa.gz | Bin 0 -> 85 bytes
 tests/test_files_in/par_test/test_5.fa.gz  | Bin 84 -> 84 bytes
 tests/test_files_in/par_test/test_6.fa.gz  | Bin 84 -> 84 bytes
 tests/test_files_in/par_test/test_7.fa.gz  | Bin 84 -> 84 bytes
 tests/test_files_in/par_test/test_8.fa.gz  | Bin 84 -> 84 bytes
 tests/test_files_in/par_test/test_9.fa.gz  | Bin 84 -> 84 bytes
 46 files changed, 2 insertions(+), 2 deletions(-)
 create mode 100644 tests/test_files_in/par_test/test_1.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_26.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_27.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_28.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_29.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_30.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_31.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_32.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_33.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_34.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_35.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_36.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_37.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_38.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_39.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_40.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_41.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_42.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_43.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_44.fa.gz
 create mode 100644 tests/test_files_in/par_test/test_45.fa.gz

diff --git a/tests/align.rs b/tests/align.rs
index e3db9dd..8f3a099 100644
--- a/tests/align.rs
+++ b/tests/align.rs
@@ -205,7 +205,7 @@ fn filters() {
 fn parallel_align() {
     let sandbox = TestSetup::setup();
     // Needs at least ten samples per threads, so make lots of copies
-    // (confimed with -v that this actually uses the multi-thread algorithm)
+    // (confirmed with -v that this actually uses the multi-thread algorithm)
     let rfile_name = sandbox.create_par_rfile();
 
     // Serial alignment
@@ -236,7 +236,7 @@ fn parallel_align() {
         .arg(rfile_name)
         .arg("-o")
         .arg("parallel_build")
-        .args(&["-v", "--threads", "2", "-k", "15"])
+        .args(&["-v", "--threads", "4", "-k", "15"])
         .assert()
         .success();
 
diff --git a/tests/test_files_in/par_test/test_1.fa.gz b/tests/test_files_in/par_test/test_1.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..7755d84357df542b44d336d7287ef3514e41c350
GIT binary patch
literal 84
zcmV-a0IUBWiwFqv{_tY}19W9`bYC$pW?=xcD^AQU$VoNia(4^~32}FHba!`l2eKT2
q>=2NUqq}p6dx)d6yCV>WIER3Qz)X-TkRo>wg9`xYifsEx0001_CL!4X

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_10.fa.gz b/tests/test_files_in/par_test/test_10.fa.gz
index abffbbffcddd153c49a502ddf1f44892ec72923a..64f8f1158b5a081dd56a295d778125dd68d15a6e 100644
GIT binary patch
delta 23
ecmWFu6_oGh;INc9o6Nvml3H96Z)h-4zy$zEU<O_Q

delta 22
dcmWFy5s>fZ;Fu<|DVc$}B(=CC-f*Iz3jjtU25bNT

diff --git a/tests/test_files_in/par_test/test_11.fa.gz b/tests/test_files_in/par_test/test_11.fa.gz
index abffbbffcddd153c49a502ddf1f44892ec72923a..73e2d33d6866c91226ccc6b2fd59e708d9edd93f 100644
GIT binary patch
delta 22
dcmWFu6_D@d;INc9o6Nvml3H96Z#a?P6#zm)1}y*p

delta 21
ccmWFy;g|2`;Fu<|DVc$}B(=CCexiUY06+BxIsgCw

diff --git a/tests/test_files_in/par_test/test_12.fa.gz b/tests/test_files_in/par_test/test_12.fa.gz
index 7b9a22a9a264712df6c426c794392bb3aaf7d247..66159c556af3632a9de8c00dc7dbb934ff0df4c1 100644
GIT binary patch
delta 22
dcmWFu6_D@d;Bb*Ro6Nvml3H96Z#a?P6#zo#1~>o!

delta 21
ccmWFy;g|2`;Fu<|DVc$}B(=CCexiUY06+BxIsgCw

diff --git a/tests/test_files_in/par_test/test_13.fa.gz b/tests/test_files_in/par_test/test_13.fa.gz
index 7b9a22a9a264712df6c426c794392bb3aaf7d247..d3ed1bcddc32055cd3c0fb90a17e90fe8b64473e 100644
GIT binary patch
delta 23
ecmWFu6_oGh;Bb*Ro6Nvml3H96Z)iMGzy$zF90qIv

delta 22
dcmWFy5s>fZ;Fu<|DVc$}B(=CC-e{tr3jjtY25kTU

diff --git a/tests/test_files_in/par_test/test_14.fa.gz b/tests/test_files_in/par_test/test_14.fa.gz
index 7b9a22a9a264712df6c426c794392bb3aaf7d247..bb87e7a71b12cee9f1b199f375f48d0a43c779aa 100644
GIT binary patch
delta 23
ecmWFu6_oGh;Bb*Ro6Nvml3H96Z)h@6zy$zFAO>v!

delta 22
dcmWFy5s>fZ;Fu<|DVc$}B(=CC-e{tr3jjtY25kTU

diff --git a/tests/test_files_in/par_test/test_15.fa.gz b/tests/test_files_in/par_test/test_15.fa.gz
index 7b9a22a9a264712df6c426c794392bb3aaf7d247..a782099ef51b7f1bcdbbfd33c9ec4044a8cfc81d 100644
GIT binary patch
delta 23
ecmWFu6_oGh;Bb*Ro6Nvml3H96Z)iGEzy$zFBnEB(

delta 22
dcmWFy5s>fZ;Fu<|DVc$}B(=CC-e{tr3jjtY25kTU

diff --git a/tests/test_files_in/par_test/test_16.fa.gz b/tests/test_files_in/par_test/test_16.fa.gz
index 7b9a22a9a264712df6c426c794392bb3aaf7d247..077d1aa9e534d268366b71bc85ea12ba56233a76 100644
GIT binary patch
delta 23
ecmWFu6_oGh;Bb*Ro6Nvml3H96Z)i4Azy$zFC<bo;

delta 22
dcmWFy5s>fZ;Fu<|DVc$}B(=CC-e{tr3jjtY25kTU

diff --git a/tests/test_files_in/par_test/test_17.fa.gz b/tests/test_files_in/par_test/test_17.fa.gz
index 7b9a22a9a264712df6c426c794392bb3aaf7d247..2cceeba645ddd2d2890a974b108d8c9644a345dc 100644
GIT binary patch
delta 23
ecmWFu6_oGh;Bb*Ro6Nvml3H96Z)iSIzy$zFECz4@

delta 22
dcmWFy5s>fZ;Fu<|DVc$}B(=CC-e{tr3jjtY25kTU

diff --git a/tests/test_files_in/par_test/test_18.fa.gz b/tests/test_files_in/par_test/test_18.fa.gz
index 7b9a22a9a264712df6c426c794392bb3aaf7d247..e8a496ca039675f51d16e1132f1d68923e512b9d 100644
GIT binary patch
delta 23
ecmWFu6_oGh;Bb*Ro6Nvml3H96Z)h=5zy$zFFa~h|

delta 22
dcmWFy5s>fZ;Fu<|DVc$}B(=CC-e{tr3jjtY25kTU

diff --git a/tests/test_files_in/par_test/test_19.fa.gz b/tests/test_files_in/par_test/test_19.fa.gz
index 7b9a22a9a264712df6c426c794392bb3aaf7d247..918295e2a35bbfa55149fce27eb4938d9bc154b6 100644
GIT binary patch
delta 23
ecmWFu6_oGh;Bb*Ro6Nvml3H96Z)iDDzy$zFGzM}2

delta 22
dcmWFy5s>fZ;Fu<|DVc$}B(=CC-e{tr3jjtY25kTU

diff --git a/tests/test_files_in/par_test/test_2.fa.gz b/tests/test_files_in/par_test/test_2.fa.gz
index abffbbffcddd153c49a502ddf1f44892ec72923a..5ba9e6e5cca4004a2f808e2e756ade5a4e8f23a8 100644
GIT binary patch
delta 22
dcmWFu5s>fZ;INc9o6Nvml3H96Z!}TB1pq=a1}y*p

delta 22
dcmWFu5s>fZ;Fu<|DVc$}B(=CC-f*IT3jjt625JBR

diff --git a/tests/test_files_in/par_test/test_20.fa.gz b/tests/test_files_in/par_test/test_20.fa.gz
index 7b9a22a9a264712df6c426c794392bb3aaf7d247..33c0d9c3efabfa080290ec407f46738cb508d0f1 100644
GIT binary patch
delta 23
ecmWFu6_oGh;Bb*Ro6Nvml3H96Z)7l0zy$zF6$WYm

delta 22
dcmWFy5s>fZ;Fu<|DVc$}B(=CC-e{tr3jjtY25kTU

diff --git a/tests/test_files_in/par_test/test_21.fa.gz b/tests/test_files_in/par_test/test_21.fa.gz
index 7b9a22a9a264712df6c426c794392bb3aaf7d247..ecc66f44651767dbb7b07ef13f3eddfc419bf95b 100644
GIT binary patch
delta 23
ecmWFu6_oGh;Bb*Ro6Nvml3H96Z)7-8zy$zF83t<r

delta 22
dcmWFy5s>fZ;Fu<|DVc$}B(=CC-e{tr3jjtY25kTU

diff --git a/tests/test_files_in/par_test/test_22.fa.gz b/tests/test_files_in/par_test/test_22.fa.gz
index 7b9a22a9a264712df6c426c794392bb3aaf7d247..a96c7d7b33361575bd1baddd17b6a3ec5308b60a 100644
GIT binary patch
delta 22
dcmWFu6_D@d;Bb*Ro6Nvml3H96Z#0qL6#zo(1~~u#

delta 21
ccmWFy;g|2`;Fu<|DVc$}B(=CCexiUY06+BxIsgCw

diff --git a/tests/test_files_in/par_test/test_23.fa.gz b/tests/test_files_in/par_test/test_23.fa.gz
index 7b9a22a9a264712df6c426c794392bb3aaf7d247..daffefc7ce71af794139ccabafe25b530f37ae5d 100644
GIT binary patch
delta 23
ecmWFu6_oGh;Bb*Ro6Nvml3H96Z)7}Czy$zFAqH&#

delta 22
dcmWFy5s>fZ;Fu<|DVc$}B(=CC-e{tr3jjtY25kTU

diff --git a/tests/test_files_in/par_test/test_24.fa.gz b/tests/test_files_in/par_test/test_24.fa.gz
index 7b9a22a9a264712df6c426c794392bb3aaf7d247..a66df39b6911bf224e09813b0741ae0216fc718f 100644
GIT binary patch
delta 23
ecmWFu6_oGh;Bb*Ro6Nvml3H96Z)7r2zy$zFB?fK)

delta 22
dcmWFy5s>fZ;Fu<|DVc$}B(=CC-e{tr3jjtY25kTU

diff --git a/tests/test_files_in/par_test/test_25.fa.gz b/tests/test_files_in/par_test/test_25.fa.gz
index 7b9a22a9a264712df6c426c794392bb3aaf7d247..12787dfcce48b0a03e9a4954a38720844d2bf692 100644
GIT binary patch
delta 61
zcmV-D0K)%NRS_SH2nYfH@M8c2bY*jNUoth35FaIwkfXbEh<k{mv%4b@hB$|SguqOY
TDv%;~5Q7T<=!$InNB{r;xThBF

delta 60
zcmWFy5s>fZ;Fu<|DVc$}B(=CC-e{trysAM-IbR`vVM$pz%OQd10uy8ecC<HyG>YtO
QmWW}vbipgogMono0660lP5=M^

diff --git a/tests/test_files_in/par_test/test_26.fa.gz b/tests/test_files_in/par_test/test_26.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..7df7602b5ee3fccc9c3e0539a53246776c1074ea
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC(yE@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;ZCW89

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_27.fa.gz b/tests/test_files_in/par_test/test_27.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..a8b40a2d48e4eab4eb3de6be7b778ef62c2f5552
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC(zE@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;ZJ!|_

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_28.fa.gz b/tests/test_files_in/par_test/test_28.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..4ba58accca6a22ac476003b0e9045cec1d2ccc05
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC(!E@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;ZR8;$

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_29.fa.gz b/tests/test_files_in/par_test/test_29.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..e0bd274a8169d578d20584bbdd74ca164530678d
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC(#E@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;ZYd!n

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_3.fa.gz b/tests/test_files_in/par_test/test_3.fa.gz
index abffbbffcddd153c49a502ddf1f44892ec72923a..d47ef1f2b436871397e30f92a4112c5283fbea00 100644
GIT binary patch
delta 22
dcmWFu5s>fZ;INc9o6Nvml3H96Z#+@J1pq=e1}*>q

delta 22
dcmWFu5s>fZ;Fu<|DVc$}B(=CC-f*IT3jjt625JBR

diff --git a/tests/test_files_in/par_test/test_30.fa.gz b/tests/test_files_in/par_test/test_30.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..a275f36a25eddd537d5e2e0f36620cacb2c40058
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC+tE@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;Yyu%0

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_31.fa.gz b/tests/test_files_in/par_test/test_31.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..c07a5708addfadd5d5ad54d1ae60d2bebf229188
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC+uE@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;Y)2s+

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_32.fa.gz b/tests/test_files_in/par_test/test_32.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..72019dbda861db0681fdfdd1b3a54cf8041f0ab8
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC+vE@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;Y>Xit

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_33.fa.gz b/tests/test_files_in/par_test/test_33.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..0ce2394724dda5b1b7ba21e3e91b10045b4243c5
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC+wE@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;Y|$Ye

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_34.fa.gz b/tests/test_files_in/par_test/test_34.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..3f73fbddda159543d097edd09557995de167a023
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC+xE@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;Z5AOP

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_35.fa.gz b/tests/test_files_in/par_test/test_35.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..3b190f24e58014273b003e353b22a406f26146d9
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC+yE@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;ZCfEA

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_36.fa.gz b/tests/test_files_in/par_test/test_36.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..7ca1d62b448637c8f43d9823ea0ba71224b3bcd5
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC+zE@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;ZJ;3`

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_37.fa.gz b/tests/test_files_in/par_test/test_37.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..6f1d417001de1e64e5482f2c54a0ba4d3ea5c765
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC+!E@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;ZRH^%

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_38.fa.gz b/tests/test_files_in/par_test/test_38.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..28599d530a99866e6b7e1ace4a97936e0807db37
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC+#E@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;ZYm)o

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_39.fa.gz b/tests/test_files_in/par_test/test_39.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..c3ad1199ce7e66d2cc388fcf44383591cfa6ccca
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC+$E@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;Zf_wZ

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_4.fa.gz b/tests/test_files_in/par_test/test_4.fa.gz
index abffbbffcddd153c49a502ddf1f44892ec72923a..88500fd58551943554f0170b687c4d1f20fd0422 100644
GIT binary patch
delta 22
dcmWFu5s>fZ;INc9o6Nvml3H96Z!%H91pq=i1}^{r

delta 22
dcmWFu5s>fZ;Fu<|DVc$}B(=CC-f*IT3jjt625JBR

diff --git a/tests/test_files_in/par_test/test_40.fa.gz b/tests/test_files_in/par_test/test_40.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..b2a6e1f2d1ffac4c12e1d3ba94a64f2d03b7f98e
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC<uE@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;Y)By-

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_41.fa.gz b/tests/test_files_in/par_test/test_41.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..76185be1afeb2a060826510b9b95210a17d130f9
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC<vE@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;Y>gou

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_42.fa.gz b/tests/test_files_in/par_test/test_42.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..85a4aaad989ea43bef2aedc61b959192cdd41b93
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC<wE@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;Y|<ef

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_43.fa.gz b/tests/test_files_in/par_test/test_43.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..a0b867ec754038b8c606fe6f53881d4cb5c965c4
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC<xE@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;Z5JUQ

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_44.fa.gz b/tests/test_files_in/par_test/test_44.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..b088d7e8b072885c5c1812fc1d68379229434712
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC<yE@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;ZCoKB

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_45.fa.gz b/tests/test_files_in/par_test/test_45.fa.gz
new file mode 100644
index 0000000000000000000000000000000000000000..e316cd94777aefe95f3f3d04c5e660bc871ba3fd
GIT binary patch
literal 85
zcmV-b0IL5ViwFn-|L|h~19W9`bYC<zE@oi>vnx)_EyzhV<Z^cm2?=p`baZ!jb_cQ?
rf$R{FkfXbEh<k{mv%4b@hB$|SguqOYDv%;~5Q7T<=!$InNB{r;ZJ{9{

literal 0
HcmV?d00001

diff --git a/tests/test_files_in/par_test/test_5.fa.gz b/tests/test_files_in/par_test/test_5.fa.gz
index abffbbffcddd153c49a502ddf1f44892ec72923a..e534b7627f735b2ff4f23af0430abb56a035fbb8 100644
GIT binary patch
delta 22
dcmWFu5s>fZ;INc9o6Nvml3H96Z#q%H1pq=m1~32s

delta 22
dcmWFu5s>fZ;Fu<|DVc$}B(=CC-f*IT3jjt625JBR

diff --git a/tests/test_files_in/par_test/test_6.fa.gz b/tests/test_files_in/par_test/test_6.fa.gz
index abffbbffcddd153c49a502ddf1f44892ec72923a..26329ca28fef83b9d4eecfdfaf90e408effa0b2d 100644
GIT binary patch
delta 22
dcmWFu5s>fZ;INc9o6Nvml3H96Z#GfD1pq=q1~C8t

delta 22
dcmWFu5s>fZ;Fu<|DVc$}B(=CC-f*IT3jjt625JBR

diff --git a/tests/test_files_in/par_test/test_7.fa.gz b/tests/test_files_in/par_test/test_7.fa.gz
index abffbbffcddd153c49a502ddf1f44892ec72923a..30cfc5abcec1d9c8d4bdfcbaf495109e0e58cf3a 100644
GIT binary patch
delta 22
dcmWFu5s>fZ;INc9o6Nvml3H96Z$44L1pq=u1~LEu

delta 22
dcmWFu5s>fZ;Fu<|DVc$}B(=CC-f*IT3jjt625JBR

diff --git a/tests/test_files_in/par_test/test_8.fa.gz b/tests/test_files_in/par_test/test_8.fa.gz
index abffbbffcddd153c49a502ddf1f44892ec72923a..5baee3ac1a7ac51aac1775f09991399e98c52578 100644
GIT binary patch
delta 22
dcmWFu5s>fZ;INc9o6Nvml3H96Z!uB81pq=y1~UKv

delta 22
dcmWFu5s>fZ;Fu<|DVc$}B(=CC-f*IT3jjt625JBR

diff --git a/tests/test_files_in/par_test/test_9.fa.gz b/tests/test_files_in/par_test/test_9.fa.gz
index abffbbffcddd153c49a502ddf1f44892ec72923a..5aafdc6cdb7546ddd048194e1d2ac9130f755500 100644
GIT binary patch
delta 22
dcmWFu5s>fZ;INc9o6Nvml3H96Z#hxG1pq=$1~dQw

delta 22
dcmWFu5s>fZ;Fu<|DVc$}B(=CC-f*IT3jjt625JBR


From 1e2b294e4dcbf3c03454dc39ed015b42d42626b5 Mon Sep 17 00:00:00 2001
From: John Lees <lees.john6@gmail.com>
Date: Sat, 18 Feb 2023 17:06:03 +0000
Subject: [PATCH 3/5] Move some of the enums/structs, add docs

---
 src/cli.rs                 | 23 +---------
 src/io_utils.rs            | 10 +++--
 src/lib.rs                 | 85 +++++++++++++++++++++++++++--------
 src/merge_ska_dict.rs      | 90 +++++---------------------------------
 src/ska_dict.rs            | 24 ++++------
 src/ska_dict/split_kmer.rs |  2 +-
 src/ska_ref.rs             |  3 +-
 7 files changed, 99 insertions(+), 138 deletions(-)

diff --git a/src/cli.rs b/src/cli.rs
index 4499d41..a51a9de 100644
--- a/src/cli.rs
+++ b/src/cli.rs
@@ -5,6 +5,8 @@ use clap::{ArgGroup, Parser, Subcommand, ValueEnum};
 
 extern crate num_cpus;
 
+use super::QualFilter;
+
 /// Default split k-mer size
 pub const DEFAULT_KMER: usize = 17;
 /// Default single strand (which is equivalent to !rc)
@@ -84,27 +86,6 @@ impl fmt::Display for FilterType {
     }
 }
 
-/// Possible quality score filters when building with reads
-#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, ValueEnum)]
-pub enum QualFilter {
-    /// Ignore quality scores in reads
-    NoFilter,
-    /// Filter middle bases below quality threshold
-    Middle,
-    /// Filter entire k-mer when any base below quality threshold
-    Strict,
-}
-
-impl fmt::Display for QualFilter {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match *self {
-            Self::NoFilter => write!(f, "No quality filtering"),
-            Self::Middle => write!(f, "Middle base quality filtering"),
-            Self::Strict => write!(f, "Whole k-mer quality filtering"),
-        }
-    }
-}
-
 /// Options that apply to all subcommands
 #[derive(Parser)]
 #[command(author, version, about, long_about = None)]
diff --git a/src/io_utils.rs b/src/io_utils.rs
index e5890ec..c06cd1e 100644
--- a/src/io_utils.rs
+++ b/src/io_utils.rs
@@ -10,6 +10,7 @@ use std::path::Path;
 
 use regex::Regex;
 
+use super::QualOpts;
 use crate::merge_ska_array::MergeSkaArray;
 use crate::merge_ska_dict::{build_and_merge, InputFastx};
 use crate::ska_dict::bit_encoding::UInt;
@@ -63,13 +64,16 @@ pub fn load_array<IntT: for<'a> UInt<'a>>(
     } else {
         log::info!("Multiple files as input, running ska build with default settings");
         let input_files = read_input_fastas(input);
+        let default_qual = QualOpts {
+            min_count: DEFAULT_MINCOUNT,
+            min_qual: DEFAULT_MINQUAL,
+            qual_filter: DEFAULT_QUALFILTER,
+        };
         let merged_dict = build_and_merge(
             &input_files,
             DEFAULT_KMER,
             !DEFAULT_STRAND,
-            DEFAULT_MINCOUNT,
-            DEFAULT_MINQUAL,
-            &DEFAULT_QUALFILTER,
+            &default_qual,
             threads,
         );
         Ok(MergeSkaArray::new(&merged_dict))
diff --git a/src/lib.rs b/src/lib.rs
index 75c6e39..8c8cfaa 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -98,6 +98,20 @@
 //! FASTQ files must be paired end. If you'd like to request more flexibility in
 //! this regard please contact us.
 //!
+//! ### Threads
+//!
+//! The maximum threads actually used will be a power of two, so if you provided
+//! `--threads 6` only four would be used. Additionally, at least ten samples per
+//! thread are required so maximums are:
+//!
+//! | Samples | Maximum threads |
+//! |---------|-----------------|
+//! | 1-19    | 1               |
+//! | 20-39   | 2               |
+//! | 40-79   | 4               |
+//!
+//! and so on. Use `-v` to see a message with the number being used.
+//!
 //! ## ska align
 //!
 //! Creates an alignment from a `.skf` file or sequence files. Sites (columns) are
@@ -280,8 +294,11 @@
 //!
 
 #![warn(missing_docs)]
+use std::fmt;
 use std::time::Instant;
 
+use clap::ValueEnum;
+
 pub mod merge_ska_dict;
 pub mod ska_dict;
 use crate::merge_ska_dict::build_and_merge;
@@ -300,6 +317,49 @@ use crate::cli::*;
 pub mod io_utils;
 use crate::io_utils::*;
 
+/// Possible quality score filters when building with reads
+#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, ValueEnum)]
+pub enum QualFilter {
+    /// Ignore quality scores in reads
+    NoFilter,
+    /// Filter middle bases below quality threshold
+    Middle,
+    /// Filter entire k-mer when any base below quality threshold
+    Strict,
+}
+
+impl fmt::Display for QualFilter {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            Self::NoFilter => write!(f, "No quality filtering"),
+            Self::Middle => write!(f, "Middle base quality filtering"),
+            Self::Strict => write!(f, "Whole k-mer quality filtering"),
+        }
+    }
+}
+
+/// Quality filtering options for FASTQ files
+pub struct QualOpts {
+    /// Minimum k-mer count across reads to be added
+    pub min_count: u16,
+    /// Minimum base quality to be added
+    pub min_qual: u8,
+    /// [`QualFilter`]: apply quality across whole k-mer, just middle base, or not at all
+    pub qual_filter: QualFilter,
+}
+
+impl fmt::Display for QualOpts {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "minimum quality {} ({}); filter: {}",
+            self.min_qual,
+            (self.min_qual + 33) as char,
+            self.qual_filter
+        )
+    }
+}
+
 #[doc(hidden)]
 pub fn main() {
     let args = cli_args();
@@ -323,34 +383,23 @@ pub fn main() {
         } => {
             // Read input
             let input_files = get_input_list(file_list, seq_files);
+            let quality = QualOpts {
+                min_count: *min_count,
+                min_qual: *min_qual,
+                qual_filter: *qual_filter,
+            };
 
             // Build, merge
             let rc = !*single_strand;
             if *k <= 31 {
                 log::info!("k={}: using 64-bit representation", *k);
-                let merged_dict = build_and_merge::<u64>(
-                    &input_files,
-                    *k,
-                    rc,
-                    *min_count,
-                    *min_qual,
-                    qual_filter,
-                    *threads,
-                );
+                let merged_dict = build_and_merge::<u64>(&input_files, *k, rc, &quality, *threads);
 
                 // Save
                 save_skf(&merged_dict, format!("{output}.skf").as_str());
             } else {
                 log::info!("k={}: using 128-bit representation", *k);
-                let merged_dict = build_and_merge::<u128>(
-                    &input_files,
-                    *k,
-                    rc,
-                    *min_count,
-                    *min_qual,
-                    qual_filter,
-                    *threads,
-                );
+                let merged_dict = build_and_merge::<u128>(&input_files, *k, rc, &quality, *threads);
 
                 // Save
                 save_skf(&merged_dict, format!("{output}.skf").as_str());
diff --git a/src/merge_ska_dict.rs b/src/merge_ska_dict.rs
index 93415ad..88835dd 100644
--- a/src/merge_ska_dict.rs
+++ b/src/merge_ska_dict.rs
@@ -13,7 +13,7 @@ use std::mem;
 use hashbrown::HashMap;
 use indicatif::ProgressIterator;
 
-use crate::cli::QualFilter;
+use super::QualOpts;
 use crate::ska_dict::bit_encoding::UInt;
 use crate::ska_dict::SkaDict;
 
@@ -228,9 +228,7 @@ fn multi_append<IntT>(
     total_size: usize,
     k: usize,
     rc: bool,
-    min_count: u16,
-    min_qual: u8,
-    qual_filter: &QualFilter,
+    qual: &QualOpts,
 ) -> MergeSkaDict<IntT>
 where
     IntT: for<'a> UInt<'a>,
@@ -243,9 +241,7 @@ where
             (filename, second_file.as_ref()),
             name,
             rc,
-            min_count,
-            qual_filter,
-            min_qual,
+            qual,
         );
         merged_dict.append(&ska_dict);
     }
@@ -263,9 +259,7 @@ fn parallel_append<IntT>(
     total_size: usize,
     k: usize,
     rc: bool,
-    min_count: u16,
-    min_qual: u8,
-    qual_filter: &QualFilter,
+    qual: &QualOpts,
 ) -> MergeSkaDict<IntT>
 where
     IntT: for<'a> UInt<'a>,
@@ -274,48 +268,14 @@ where
     let (bottom, top) = file_list.split_at(split_point);
     if depth == 1 {
         let (mut bottom_merge, mut top_merge) = rayon::join(
-            || {
-                multi_append(
-                    bottom,
-                    offset,
-                    total_size,
-                    k,
-                    rc,
-                    min_count,
-                    min_qual,
-                    qual_filter,
-                )
-            },
-            || {
-                multi_append(
-                    top,
-                    offset + split_point,
-                    total_size,
-                    k,
-                    rc,
-                    min_count,
-                    min_qual,
-                    qual_filter,
-                )
-            },
+            || multi_append(bottom, offset, total_size, k, rc, qual),
+            || multi_append(top, offset + split_point, total_size, k, rc, qual),
         );
         bottom_merge.merge(&mut top_merge);
         bottom_merge
     } else {
         let (mut bottom_merge, mut top_merge) = rayon::join(
-            || {
-                parallel_append(
-                    depth - 1,
-                    offset,
-                    bottom,
-                    total_size,
-                    k,
-                    rc,
-                    min_count,
-                    min_qual,
-                    qual_filter,
-                )
-            },
+            || parallel_append(depth - 1, offset, bottom, total_size, k, rc, qual),
             || {
                 parallel_append(
                     depth - 1,
@@ -324,9 +284,7 @@ where
                     total_size,
                     k,
                     rc,
-                    min_count,
-                    min_qual,
-                    qual_filter,
+                    qual,
                 )
             },
         );
@@ -363,9 +321,7 @@ pub fn build_and_merge<IntT>(
     input_files: &[InputFastx],
     k: usize,
     rc: bool,
-    min_count: u16,
-    min_qual: u8,
-    qual_filter: &QualFilter,
+    qual: &QualOpts,
     threads: usize,
 ) -> MergeSkaDict<IntT>
 where
@@ -373,10 +329,7 @@ where
 {
     // Build indexes
     log::info!("Building skf dicts from sequence input");
-    log::info!(
-        "FASTQ (may not be used) quality filtering criteria: minimum quality {min_qual}/'{}'; filter: {qual_filter}",
-        (min_qual + 33) as char
-    );
+    log::info!("If FASTQ input: {qual}");
     if threads > 1 {
         rayon::ThreadPoolBuilder::new()
             .num_threads(threads)
@@ -397,30 +350,11 @@ where
                 1 << max_depth
             )
         );
-        merged_dict = parallel_append(
-            max_depth,
-            0,
-            &input_files,
-            total_size,
-            k,
-            rc,
-            min_count,
-            min_qual,
-            qual_filter,
-        );
+        merged_dict = parallel_append(max_depth, 0, input_files, total_size, k, rc, qual);
     } else {
         log::info!("Build and merge serially");
         for (idx, (name, filename, second_file)) in input_files.iter().progress().enumerate() {
-            let ska_dict = SkaDict::new(
-                k,
-                idx,
-                (filename, second_file.as_ref()),
-                name,
-                rc,
-                min_count,
-                qual_filter,
-                min_qual,
-            );
+            let ska_dict = SkaDict::new(k, idx, (filename, second_file.as_ref()), name, rc, qual);
             merged_dict.append(&ska_dict);
         }
     }
diff --git a/src/ska_dict.rs b/src/ska_dict.rs
index c8f06c0..3ddc42d 100644
--- a/src/ska_dict.rs
+++ b/src/ska_dict.rs
@@ -21,7 +21,7 @@ use hashbrown::HashMap;
 use needletail::{parse_fastx_file, parser::Format};
 
 pub mod split_kmer;
-use crate::cli::QualFilter;
+use super::QualOpts;
 use crate::ska_dict::split_kmer::SplitKmer;
 
 pub mod bit_encoding;
@@ -70,13 +70,7 @@ where
 
     /// Iterates through all the k-mers from an input fastx file and adds them
     /// to the dictionary
-    fn add_file_kmers(
-        &mut self,
-        filename: &str,
-        is_reads: bool,
-        qual_filter: &QualFilter,
-        min_qual: u8,
-    ) {
+    fn add_file_kmers(&mut self, filename: &str, is_reads: bool, qual: &QualOpts) {
         let mut reader =
             parse_fastx_file(filename).unwrap_or_else(|_| panic!("Invalid path/file: {filename}"));
         while let Some(record) = reader.next() {
@@ -87,8 +81,8 @@ where
                 seqrec.qual(),
                 self.k,
                 self.rc,
-                min_qual,
-                *qual_filter,
+                qual.min_qual,
+                qual.qual_filter,
                 is_reads,
             );
             if let Some(mut kmer_it) = kmer_opt {
@@ -154,9 +148,7 @@ where
         files: (&str, Option<&String>),
         name: &str,
         rc: bool,
-        min_count: u16,
-        qual_filter: &QualFilter,
-        min_qual: u8,
+        qual: &QualOpts,
     ) -> Self {
         if !(5..=63).contains(&k) || k % 2 == 0 {
             panic!("Invalid k-mer length");
@@ -168,7 +160,7 @@ where
             sample_idx,
             name: name.to_string(),
             split_kmers: HashMap::default(),
-            cm_filter: CountMin::empty(CM_WIDTH, CM_HEIGHT, min_count),
+            cm_filter: CountMin::empty(CM_WIDTH, CM_HEIGHT, qual.min_count),
         };
 
         // Check if we're working with reads, and initalise the CM filter if so
@@ -185,9 +177,9 @@ where
         }
 
         // Build the dict
-        sk_dict.add_file_kmers(files.0, is_reads, qual_filter, min_qual);
+        sk_dict.add_file_kmers(files.0, is_reads, qual);
         if let Some(second_filename) = files.1 {
-            sk_dict.add_file_kmers(second_filename, is_reads, qual_filter, min_qual);
+            sk_dict.add_file_kmers(second_filename, is_reads, qual);
         }
 
         if sk_dict.ksize() == 0 {
diff --git a/src/ska_dict/split_kmer.rs b/src/ska_dict/split_kmer.rs
index 5c38820..c67714a 100644
--- a/src/ska_dict/split_kmer.rs
+++ b/src/ska_dict/split_kmer.rs
@@ -13,7 +13,7 @@
 use std::borrow::Cow;
 use std::cmp::Ordering;
 
-use crate::cli::QualFilter;
+use super::super::QualFilter;
 
 use super::bit_encoding::*;
 use super::nthash::NtHashIterator;
diff --git a/src/ska_ref.rs b/src/ska_ref.rs
index 6f1ff14..4d4ac50 100644
--- a/src/ska_ref.rs
+++ b/src/ska_ref.rs
@@ -55,8 +55,9 @@ use needletail::{
     parser::{write_fasta, Format},
 };
 
+use super::QualFilter;
 pub mod aln_writer;
-use crate::{cli::QualFilter, ska_ref::aln_writer::AlnWriter};
+use crate::ska_ref::aln_writer::AlnWriter;
 pub mod idx_check;
 use crate::ska_ref::idx_check::IdxCheck;
 

From ea282aae93720305eb3bb9072ad536db6ebff475 Mon Sep 17 00:00:00 2001
From: John Lees <lees.john6@gmail.com>
Date: Sat, 18 Feb 2023 17:38:02 +0000
Subject: [PATCH 4/5] Change core check to a warning

---
 src/cli.rs | 15 ++++++++++-----
 src/lib.rs |  7 +++++++
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/src/cli.rs b/src/cli.rs
index a51a9de..aaf5afd 100644
--- a/src/cli.rs
+++ b/src/cli.rs
@@ -3,8 +3,6 @@ use std::fmt;
 
 use clap::{ArgGroup, Parser, Subcommand, ValueEnum};
 
-extern crate num_cpus;
-
 use super::QualFilter;
 
 /// Default split k-mer size
@@ -47,14 +45,21 @@ fn valid_cpus(s: &str) -> Result<usize, String> {
     let threads: usize = s
         .parse()
         .map_err(|_| format!("`{s}` isn't a valid number of cores"))?;
-    let max_threads = num_cpus::get();
-    if threads < 1 || threads > max_threads {
-        Err("Threads must be between 1 and {max_threads}".to_string())
+    if threads < 1 {
+        Err("Threads must be one or higher".to_string())
     } else {
         Ok(threads)
     }
 }
 
+/// Prints a warning if more threads than available have been requested
+pub fn check_threads(threads: usize) {
+    let max_threads = num_cpus::get();
+    if threads > max_threads {
+        log::warn!("{threads} threads is greater than available cores {max_threads}");
+    }
+}
+
 /// Possible output file types
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)]
 pub enum FileType {
diff --git a/src/lib.rs b/src/lib.rs
index 8c8cfaa..57be2d6 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -298,6 +298,7 @@ use std::fmt;
 use std::time::Instant;
 
 use clap::ValueEnum;
+extern crate num_cpus;
 
 pub mod merge_ska_dict;
 pub mod ska_dict;
@@ -365,6 +366,8 @@ pub fn main() {
     let args = cli_args();
     if args.verbose {
         simple_logger::init_with_level(log::Level::Info).unwrap();
+    } else {
+        simple_logger::init_with_level(log::Level::Warn).unwrap();
     }
 
     eprintln!("SKA: Split K-mer Analysis (the alignment-free aligner)");
@@ -381,6 +384,8 @@ pub fn main() {
             qual_filter,
             threads,
         } => {
+            check_threads(*threads);
+
             // Read input
             let input_files = get_input_list(file_list, seq_files);
             let quality = QualOpts {
@@ -412,6 +417,7 @@ pub fn main() {
             filter,
             threads,
         } => {
+            check_threads(*threads);
             if let Ok(mut ska_array) = load_array::<u64>(input, *threads) {
                 // In debug mode (cannot be set from CLI, give details)
                 log::debug!("{ska_array}");
@@ -431,6 +437,7 @@ pub fn main() {
             format,
             threads,
         } => {
+            check_threads(*threads);
             log::info!("Loading skf as dictionary");
             if let Ok(mut ska_array) = load_array::<u64>(input, *threads) {
                 log::info!(

From 203d7802cd84cb2357c97cdd791180b3325d14b5 Mon Sep 17 00:00:00 2001
From: John Lees <lees.john6@gmail.com>
Date: Sat, 18 Feb 2023 17:48:58 +0000
Subject: [PATCH 5/5] Fix the doc tests

---
 src/lib.rs            |  8 +++-----
 src/merge_ska_dict.rs |  5 +++--
 src/ska_dict.rs       | 13 ++++++-------
 3 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index 57be2d6..b78ab19 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -236,7 +236,7 @@
 //! ```rust
 //! use ska::merge_ska_dict::{InputFastx, build_and_merge};
 //! use ska::merge_ska_array::MergeSkaArray;
-//! use ska::cli::QualFilter;
+//! use ska::{QualOpts, QualFilter};
 //!
 //! // Build, merge
 //! let input_files: [InputFastx; 2] = [("test1".to_string(),
@@ -247,13 +247,11 @@
 //!                                      None)];
 //! let rc = true;
 //! let k = 17;
-//! let min_count = 1;
-//! let min_qual = 0;
+//! let quality = QualOpts {min_count: 1, min_qual: 0, qual_filter: QualFilter::NoFilter};
 //! let threads = 2;
-//! let qual_filter = QualFilter::NoFilter;
 //! // NB u64 for k<=31, u128 for k<=63
 //! let merged_dict =
-//!     build_and_merge::<u64>(&input_files, k, rc, min_count, min_qual, &qual_filter, threads);
+//!     build_and_merge::<u64>(&input_files, k, rc, &quality, threads);
 //!
 //! // Save
 //! let ska_array = MergeSkaArray::new(&merged_dict);
diff --git a/src/merge_ska_dict.rs b/src/merge_ska_dict.rs
index 88835dd..19788fe 100644
--- a/src/merge_ska_dict.rs
+++ b/src/merge_ska_dict.rs
@@ -303,15 +303,16 @@ where
 /// # Examples
 /// ```
 /// use ska::merge_ska_dict::{InputFastx, build_and_merge};
-/// use ska::cli::QualFilter;
+/// use ska::{QualOpts, QualFilter};
 ///
+/// let quality = QualOpts {min_count: 1, min_qual: 0, qual_filter: QualFilter::NoFilter};
 /// let input_files: [InputFastx; 2] = [("test1".to_string(),
 ///                                      "tests/test_files_in/test_1.fa".to_string(),
 ///                                      None),
 ///                                     ("test2".to_string(),
 ///                                      "tests/test_files_in/test_2.fa".to_string(),
 ///                                      None)];
-/// let merged_dict = build_and_merge::<u64>(&input_files, 17, true, 0, 0, &QualFilter::NoFilter, 1);
+/// let merged_dict = build_and_merge::<u64>(&input_files, 17, true, &quality, 1);
 /// ```
 ///
 /// # Panics
diff --git a/src/ska_dict.rs b/src/ska_dict.rs
index 3ddc42d..b882018 100644
--- a/src/ska_dict.rs
+++ b/src/ska_dict.rs
@@ -110,28 +110,27 @@ where
     /// To build with a FASTA
     /// ```
     /// use ska::ska_dict::SkaDict;
-    /// use ska::cli::QualFilter;
+    /// use ska::{QualOpts, QualFilter};
     ///
     /// let k = 31;
     /// let sample_idx = 0;
-    /// let ska_dict = SkaDict::<u64>::new(k, sample_idx, (&"tests/test_files_in/test_1.fa", None), "test_1", true, 0, &QualFilter::NoFilter, 0);
+    /// let quality = QualOpts {min_count: 1, min_qual: 0, qual_filter: QualFilter::NoFilter};
+    /// let ska_dict = SkaDict::<u64>::new(k, sample_idx, (&"tests/test_files_in/test_1.fa", None), "test_1", true, &quality);
     /// ```
     ///
     /// With FASTQ pair, only allowing k-mers with a count over 2, and where all
     /// bases have a PHRED score of 20 or more
     /// ```
     /// use ska::ska_dict::SkaDict;
-    /// use ska::cli::QualFilter;
+    /// use ska::{QualOpts, QualFilter};
     ///
-    /// let min_count = 2;
-    /// let min_qual = 20;
-    /// let qual_filter = QualFilter::Middle;
+    /// let quality = QualOpts {min_count: 2, min_qual: 20, qual_filter: QualFilter::Middle};
     /// let k = 9;
     /// let sample_idx = 0;
     /// let ska_dict = SkaDict::<u64>::new(k, sample_idx,
     ///                             (&"tests/test_files_in/test_1_fwd.fastq.gz",
     ///                             Some(&"tests/test_files_in/test_2_fwd.fastq.gz".to_string())),
-    ///                             "sample1", true, min_count, &qual_filter, min_qual);
+    ///                             "sample1", true, &quality);
     /// ```
     ///
     /// # Panics