chore: Update tokenizers

Cargo.toml

@@ -12,7 +12,7 @@ serde_json = "1.0.127"
 serde = { version = "1.0.209", features = ["derive"] }
 tokio = { version = "1.40.0", features = ["rt", "rt-multi-thread", "macros", "signal"] }
 anyhow = "1.0.86"
-tokenizers = { version = "0.20.0", features = ["http"] }
+tokenizers = { version = "0.20.2", features = ["http"] }
 rand_distr = "0.4.3"
 rand = "0.8.5"
 futures-util = "0.3.30"

src/lib.rs

@@ -54,7 +54,7 @@ pub async fn run(run_config: RunConfiguration, stop_sender: Sender<()>) -> anyho
     sysinfo::set_open_files_limit(0);
     // initialize tokenizer
     let params = FromPretrainedParameters {
-        auth_token: run_config.hf_token.clone(),
+        token: run_config.hf_token.clone(),
         ..Default::default()
     };
     let tokenizer =

src/requests.rs

@@ -375,7 +375,7 @@ impl ConversationTextRequestGenerator {
         hf_token: Option<String>,
     ) -> anyhow::Result<Self> {
         let params = FromPretrainedParameters {
-            auth_token: hf_token,
+            token: hf_token,
             ..Default::default()
         };
         let tokenizer = match Tokenizer::from_pretrained(tokenizer, Some(params)) {