diff --git a/tokenizers/src/models/bpe/trainer.rs b/tokenizers/src/models/bpe/trainer.rs
index 977593d6a..955a5a865 100644
--- a/tokenizers/src/models/bpe/trainer.rs
+++ b/tokenizers/src/models/bpe/trainer.rs
@@ -277,7 +277,7 @@ impl BpeTrainer {
 
         // Also include anything from the provided initial alphabet
         for c in &self.initial_alphabet {
-            *alphabet.entry(*c).or_default() += usize::MAX;
+            *alphabet.entry(*c).or_default() = usize::MAX;
         }
 
         let mut kept = alphabet.iter().collect::<Vec<_>>();