Wordpiece Detokenizer Support

README.md

@@ -487,13 +487,14 @@ int main(int argc, char* argv[]) {
 ## Supported Tokenizer Types
 
 | Huggingface <br/>Tokenizer Type | Tokenizer Model Type | Tokenizer | Detokenizer |
-|---------------------------------|----------------------|----------|------------|
-| Fast                            | WordPiece            | ✅        | ❌          |
-|                                 | BPE                  | ✅        | ✅          |
-|                                 | Unigram              | ❌         | ❌          |
-| Legacy                          | SentencePiece .model | ✅        | ✅          |
-| Custom                          | tiktoken             | ✅        | ✅          |
-| RWKV                            | Trie                 | ✅        | ✅          |
+|---------------------------------|----------------------|----------|-----------|
+| Fast                            | WordPiece            | ✅        | ✅          |
+|                                 | BPE                  | ✅        | ✅         |
+|                                 | Unigram              | ❌         | ❌         |
+|                                 | WordLevel*           | ✅         | ✅         |
+| Legacy                          | SentencePiece .model | ✅        | ✅         |
+| Custom                          | tiktoken             | ✅        | ✅         |
+| RWKV                            | Trie                 | ✅        | ✅         |
 
 ## Test Results
 
@@ -532,8 +533,8 @@ This report is autogenerated and includes tokenizers and detokenizers tests. The
     </tr>
     <tr>
       <td >WordPiece</td>
-      <td >98.39</td>
-      <td >747</td>
+      <td >99.34</td>
+      <td >1811</td>
     </tr>
   </tbody>
 </table>
@@ -842,43 +843,43 @@ This report is autogenerated and includes tokenizers and detokenizers tests. The
       <td >WordPiece</td>
       <td >ProsusAI/finbert</td>
       <td >100.00</td>
-      <td >109</td>
+      <td >261</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >bert-base-multilingual-cased</td>
       <td >100.00</td>
-      <td >109</td>
+      <td >261</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >cointegrated/rubert-tiny2</td>
       <td >100.00</td>
-      <td >109</td>
+      <td >261</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >distilbert-base-uncased-finetuned-sst-2-english</td>
       <td >100.00</td>
-      <td >109</td>
+      <td >261</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >google/mobilebert-uncased</td>
       <td >100.00</td>
-      <td >93</td>
+      <td >245</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >rasa/LaBSE</td>
-      <td >88.99</td>
-      <td >109</td>
+      <td >95.40</td>
+      <td >261</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >sentence-transformers/all-MiniLM-L6-v2</td>
       <td >100.00</td>
-      <td >109</td>
+      <td >261</td>
     </tr>
   </tbody>
 </table>

python/openvino_tokenizers/hf_parser.py

@@ -383,50 +383,36 @@ def add_padding(self, use_max_padding: bool = False) -> None:
         str,
         Callable[[Dict[str, Any]], Union[DecodingStep, List[DecodingStep]]],
     ] = {
-        "Replace": lambda decode_dict: RegexDecodingStep.parse_replace_dict(decode_dict),
+        "Replace": RegexDecodingStep.parse_replace_dict,
         "Fuse": lambda decode_dict: FuseStep(),
-        "Strip": lambda decode_dict: RegexDecodingStep.parse_strip_dict(decode_dict),
+        "Strip": RegexDecodingStep.parse_strip_dict,
         "ByteFallback": lambda decode_dict: ByteFallbackStep(),
     }
 
     def decoding(self) -> None:
         skip_tokens = parse_special_tokens(self.original_tokenizer)
+        self.pipeline.add_steps(VocabDecoderStep.from_hf_json(self.tokenizer_json, self.pipeline.vocab, list(skip_tokens), do_skip_tokens=self.skip_special_tokens))
 
-        if self.tokenizer_json["model"]["type"] == "WordLevel":
-            self.pipeline.add_steps(
-                [
-                    VocabDecoderStep(
-                        vocab=[f" {token}" for token in self.pipeline.vocab],
-                        skip_tokens=list(skip_tokens),
-                        do_skip_tokens=self.skip_special_tokens,
-                    ),
-                    FuseStep(),
-                    RegexDecodingStep.strip_forward_space(),
-                ]
-            )
-            if self.clean_up_tokenization_spaces:
-                self.pipeline.add_steps(RegexDecodingStep.clean_up_tokenization_spaces())
-            return
-        elif self.tokenizer_json["decoder"] is None or self.tokenizer_json["model"]["type"] == "WordPiece":
-            return
-
-        self.pipeline.add_steps(
-            VocabDecoderStep(skip_tokens=list(skip_tokens), do_skip_tokens=self.skip_special_tokens)
-        )
-
-        if self.tokenizer_json["decoder"]["type"] == "Sequence":
+        has_decoder = self.tokenizer_json.get("decoder") is not None
+        if has_decoder and self.tokenizer_json["decoder"]["type"] == "Sequence":
             for decoder_dict in self.tokenizer_json["decoder"]["decoders"]:
                 decoder_parser = self.decoding_map.get(decoder_dict.get("type"))
                 if decoder_parser is None:
                     pass
                     # raise ValueError(f"Decoder {decoder_dict} is not supported yet.")
                 else:
                     self.pipeline.add_steps(decoder_parser(decoder_dict))
-        elif self.tokenizer_json["decoder"]["type"] == "ByteLevel":
+        elif has_decoder and self.tokenizer_json["decoder"]["type"] == "ByteLevel":
             self.pipeline.add_steps(CharsToBytesStep())
         else:
             self.pipeline.add_steps(FuseStep())
 
+        # strip forward space because VocabDecoderStep.from_hf_json modifies vocabulary
+        if self.tokenizer_json["model"]["type"] == "WordLevel":
+            self.pipeline.add_steps(RegexDecodingStep.strip_forward_space())
+        elif self.tokenizer_json["model"]["type"] == "WordPiece":
+            self.pipeline.add_steps(RegexDecodingStep.strip_forward_space())
+
         if self.utf8_replace_mode is not None and (self.utf8_replace_mode != UTF8ReplaceMode.DISABLE):
             self.pipeline.add_steps(UTF8ValidateStep(mode=self.utf8_replace_mode))
 

python/openvino_tokenizers/tokenizer_pipeline.py

@@ -1034,6 +1034,22 @@ def finalize(self) -> None:
         elif self.skip_tokens is None:
             self.skip_tokens = pipeline.skip_tokens or []
 
+    @classmethod
+    def from_hf_json(cls, tokenizer_json: Dict[str, Any], pipeline_vocab: Optional[List[str]], skip_tokens: Optional[List[int]] = None, do_skip_tokens: bool = True) -> "VocabDecoderStep":
+        model_type = tokenizer_json["model"]["type"]
+
+        if pipeline_vocab is not None and model_type == "WordLevel":
+            vocab = [f" {token}" for token in pipeline_vocab]
+        elif pipeline_vocab is not None and model_type == "WordPiece":
+            vocab = [
+                token if token in ".,!?" else token[2:] if token.startswith("##") else f" {token}"
+                for token in pipeline_vocab
+            ]
+        else:  # Use vocab node from pipeline
+            vocab = None
+
+        return cls(vocab, skip_tokens, do_skip_tokens)
+
     def get_vocab_node_outputs(self) -> Optional[List[Output]]:
         return self.get_pipeline().vocab_node_outputs if self.get_pipeline() is not None else None
 

tests/pass_rates.json

@@ -1,3 +1,3 @@
 {
-    "tests/tokenizers_test.py::test_": 0.9306500079076387
+    "tests/tokenizers_test.py::test_": 0.9360320933625091
 }