cleanup

Signed-off-by: Alexandros Koumparoulis <[email protected]>
NVIDIA · Jan 6, 2025 · a895489 · a895489
1 parent da653c0
commit a895489
Showing 1 changed file with 1 addition and 6 deletions.
diff --git a/tests/collections/nlp/test_tokenizer_with_special_tokens.py b/tests/collections/nlp/test_tokenizer_with_special_tokens.py
@@ -13,19 +13,14 @@
 # limitations under the License.
 
 
-import json
-import os
-import random
-from functools import partial
-
 import pytest
 
 from nemo.collections.nlp.modules.common.tokenizer_utils import get_nmt_tokenizer
 
 TOKENIZER_SPM_FILE = '/home/TestData/nlp/tokenizer_with_special_tokens/tokenizer.model'
 
 def test_spm_with_special_tokens() -> None:
-    special_tokens = ['[INST]', '[/INST]', '[TOOL_CALLS]', '[AVAILABLE_TOOLS]', '[/AVAILABLE_TOOLS]', '[TOOL_RESULTS]', '[/TOOL_RESULTS]']
+    special_tokens = ['<s>', '</s>', '[INST]', '[/INST]', '[TOOL_CALLS]', '[AVAILABLE_TOOLS]', '[/AVAILABLE_TOOLS]', '[TOOL_RESULTS]', '[/TOOL_RESULTS]']
     tokenizer_cfg = {
         "library": "sentencepiece",
         "type": None,