Merge branch 'master' into icu_build_sources_fecth_content

CMakeLists.txt

@@ -38,7 +38,7 @@ else()
 endif()
 
 project(openvino_tokenizers
-        VERSION 2025.0.0.0
+        VERSION 2025.1.0.0
         DESCRIPTION "OpenVINO Tokenizers"
         HOMEPAGE_URL "https://github.com/openvinotoolkit/openvino_tokenizers"
         LANGUAGES CXX)

benchmark/benchmark.py

@@ -12,7 +12,7 @@
 import pandas as pd
 import seaborn as sns
 from openvino import AsyncInferQueue, CompiledModel, InferRequest
-from openvino.runtime import ProfilingInfo, properties
+from openvino import ProfilingInfo, properties
 from openvino_tokenizers import convert_tokenizer
 from tqdm.auto import tqdm
 from transformers import AutoTokenizer, PreTrainedTokenizerBase

js/openvino-tokenizers.js

@@ -19,19 +19,19 @@ function getPathToBinary(osProps) {
     __dirname,
     'bin/runtime',
     libOrBin(platform),
-    getDirnameByArch(arch),
+    getDirnameByArchAndPlatform(arch, platform),
     platform === 'linux' ? '' : 'Release',
     getBinaryFilename(platform),
   );
 }
 
-function getDirnameByArch(arch) {
+function getDirnameByArchAndPlatform(arch, platform) {
   switch (arch) {
     case 'x64':
       return 'intel64';
     case 'arm64':
     case 'armhf':
-      return 'arm64';
+      return platform === 'darwin' ? 'arm64' : 'aarch64';
     default:
       throw new Error(`Unsupported architecture: ${arch}`);
   }

js/tests/openvino-tokenizers.test.js

@@ -51,7 +51,7 @@ describe('getPathToBinary for arm64', () => {
 
     assert.equal(
       relatedFromBin,
-      '/bin/runtime/lib/arm64/libopenvino_tokenizers.so',
+      '/bin/runtime/lib/aarch64/libopenvino_tokenizers.so',
     );
   });
 
@@ -80,7 +80,7 @@ describe('getPathToBinary for armhf', () => {
 
     assert.equal(
       relatedFromBin,
-      '/bin/runtime/lib/arm64/libopenvino_tokenizers.so',
+      '/bin/runtime/lib/aarch64/libopenvino_tokenizers.so',
     );
   });
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "openvino-tokenizers"
-version = "2025.0.0.0"
+version = "2025.1.0.0"
 description = "Convert tokenizers into OpenVINO models"
 requires-python = ">=3.9"
 readme = { file = "README.md", content-type="text/markdown" }
@@ -33,7 +33,7 @@ classifiers = [
 
 dependencies = [
     # support of nightly openvino packages with dev suffix
-    "openvino~=2025.0.0.dev"
+    "openvino~=2025.1.0.dev"
 ]
 
 [project.optional-dependencies]
@@ -117,6 +117,6 @@ python_abi = "none"
 requires = [
     "py-build-cmake==0.3.4",
     "cmake~=3.14",
-    "openvino~=2025.0.0.dev"
+    "openvino~=2025.1.0.dev"
 ]
 build-backend = "py_build_cmake.build"

python/openvino_tokenizers/__init__.py

@@ -10,7 +10,7 @@
 from typing import Callable, Optional
 
 import openvino
-from openvino.runtime.utils.node_factory import NodeFactory
+from openvino.utils.node_factory import NodeFactory
 
 
 _ext_name = "openvino_tokenizers"
@@ -42,8 +42,8 @@
 del _ext_name
 
 # patching openvino
-old_core_init = openvino.runtime.Core.__init__
-old_factory_init = openvino.runtime.utils.node_factory.NodeFactory.__init__
+old_core_init = openvino.Core.__init__
+old_factory_init = openvino.utils.node_factory.NodeFactory.__init__
 old_fe_init = openvino.frontend.frontend.FrontEnd.__init__
 
 
@@ -65,8 +65,8 @@ def new_fe_init(self, *args, **kwargs):
     self.add_extension(str(_ext_path))
 
 
-openvino.runtime.Core.__init__ = new_core_init
-openvino.runtime.utils.node_factory.NodeFactory.__init__ = new_factory_init
+openvino.Core.__init__ = new_core_init
+openvino.utils.node_factory.NodeFactory.__init__ = new_factory_init
 openvino.frontend.frontend.FrontEnd.__init__ = new_fe_init
 
 

python/openvino_tokenizers/build_tokenizer.py

@@ -1,9 +1,9 @@
 from typing import Iterable, Tuple
 
 from openvino import Model, PartialShape, Type
-from openvino.runtime import op
-from openvino.runtime import opset12 as opset
-from openvino.runtime.utils.types import make_constant_node
+from openvino import op
+from openvino import opset12 as opset
+from openvino.utils.types import make_constant_node
 
 from openvino_tokenizers.constants import DETOKENIZER_NAME, STRING_OUTPUT_NAME, TOKEN_IDS_INPUT_NAME, TOKENIZER_NAME
 from openvino_tokenizers.tokenizer_pipeline import (

python/openvino_tokenizers/convert_tokenizer.py

@@ -8,8 +8,8 @@
 from functools import wraps
 from typing import Any, Optional, Tuple, Union
 
-from openvino.runtime import Model, Type
-from openvino.runtime.exceptions import OVTypeError
+from openvino import Model, Type
+from openvino.exceptions import OVTypeError
 
 from openvino_tokenizers.constants import UTF8ReplaceMode
 from openvino_tokenizers.utils import (

python/openvino_tokenizers/hf_parser.py

@@ -12,11 +12,11 @@
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import numpy as np
-import openvino.runtime.opset14 as opset
+import openvino.opset14 as opset
 from openvino import Model, PartialShape, Type
-from openvino.runtime import Node, op
-from openvino.runtime.exceptions import OVTypeError
-from openvino.runtime.utils.types import as_node, make_constant_node
+from openvino import Node, op
+from openvino.exceptions import OVTypeError
+from openvino.utils.types import as_node, make_constant_node
 from transformers import PreTrainedTokenizerBase, PreTrainedTokenizerFast
 from transformers.convert_slow_tokenizer import import_protobuf
 
@@ -391,7 +391,11 @@ def add_padding(self, use_max_padding: bool = False) -> None:
 
     def decoding(self) -> None:
         skip_tokens = parse_special_tokens(self.original_tokenizer)
-        self.pipeline.add_steps(VocabDecoderStep.from_hf_json(self.tokenizer_json, self.pipeline.vocab, list(skip_tokens), do_skip_tokens=self.skip_special_tokens))
+        self.pipeline.add_steps(
+            VocabDecoderStep.from_hf_json(
+                self.tokenizer_json, self.pipeline.vocab, list(skip_tokens), do_skip_tokens=self.skip_special_tokens
+            )
+        )
 
         has_decoder = self.tokenizer_json.get("decoder") is not None
         if has_decoder and self.tokenizer_json["decoder"]["type"] == "Sequence":

python/openvino_tokenizers/tokenizer_pipeline.py

@@ -14,10 +14,10 @@
 from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
 
 import numpy as np
-from openvino.runtime import Model, Output, PartialShape, Shape, Type, op
-from openvino.runtime import opset12 as opset
-from openvino.runtime.exceptions import OVTypeError, UserInputError
-from openvino.runtime.utils.types import as_node, make_constant_node
+from openvino import Model, Output, PartialShape, Shape, Type, op
+from openvino import opset12 as opset
+from openvino.exceptions import OVTypeError, UserInputError
+from openvino.utils.types import as_node, make_constant_node
 
 from . import _get_factory
 from .constants import (

python/openvino_tokenizers/utils.py

@@ -10,7 +10,7 @@
 
 from openvino import Model, Type
 from openvino.preprocess import PrePostProcessor
-from openvino.runtime import opset12 as opset
+from openvino import opset12 as opset
 
 from .constants import (
     LOGITS_OUTPUT_NAME,

tests/layer_tests.py

@@ -8,7 +8,7 @@
 import pytest
 import requests
 from openvino import Model, PartialShape, Type
-from openvino.runtime import op
+from openvino import op
 from openvino_tokenizers import _get_factory
 from openvino_tokenizers.constants import UTF8ReplaceMode
 from openvino_tokenizers.tokenizer_pipeline import (