Custom ICU build

.github/workflows/linux.yml

@@ -63,10 +63,9 @@ jobs:
 
 
   openvino_tokenizers_cpack:
-    name: OpenVINO tokenizers cpack (BUILD_FAST_TOKENIZERS=${{ matrix.build_fast_tokenizers }}, BUILD_TYPE=${{ matrix.build_type }})
+    name: OpenVINO tokenizers cpack, BUILD_TYPE=${{ matrix.build_type }})
     strategy:
       matrix:
-        build_fast_tokenizers: [ON]
         build_type: [Release] # TODO: Add Debug build when OV provider is ready or use OV package
     needs: [ openvino_download ]
     if: |
@@ -110,8 +109,7 @@ jobs:
       - name: CMake configure - tokenizers
         run: |
           source ${INSTALL_DIR}/setupvars.sh
-          cmake -DBUILD_FAST_TOKENIZERS="${{ matrix.build_fast_tokenizers }}" \
-                -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+          cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
                 -S ${{ env.OPENVINO_TOKENIZERS_REPO }} \
                 -B ${{ env.BUILD_DIR }}
 
@@ -138,15 +136,13 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
-          name: openvino_tokenizers_cpack_${{ matrix.build_fast_tokenizers }}_${{ matrix.build_type }}
+          name: openvino_tokenizers_cpack_${{ matrix.build_type }}
           path: ${{ env.BUILD_DIR }}/*.tar.gz
           if-no-files-found: 'error'
 
   openvino_tokenizers_wheel:
-    name: OpenVINO tokenizers extension (BUILD_FAST_TOKENIZERS=${{ matrix.build_fast_tokenizers }})
-    strategy:
-      matrix:
-        build_fast_tokenizers: [ON, OFF]
+    name: OpenVINO tokenizers extension wheel
+
     needs: [ openvino_download ]
     if: |
       always() &&
@@ -188,7 +184,6 @@ jobs:
         run: |
           python -m pip wheel -v --no-deps --wheel-dir ${BUILD_DIR} \
               --config-settings=override=cross.arch="manylinux_2_31_x86_64" \
-              --config-settings=override=cmake.options.BUILD_FAST_TOKENIZERS="${{ matrix.build_fast_tokenizers }}" \
               ${{ needs.openvino_download.outputs.ov_wheel_source }} \
               ${OPENVINO_TOKENIZERS_REPO}
         env:
@@ -204,15 +199,12 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
-          name: openvino_tokenizers_wheel_${{ matrix.build_fast_tokenizers }}
+          name: openvino_tokenizers_wheel
           path: ${{ env.BUILD_DIR }}/*.whl
           if-no-files-found: 'error'
 
   openvino_tokenizers_tests:
-    name: OpenVINO tokenizers tests (BUILD_FAST_TOKENIZERS=${{ matrix.build_fast_tokenizers }})
-    strategy:
-      matrix:
-        build_fast_tokenizers: [ON, OFF]
+    name: OpenVINO tokenizers tests
     needs: [ openvino_download, openvino_tokenizers_wheel]
     if: always() && needs.openvino_tokenizers_wheel.result == 'success'
     timeout-minutes: 45
@@ -242,7 +234,7 @@ jobs:
       - name: Download tokenizers package
         uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
-          name: openvino_tokenizers_wheel_${{ matrix.build_fast_tokenizers }}
+          name: openvino_tokenizers_wheel
           path: ${{ env.INSTALL_DIR }}/ov_tokenizers
 
       - name: Download OpenVINO package

.github/workflows/mac.yml

@@ -176,10 +176,9 @@ jobs:
           if-no-files-found: 'error'
 
   openvino_tokenizers_cpack:
-    name: OpenVINO tokenizers cpack (BUILD_FAST_TOKENIZERS=${{ matrix.build_fast_tokenizers }}, BUILD_TYPE=${{ matrix.build_type }})
+    name: OpenVINO tokenizers cpack (BUILD_TYPE=${{ matrix.build_type }})
     strategy:
       matrix:
-        build_fast_tokenizers: [ON]
         build_type: [Release] # TODO: Add Debug build when OV provider is ready or use OV package
     needs: [ openvino_download, openvino_build ]
     if: |
@@ -221,13 +220,12 @@ jobs:
       # Build
       #
       - name: Install build dependencies
-        run: brew install coreutils ninja
+        run: brew install coreutils ninja icu4c
 
       - name: CMake configure - tokenizers
         run: |
           source ${INSTALL_DIR}/setupvars.sh
-          cmake -DBUILD_FAST_TOKENIZERS="${{ matrix.build_fast_tokenizers }}" \
-                -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+          cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
                 -S ${{ env.OPENVINO_TOKENIZERS_REPO }} \
                 -B ${{ env.BUILD_DIR }}
 
@@ -254,7 +252,7 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
-          name: openvino_tokenizers_cpack_${{ matrix.build_fast_tokenizers }}_${{ matrix.build_type }}
+          name: openvino_tokenizers_cpack_${{ matrix.build_type }}
           path: ${{ env.BUILD_DIR }}/*.tar.gz
           if-no-files-found: 'error'
 
@@ -314,7 +312,7 @@ jobs:
       #
 
       - name: Install build dependencies
-        run: brew install coreutils ninja
+        run: brew install coreutils ninja icu4c
 
       #
       # Build

.github/workflows/windows.yml

@@ -47,10 +47,9 @@ jobs:
         revision: 'latest_available_commit'
 
   openvino_tokenizers_cpack:
-    name: OpenVINO tokenizers cpack (BUILD_FAST_TOKENIZERS=${{ matrix.build_fast_tokenizers }}, BUILD_TYPE=${{ matrix.build_type }})
+    name: OpenVINO tokenizers cpack (BUILD_TYPE=${{ matrix.build_type }})
     strategy:
       matrix:
-        build_fast_tokenizers: [ON]
         build_type: [Release] # TODO: Add Debug build when OV provider is ready or use OV package
     needs: [ openvino_download ]
     if: |
@@ -115,8 +114,7 @@ jobs:
         shell: pwsh
         run: |
           ${{ env.OV_INSTALL_DIR }}/setupvars.ps1
-          cmake -DBUILD_FAST_TOKENIZERS="${{ matrix.build_fast_tokenizers }}" `
-                -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} `
+          cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} `
                 -S ${{ env.OPENVINO_TOKENIZERS_REPO }} `
                 -B ${{ env.BUILD_DIR }}
         env:
@@ -149,7 +147,7 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
-          name: openvino_tokenizers_cpack_${{ matrix.build_fast_tokenizers }}_${{ matrix.build_type }}
+          name: openvino_tokenizers_cpack_${{ matrix.build_type }}
           path: ${{ env.BUILD_DIR }}/*.zip
           if-no-files-found: 'error'
 

README.md

@@ -150,77 +150,6 @@ make
 
 After that, you can transfer all binaries from `build/src` to `<openvino_dir>` as described in the C++ installation instruction above.
 
-### Reducing the ICU Data Size
-
-By default, all available ICU locales are supported, which significantly increases the package size. To reduce the size of the ICU libraries included in your final package, follow these steps:
-
-1. **Use the ICU Data Configuration File**:
-    - This file specifies which features and locales to include in a custom data bundle. You can find more information [here](https://unicode-org.github.io/icu/userguide/icu_data/buildtool.html#icu-data-configuration-file).
-
-2. **Set the ICU Data Filter File as an Environment Variable**:
-    - **On Unix-like systems (Linux, macOS)**:
-      Set the `ICU_DATA_FILTER_FILE` environment variable to the path of your configuration file (`filters.json`):
-
-      ```bash
-      export ICU_DATA_FILTER_FILE="filters.json"
-      ```
-
-    - **On Windows**:
-      Set the `ICU_DATA_FILTER_FILE` environment variable using the Command Prompt or PowerShell:
-
-      **Command Prompt:**
-      ```cmd
-      set ICU_DATA_FILTER_FILE=filters.json
-      ```
-
-      **PowerShell:**
-      ```powershell
-      $env:ICU_DATA_FILTER_FILE="filters.json"
-      ```
-
-3. **Create a Configuration File**:
-    - An example configuration file (`filters.json`) might look like this:
-
-    ```json
-    {
-      "localeFilter": {
-        "filterType": "language",
-        "includelist": [
-          "en"
-        ]
-      }
-    }
-    ```
-
-4. **Configure OpenVINO Tokenizers**:
-    - When building OpenVINO tokenizers, set the following CMake option during the project configuration:
-
-    ```bash
-    -DBUILD_FAST_TOKENIZERS=ON
-    ```
-   - Example for a pip installation path:
-   ```bash
-   ICU_DATA_FILTER_FILE=</path/to/filters.json> pip install git+https://github.com/openvinotoolkit/openvino_tokenizers.git --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly --config-settings=override=cmake.options.BUILD_FAST_TOKENIZERS=ON
-   ```
-
-By following these instructions, you can effectively reduce the size of the ICU libraries in your final package.
-
-### Build OpenVINO Tokenizers without FastTokenizer Library
-
-If a tokenizer doesn't use `CaseFold`, `UnicodeNormalization` or `Wordpiece` operations, you can drastically reduce package binary size by building OpenVINO Tokenizers without FastTokenizer dependency with this flag:
-
-```bash
--DENABLE_FAST_TOKENIZERS=OFF
-```
-
-This option can also help with building for platform that is supported by FastTokenizer, for example `Android x86_64`.
-
-Example for a pip installation path:
-```bash
-
-pip install git+https://github.com/openvinotoolkit/openvino_tokenizers.git --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly --config-settings=override=cmake.options.ENABLE_FAST_TOKENIZERS=OFF
-```
-
 ## Usage
 
 :warning: OpenVINO Tokenizers can be inferred on a `CPU` device only.

pyproject.toml

@@ -41,17 +41,14 @@ transformers = [
     "transformers[sentencepiece] >= 4.36.0",
     "tiktoken"
 ]
-# chatglm2 custom tokenizer file imports torch, have to add torch dependency for tests
-torch = [
-    'torch'
-]
 dev = [
     "ruff",
     "bandit",
     "pytest",
     "pytest_harvest",
     "pandas",
-    "openvino_tokenizers[transformers, torch]"
+    "jinja2",
+    "openvino_tokenizers[transformers]"
 ]
 benchmark = [
     "pandas",

python/openvino_tokenizers/tokenizer_pipeline.py

@@ -158,13 +158,23 @@ class NormalizationStep(BasePipelineStep):
 class NormalizeUnicode(NormalizationStep):
     normalization_form: str = "NFD"
 
+    def __post_init__(self):
+        if self.normalization_form not in ["NFD", "NFC", "NFKD", "NFKC"]:
+            raise ValueError(
+                '[ NormalizeUnicode ] `normalization_form` attribute must be one of ["NFD", "NFC", "NFKD", "NFKC"], '
+                f"got {self.normalization_form}."
+            )
+
     def get_ov_subgraph(self, input_nodes: List[Output]) -> List[Output]:
         return (
             _get_factory()
             .create(
-                "NormalizeUnicode",
+                "CharsMapNormalization",
                 input_nodes,
-                {"normalization_form": self.normalization_form},
+                {
+                    "normalization_form": self.normalization_form.lower(),
+                    "remove_extra_whitespaces": False,
+                },
             )
             .outputs()
         )
@@ -182,7 +192,22 @@ def __post_init__(self):
             )
 
     def get_ov_subgraph(self, input_nodes: List[Output]) -> List[Output]:
-        return _get_factory().create("CaseFold", input_nodes, {"encoding": self.encoding}).outputs()
+        if self.encoding == "":
+            return _get_factory().create("CaseFold", input_nodes, {"encoding": self.encoding}).outputs()
+        else:
+            return (
+                _get_factory()
+                .create(
+                    "CharsMapNormalization",
+                    input_nodes,
+                    {
+                        "normalization_form": "identity",
+                        "case_fold": True,
+                        "remove_extra_whitespaces": False,
+                    },
+                )
+                .outputs()
+            )
 
 
 @dataclass
@@ -245,15 +270,40 @@ def get_ov_subgraph(self, input_nodes: List[Output]) -> List[Output]:
 
 @dataclass
 class CharsmapStep(NormalizationStep):
-    charsmap: bytes
+    charsmap: Optional[bytes] = None
+    normalization_form: Optional[str] = None
+    add_dummy_prefix: bool = False
+    remove_extra_whitespaces: bool = True
+    escape_whitespaces: bool = False
+    case_fold: bool = False
+    nmt: bool = False
+
+    def __post_init__(self):
+        if self.charsmap is None and self.normalization_form is None:
+            raise ValueError("[ CharsmapStep ] `charsmap` or `normalization_form` attribute must be set")
 
     @classmethod
     def from_hf_step_json(cls, step_json: Dict[str, Any]) -> "CharsmapStep":
         return cls(charsmap=base64.b64decode(step_json["precompiled_charsmap"]))
 
     def get_ov_subgraph(self, input_nodes: List[Output]) -> List[Output]:
         input_nodes += make_constant_node(np.frombuffer(self.charsmap, dtype=np.uint8), dtype=Type.u8).outputs()
-        return _get_factory().create("CharsMapNormalization", input_nodes).outputs()
+        return (
+            _get_factory()
+            .create(
+                "CharsMapNormalization",
+                input_nodes,
+                {
+                    "normalization_form": self.normalization_form or "",
+                    "add_dummy_prefix": self.add_dummy_prefix,
+                    "remove_extra_whitespaces": self.remove_extra_whitespaces,
+                    "escape_whitespaces": self.escape_whitespaces,
+                    "case_fold": self.case_fold,
+                    "nmt": self.nmt,
+                },
+            )
+            .outputs()
+        )
 
 
 @dataclass