Update Normalization: attempt 2

.github/workflows/linux.yml

@@ -63,10 +63,9 @@ jobs:
 
 
   openvino_tokenizers_cpack:
-    name: OpenVINO tokenizers cpack (BUILD_FAST_TOKENIZERS=${{ matrix.build_fast_tokenizers }}, BUILD_TYPE=${{ matrix.build_type }})
+    name: OpenVINO tokenizers cpack, BUILD_TYPE=${{ matrix.build_type }})
     strategy:
       matrix:
-        build_fast_tokenizers: [ON]
         build_type: [Release] # TODO: Add Debug build when OV provider is ready or use OV package
     needs: [ openvino_download ]
     if: |
@@ -110,8 +109,7 @@ jobs:
       - name: CMake configure - tokenizers
         run: |
           source ${INSTALL_DIR}/setupvars.sh
-          cmake -DBUILD_FAST_TOKENIZERS="${{ matrix.build_fast_tokenizers }}" \
-                -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+          cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
                 -S ${{ env.OPENVINO_TOKENIZERS_REPO }} \
                 -B ${{ env.BUILD_DIR }}
 
@@ -138,15 +136,13 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
-          name: openvino_tokenizers_cpack_${{ matrix.build_fast_tokenizers }}_${{ matrix.build_type }}
+          name: openvino_tokenizers_cpack_${{ matrix.build_type }}
           path: ${{ env.BUILD_DIR }}/*.tar.gz
           if-no-files-found: 'error'
 
   openvino_tokenizers_wheel:
-    name: OpenVINO tokenizers extension (BUILD_FAST_TOKENIZERS=${{ matrix.build_fast_tokenizers }})
-    strategy:
-      matrix:
-        build_fast_tokenizers: [ON, OFF]
+    name: OpenVINO tokenizers extension wheel
+
     needs: [ openvino_download ]
     if: |
       always() &&
@@ -188,7 +184,6 @@ jobs:
         run: |
           python -m pip wheel -v --no-deps --wheel-dir ${BUILD_DIR} \
               --config-settings=override=cross.arch="manylinux_2_31_x86_64" \
-              --config-settings=override=cmake.options.BUILD_FAST_TOKENIZERS="${{ matrix.build_fast_tokenizers }}" \
               ${{ needs.openvino_download.outputs.ov_wheel_source }} \
               ${OPENVINO_TOKENIZERS_REPO}
         env:
@@ -204,15 +199,12 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
-          name: openvino_tokenizers_wheel_${{ matrix.build_fast_tokenizers }}
+          name: openvino_tokenizers_wheel
           path: ${{ env.BUILD_DIR }}/*.whl
           if-no-files-found: 'error'
 
   openvino_tokenizers_tests:
-    name: OpenVINO tokenizers tests (BUILD_FAST_TOKENIZERS=${{ matrix.build_fast_tokenizers }})
-    strategy:
-      matrix:
-        build_fast_tokenizers: [ON, OFF]
+    name: OpenVINO tokenizers tests
     needs: [ openvino_download, openvino_tokenizers_wheel]
     if: always() && needs.openvino_tokenizers_wheel.result == 'success'
     timeout-minutes: 45
@@ -242,7 +234,7 @@ jobs:
       - name: Download tokenizers package
         uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
-          name: openvino_tokenizers_wheel_${{ matrix.build_fast_tokenizers }}
+          name: openvino_tokenizers_wheel
           path: ${{ env.INSTALL_DIR }}/ov_tokenizers
 
       - name: Download OpenVINO package

.github/workflows/mac.yml

@@ -47,10 +47,9 @@ jobs:
         revision: latest_nightly
 
   openvino_tokenizers_cpack:
-    name: OpenVINO tokenizers cpack (BUILD_FAST_TOKENIZERS=${{ matrix.build_fast_tokenizers }}, BUILD_TYPE=${{ matrix.build_type }})
+    name: OpenVINO tokenizers cpack (BUILD_TYPE=${{ matrix.build_type }})
     strategy:
       matrix:
-        build_fast_tokenizers: [ON]
         build_type: [Release] # TODO: Add Debug build when OV provider is ready or use OV package
     needs: [ openvino_download ]
     timeout-minutes: 45
@@ -89,8 +88,7 @@ jobs:
       - name: CMake configure - tokenizers
         run: |
           source ${INSTALL_DIR}/setupvars.sh
-          cmake -DBUILD_FAST_TOKENIZERS="${{ matrix.build_fast_tokenizers }}" \
-                -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+          cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
                 -S ${{ env.OPENVINO_TOKENIZERS_REPO }} \
                 -B ${{ env.BUILD_DIR }}
 
@@ -115,7 +113,7 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
-          name: openvino_tokenizers_cpack_${{ matrix.build_fast_tokenizers }}_${{ matrix.build_type }}
+          name: openvino_tokenizers_cpack_${{ matrix.build_type }}
           path: ${{ env.BUILD_DIR }}/*.tar.gz
           if-no-files-found: 'error'
 

.github/workflows/windows.yml

@@ -47,10 +47,9 @@ jobs:
         revision: 'latest_available_commit'
 
   openvino_tokenizers_cpack:
-    name: OpenVINO tokenizers cpack (BUILD_FAST_TOKENIZERS=${{ matrix.build_fast_tokenizers }}, BUILD_TYPE=${{ matrix.build_type }})
+    name: OpenVINO tokenizers cpack (BUILD_TYPE=${{ matrix.build_type }})
     strategy:
       matrix:
-        build_fast_tokenizers: [ON]
         build_type: [Release] # TODO: Add Debug build when OV provider is ready or use OV package
     needs: [ openvino_download ]
     if: |
@@ -115,8 +114,7 @@ jobs:
         shell: pwsh
         run: |
           ${{ env.OV_INSTALL_DIR }}/setupvars.ps1
-          cmake -DBUILD_FAST_TOKENIZERS="${{ matrix.build_fast_tokenizers }}" `
-                -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} `
+          cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} `
                 -S ${{ env.OPENVINO_TOKENIZERS_REPO }} `
                 -B ${{ env.BUILD_DIR }}
         env:
@@ -149,7 +147,7 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
-          name: openvino_tokenizers_cpack_${{ matrix.build_fast_tokenizers }}_${{ matrix.build_type }}
+          name: openvino_tokenizers_cpack_${{ matrix.build_type }}
           path: ${{ env.BUILD_DIR }}/*.zip
           if-no-files-found: 'error'
 

README.md

@@ -150,77 +150,6 @@ make
 
 After that, you can transfer all binaries from `build/src` to `<openvino_dir>` as described in the C++ installation instruction above.
 
-### Reducing the ICU Data Size
-
-By default, all available ICU locales are supported, which significantly increases the package size. To reduce the size of the ICU libraries included in your final package, follow these steps:
-
-1. **Use the ICU Data Configuration File**:
-    - This file specifies which features and locales to include in a custom data bundle. You can find more information [here](https://unicode-org.github.io/icu/userguide/icu_data/buildtool.html#icu-data-configuration-file).
-
-2. **Set the ICU Data Filter File as an Environment Variable**:
-    - **On Unix-like systems (Linux, macOS)**:
-      Set the `ICU_DATA_FILTER_FILE` environment variable to the path of your configuration file (`filters.json`):
-
-      ```bash
-      export ICU_DATA_FILTER_FILE="filters.json"
-      ```
-
-    - **On Windows**:
-      Set the `ICU_DATA_FILTER_FILE` environment variable using the Command Prompt or PowerShell:
-
-      **Command Prompt:**
-      ```cmd
-      set ICU_DATA_FILTER_FILE=filters.json
-      ```
-
-      **PowerShell:**
-      ```powershell
-      $env:ICU_DATA_FILTER_FILE="filters.json"
-      ```
-
-3. **Create a Configuration File**:
-    - An example configuration file (`filters.json`) might look like this:
-
-    ```json
-    {
-      "localeFilter": {
-        "filterType": "language",
-        "includelist": [
-          "en"
-        ]
-      }
-    }
-    ```
-
-4. **Configure OpenVINO Tokenizers**:
-    - When building OpenVINO tokenizers, set the following CMake option during the project configuration:
-
-    ```bash
-    -DBUILD_FAST_TOKENIZERS=ON
-    ```
-   - Example for a pip installation path:
-   ```bash
-   ICU_DATA_FILTER_FILE=</path/to/filters.json> pip install git+https://github.com/openvinotoolkit/openvino_tokenizers.git --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly --config-settings=override=cmake.options.BUILD_FAST_TOKENIZERS=ON
-   ```
-
-By following these instructions, you can effectively reduce the size of the ICU libraries in your final package.
-
-### Build OpenVINO Tokenizers without FastTokenizer Library
-
-If a tokenizer doesn't use `CaseFold`, `UnicodeNormalization` or `Wordpiece` operations, you can drastically reduce package binary size by building OpenVINO Tokenizers without FastTokenizer dependency with this flag:
-
-```bash
--DENABLE_FAST_TOKENIZERS=OFF
-```
-
-This option can also help with building for platform that is supported by FastTokenizer, for example `Android x86_64`.
-
-Example for a pip installation path:
-```bash
-
-pip install git+https://github.com/openvinotoolkit/openvino_tokenizers.git --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly --config-settings=override=cmake.options.ENABLE_FAST_TOKENIZERS=OFF
-```
-
 ## Usage
 
 :warning: OpenVINO Tokenizers can be inferred on a `CPU` device only.