From a040b034f98ad35e1504ec5ed3cce0112a652463 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 22 Nov 2023 23:16:06 +0100 Subject: [PATCH] win build --- .../{user_ie_extensions => }/pyproject.toml | 0 .../{user_ie_extensions => }/setup.py | 4 ++-- .../user_ie_extensions/CMakeLists.txt | 5 ++++- .../src/tokenizer/CMakeLists.txt | 17 ++++++++++------- .../tokenizer/python/ov_tokenizer/__init__.py | 3 ++- 5 files changed, 18 insertions(+), 11 deletions(-) rename modules/custom_operations/{user_ie_extensions => }/pyproject.toml (100%) rename modules/custom_operations/{user_ie_extensions => }/setup.py (77%) diff --git a/modules/custom_operations/user_ie_extensions/pyproject.toml b/modules/custom_operations/pyproject.toml similarity index 100% rename from modules/custom_operations/user_ie_extensions/pyproject.toml rename to modules/custom_operations/pyproject.toml diff --git a/modules/custom_operations/user_ie_extensions/setup.py b/modules/custom_operations/setup.py similarity index 77% rename from modules/custom_operations/user_ie_extensions/setup.py rename to modules/custom_operations/setup.py index e21163dfb..a8cd8d6fe 100644 --- a/modules/custom_operations/user_ie_extensions/setup.py +++ b/modules/custom_operations/setup.py @@ -3,8 +3,8 @@ setup( packages=["ov_tokenizer"], - package_dir={"": "src/tokenizer/python"}, - cmake_install_dir="src/tokenizer/python/ov_tokenizer/libs", + package_dir={"": "user_ie_extensions/src/tokenizer/python"}, + cmake_install_dir="user_ie_extensions/src/tokenizer/python/ov_tokenizer/libs", cmake_args=['-DCUSTOM_OPERATIONS:STRING=tokenizer', '-DBUILD_FAST_TOKENIZERS=OFF'] ) diff --git a/modules/custom_operations/user_ie_extensions/CMakeLists.txt b/modules/custom_operations/user_ie_extensions/CMakeLists.txt index 2fabfa37d..f1f2cbbfa 100644 --- a/modules/custom_operations/user_ie_extensions/CMakeLists.txt +++ b/modules/custom_operations/user_ie_extensions/CMakeLists.txt @@ -28,6 +28,10 @@ set(OP_REQ_TBB "complex_mul" "fft") set(SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src") set(HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include") +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) + # # Select specific operations # @@ -101,7 +105,6 @@ endif() # Left sentence_piece for backward compatibility if("tokenizer" IN_LIST CUSTOM_OPERATIONS) add_subdirectory(${SOURCES}/tokenizer) - install(FILES ${FAST_TOKENIZER_LIBS} DESTINATION .) endif() target_link_libraries(${TARGET_NAME} PRIVATE openvino::runtime) diff --git a/modules/custom_operations/user_ie_extensions/src/tokenizer/CMakeLists.txt b/modules/custom_operations/user_ie_extensions/src/tokenizer/CMakeLists.txt index 606115e27..bc9d551f5 100644 --- a/modules/custom_operations/user_ie_extensions/src/tokenizer/CMakeLists.txt +++ b/modules/custom_operations/user_ie_extensions/src/tokenizer/CMakeLists.txt @@ -8,10 +8,6 @@ endif() option(BUILD_FAST_TOKENIZERS OFF) -# to build only sentencepiece-static target -# set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -# PROPERTY EXCLUDE_FROM_ALL ON) - # # Compile flags # @@ -116,7 +112,8 @@ else() if(WIN32 AND X86_64) # we use re2 library in regex_normalization operation, so have to add to this list # because prebuilt fast_tokenizers package does not provide this library - list(APPEND FAST_TOKENIZER_LIBS re2) + # list(APPEND FAST_TOKENIZER_LIBS re2) + set (RE2_LIBS re2) endif() endif() @@ -138,15 +135,21 @@ target_include_directories(${TARGET_NAME} PRIVATE set_property(DIRECTORY ${sentencepiece_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL ON) +set_property(DIRECTORY ${re2_SOURCE_DIR} + PROPERTY EXCLUDE_FROM_ALL ON) + if(CMAKE_CL_64) target_compile_definitions(sentencepiece-static PRIVATE _CRT_SECURE_NO_WARNINGS _SCL_SECURE_NO_WARNINGS) endif() -target_link_libraries(${TARGET_NAME} PRIVATE ${FAST_TOKENIZER_LIBS} sentencepiece-static) +target_link_libraries(${TARGET_NAME} PRIVATE ${FAST_TOKENIZER_LIBS} ${RE2_LIBS} sentencepiece-static) # string_view is used from cxx17 string(REPLACE " " ";" cxx_flags "${cxx_flags}") set_target_properties(${TARGET_NAME} PROPERTIES CXX_STANDARD 17 COMPILE_OPTIONS "${cxx_flags}") -install(FILES ${FAST_TOKENIZER_LIBS} DESTINATION .) \ No newline at end of file +file(GLOB DEPS_LIBS ${fast_tokenizer_SOURCE_DIR}/lib/*.dll ${fast_tokenizer_SOURCE_DIR}/third_party/lib/*.dll) +file(COPY ${DEPS_LIBS} DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${CMAKE_BUILD_TYPE}) + +install(FILES ${DEPS_LIBS} DESTINATION .) \ No newline at end of file diff --git a/modules/custom_operations/user_ie_extensions/src/tokenizer/python/ov_tokenizer/__init__.py b/modules/custom_operations/user_ie_extensions/src/tokenizer/python/ov_tokenizer/__init__.py index 5803a64ae..623965673 100644 --- a/modules/custom_operations/user_ie_extensions/src/tokenizer/python/ov_tokenizer/__init__.py +++ b/modules/custom_operations/user_ie_extensions/src/tokenizer/python/ov_tokenizer/__init__.py @@ -15,7 +15,8 @@ _ext_path = os.path.join(_ext_libs_path, "libuser_ov_extensions.so") if sys.platform == "win32": - _ext_path = os.path.join(_ext_libs_path, "libuser_ov_extensions.dll") + _ext_libs_path = os.path.join(_ext_libs_path, "bin") + _ext_path = os.path.join(_ext_libs_path, "user_ov_extensions.dll") if os.path.isdir(_ext_libs_path): # On Windows, with Python >= 3.8, DLLs are no longer imported from the PATH. os.add_dll_directory(os.path.abspath(_ext_path))