BUG: fix xoscar installed by pypi not contains xoscar_pygloo.so (#62)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

.github/workflows/build-wheel.yaml

@@ -65,8 +65,8 @@ jobs:
           CIBW_PROJECT_REQUIRES_PYTHON: ${{ matrix.requires-python }}
           CIBW_BEFORE_BUILD_MACOS: pip install -r CI/requirements-wheel.txt
           CIBW_BEFORE_BUILD_WINDOWS: pip install -r CI/requirements-wheel.txt
-          CIBW_TEST_COMMAND: pytest {project}/CI/test_functionality.py
-          CIBW_TEST_REQUIRES: pytest requests
+          CIBW_TEST_COMMAND: pytest {project}/CI/test_functionality.py --capture=no
+          CIBW_TEST_REQUIRES: pytest requests pytest-asyncio
           CIBW_BUILD_VERBOSITY: 1
         with:
           package-dir: ./python/

CI/test_functionality.py

@@ -4,6 +4,9 @@
 
 import xoscar as mo
 
+import platform
+import sys
+
 
 class MyActor(mo.Actor):
     def __init__(self):
@@ -38,3 +41,10 @@ async def test_basic_cases():
         assert await ref1.add(1) == 1
         assert await ref2.add(2) == 2
         assert await ref1.add_from(ref2) == 3
+
+def test_pygloo():
+    is_windows = sys.platform.startswith('win')
+    bit_number = platform.architecture()[0]
+    if not (is_windows and bit_number=="32bit"):
+        import xoscar.collective.xoscar_pygloo as xp
+        print(type(xp.ReduceOp.SUM))

cpp/CMakeLists.txt

@@ -3,9 +3,11 @@ cmake_minimum_required(VERSION 3.11...3.21)
 project(XoscarCollective)
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
-if(NOT DEFINED LIBRARY_OUTPUT_DIRECTORY)
-  set(LIBRARY_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}/python/xoscar/collective)
-endif()
+file(GLOB TMP_DIRS "../python/build/lib*")
+foreach(TMP_DIR ${TMP_DIRS})
+  message(${TMP_DIR})
+  set(LIBRARY_OUTPUT_DIRECTORY ${TMP_DIR}/xoscar/collective)
+endforeach()
 
 include_directories(${CMAKE_SOURCE_DIR}/cpp/collective/rendezvous/include)
 include_directories(${CMAKE_SOURCE_DIR}/cpp/collective/gloo/include)

python/setup.py

@@ -18,7 +18,7 @@
 import subprocess
 import sys
 from distutils.command.build_ext import build_ext as _du_build_ext
-from distutils.file_util import copy_file
+from distutils.file_util import copy_file, move_file
 from pathlib import Path
 
 from sysconfig import get_config_vars
@@ -176,27 +176,19 @@ def copy_extensions_to_source(self):
                 )
                 if ext._needs_stub:
                     self.write_stub(package_dir or os.curdir, ext, True)
-            elif sys.platform.startswith('win'):
+            else:
                 fullname = self.get_ext_fullname(ext.name)
+                collective_dir = os.path.join("xoscar" , "collective")
                 filename = self.get_ext_filename(fullname)
-                modpath = fullname.split('.')
-                package = '.'.join(modpath[:-1])
-                package_dir = build_py.get_package_dir(package)
-                if package_dir=="" and ext.name=="xoscar_pygloo":
-                    package_dir="xoscar/collective"
-                dest_filename = os.path.join(package_dir,
+                src_dir = os.path.join(self.build_lib , collective_dir)
+                src_filename = os.path.join(src_dir , filename)
+                dest_filename = os.path.join(collective_dir,
                                                 os.path.basename(filename))
-                src_filename = os.path.join(self.build_lib, filename)
-
-                # Always copy, even if source is older than destination, to ensure
-                # that the right extensions for the current Python/platform are
-                # used.
                 copy_file(
                     src_filename, dest_filename, verbose=self.verbose,
                     dry_run=self.dry_run
                 )
-                if ext._needs_stub:
-                    self.write_stub(package_dir or os.curdir, ext, True)
+
 
     def build_extension(self, ext):
         # TODO: support windows compilation
@@ -224,7 +216,6 @@ def build_Cmake(self, ext: XoscarCmakeExtension) -> None:
         ext_fullpath = Path.cwd() / self.get_ext_fullpath(ext.name)
         extdir = ext_fullpath.parent.resolve()
         source_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-        output_directory_collective = Path(source_dir) / "python" / "xoscar" / "collective"
         build_temp = Path(self.build_temp) / ext.name
         if not build_temp.exists():
             build_temp.mkdir(parents=True)
@@ -244,7 +235,6 @@ def build_Cmake(self, ext: XoscarCmakeExtension) -> None:
         # from Python.
         cmake_args = [
             f"-DBUILD_TMP_DIR={build_temp}",
-            f"-DLIBRARY_OUTPUT_DIRECTORY={output_directory_collective}",
             f"-DPYTHON_PATH={sys.executable}",
             f"-DCMAKE_BUILD_TYPE={cfg}",  # not used on MSVC, but no harm
         ]
@@ -315,6 +305,27 @@ def build_Cmake(self, ext: XoscarCmakeExtension) -> None:
         subprocess.run(
             ["cmake", "--build", ".", *build_args], cwd=build_temp, check=True
         )
+        if sys.platform.startswith('win'):
+            for file in os.listdir(self.build_lib):
+                if file.startswith("xoscar_pygloo"):
+                    src_filename = os.path.join(self.build_lib,
+                                            os.path.basename(file))
+                    dest_dir = os.path.join(self.build_lib,
+                                                    "xoscar\\collective")
+                    if not os.path.exists(dest_dir):
+                        os.mkdir(dest_dir)
+                    dest_filename = os.path.join(dest_dir,
+                                            os.path.basename(file))
+                    move_file(
+                        src_filename, dest_filename, verbose=self.verbose,
+                        dry_run=self.dry_run
+                    )
+                    libuv_filename = "xoscar\\collective\\uv.dll"
+                    libuv_dest_filename = os.path.join(dest_dir, "uv.dll")
+                    copy_file(
+                        libuv_filename, libuv_dest_filename, verbose=self.verbose,
+                        dry_run=self.dry_run
+                    )
 
 
 setup_options = dict(