BLD: Remove Python 3.8 & Supports Python 3.12

.github/workflows/build-wheel.yaml

@@ -22,11 +22,11 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-13, windows-latest]
         arch: [auto]
-        requires-python: [">=3.8,<3.10", ">=3.10,<3.12"]
+        # Cannot set like this: ">=3.9,<3.11" here,
+        # which will result in two packages running within a single runner,
+        # potentially causing conflicts in the Python environment.
+        requires-python: [">=3.9,<3.10", ">=3.10,<3.11", ">=3.11,<3.12", ">=3.12,<3.13"]
         include:
-          - os: ubuntu-latest
-            arch: aarch64
-            requires-python: ">=3.8,<3.9"
           - os: ubuntu-latest
             arch: aarch64
             requires-python: ">=3.9,<3.10"
@@ -36,18 +36,33 @@ jobs:
           - os: ubuntu-latest
             arch: aarch64
             requires-python: ">=3.11,<3.12"
+          - os: ubuntu-latest
+            arch: aarch64
+            requires-python: ">=3.12,<3.13"
+          - os: macos-13
+            arch: universal2
+            requires-python: ">=3.9,<3.10"
+          - os: macos-13
+            arch: universal2
+            requires-python: ">=3.10,<3.11"
           - os: macos-13
             arch: universal2
-            requires-python: ">=3.8,<3.10"
+            requires-python: ">=3.11,<3.12"
           - os: macos-13
             arch: universal2
-            requires-python: ">=3.10,<3.12"
+            requires-python: ">=3.12,<3.13"
+          - os: macos-13
+            arch: arm64
+            requires-python: ">=3.9,<3.10"
           - os: macos-13
             arch: arm64
-            requires-python: ">=3.8,<3.10"
+            requires-python: ">=3.10,<3.11"
+          - os: macos-13
+            arch: arm64
+            requires-python: ">=3.11,<3.12"
           - os: macos-13
             arch: arm64
-            requires-python: ">=3.10,<3.12"
+            requires-python: ">=3.12,<3.13"
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/python.yaml

@@ -79,13 +79,13 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest", "macos-13", "windows-latest"]
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         module: ["xoscar"]
         exclude:
-          - { os: macos-13, python-version: 3.9}
           - { os: macos-13, python-version: 3.10}
-          - { os: windows-latest, python-version: 3.9}
+          - { os: macos-13, python-version: 3.11}
           - { os: windows-latest, python-version: 3.10}
+          - { os: windows-latest, python-version: 3.11}
         include:
           - { os: self-hosted, module: gpu, python-version: 3.11}
           - { os: ubuntu-20.04, module: doc-build, python-version: 3.9}
@@ -98,7 +98,7 @@ jobs:
         submodules: recursive
     - name: Add msbuild to PATH
       if: ${{ matrix.os == 'windows-latest'}}
-      uses: microsoft/setup-msbuild@v1.1
+      uses: microsoft/setup-msbuild@v2
     - name: Set up conda ${{ matrix.python-version }}
       uses: conda-incubator/setup-miniconda@v3
       if: ${{ matrix.module != 'gpu' }}
@@ -112,6 +112,12 @@ jobs:
       run: |
         conda install -c conda-forge libstdcxx-ng
 
+    # Important for python == 3.12
+    - name: Update pip and setuptools
+      if: ${{ matrix.python-version == '3.12' }}
+      run: |
+        python -m pip install -U pip setuptools
+
     - name: Install dependencies
       env:
         MODULE: ${{ matrix.module }}
@@ -122,7 +128,7 @@ jobs:
       working-directory: ./python
 
     - name: Install ucx dependencies
-      if: ${{ (matrix.os == 'ubuntu-latest') && (matrix.python-version >= '3.9')}}
+      if: ${{ matrix.os == 'ubuntu-latest' }}
       run: |
         # ucx-py move to ucxx and ucxx-cu12 can be run on CPU
         # conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py

.gitignore

@@ -9,6 +9,7 @@ __pycache__/
 # Distribution / packaging
 .Python
 build/
+.DS_Store
 develop-eggs/
 dist/
 downloads/

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/psf/black
-    rev: 24.2.0
+    rev: 24.10.0
     hooks:
       - id: black
         files: python/xoscar

CMakeLists.txt

@@ -16,6 +16,11 @@ else()
   option(USE_LIBUV "Build libuv transport on others" ON)
 endif()
 
+if(MSVC)
+  add_compile_options(/utf-8)
+  message(STATUS "Done setting /utf-8 for MSVC")
+endif()
+
 include_directories(${CMAKE_SOURCE_DIR})
 #find python3 include dir
 execute_process(COMMAND python -c "import sysconfig; print(sysconfig.get_path('include'))"
@@ -29,8 +34,9 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
     execute_process(
       COMMAND
         cmd /c
-        "cd ..\\..\\..\\..\\..\\third_party\\libuv&&mkdir build&&cd build && mkdir uvlib&&cmake .. -DCMAKE_INSTALL_PREFIX=uvlib&&msbuild.exe INSTALL.vcxproj"
+        "echo %cd% && cd ..\\..\\..\\..\\..\\third_party\\libuv && mkdir build && cd build && mkdir uvlib && cmake .. -DCMAKE_INSTALL_PREFIX=uvlib && msbuild.exe INSTALL.vcxproj"
     )
+    message(STATUS "Done creating libuv_dir = ${libuv_dir}")
   endif()
   set(libuv_ROOT ${CMAKE_SOURCE_DIR}/third_party/libuv/build/uvlib)
   set(uv_HEADER_PATH ${CMAKE_SOURCE_DIR}/third_party/libuv/include)

cpp/collective/rendezvous/include/error.h

@@ -47,7 +47,7 @@ struct formatter<std::error_code> {
     decltype(auto) format(const std::error_code &err,
                           FormatContext &ctx) const {
         return format_to(ctx.out(),
-                         "({}: {} - {})",
+                         fmt::runtime("({}: {} - {})"),
                          err.category(),
                          err.value(),
                          err.message());

cpp/collective/rendezvous/src/socket.cpp

@@ -16,6 +16,7 @@ limitations under the License. */
 #include "error.h"
 #include "exception.h"
 #include "fmt/chrono.h"
+#include "fmt/ranges.h"
 
 #include <system_error>
 #include <utility>

python/pyproject.toml

@@ -1,6 +1,7 @@
 [build-system]
 requires = [
-    "setuptools<64",
+    "setuptools<64; python_version<'3.12'",
+    "setuptools>=75; python_version>='3.12'",
     "packaging",
     "wheel",
     "oldest-supported-numpy",
@@ -42,6 +43,6 @@ markers = [
 ]
 
 [tool.cibuildwheel]
-build = ["cp38-*", "cp39-*", "cp310-*", "cp311-*"]
-skip = "pp* *musllinux* *i686 cp36* cp38-win32 cp39-win32 cp310-win32 cp311-win32"
+build = ["cp39-*", "cp310-*", "cp311-*", "cp312-*"]
+skip = "pp* *musllinux* *i686 cp36* cp38-win32 cp39-win32 cp310-win32 cp311-win32 cp312-win32"
 manylinux-x86_64-image = "manylinux2014"

python/setup.cfg

@@ -7,15 +7,15 @@ maintainer = Qin Xuye
 maintainer_email = [email protected]
 license = Apache License 2.0
 url = http://github.com/xorbitsai/xoscar
-python_requires = >=3.8
+python_requires = >=3.9
 classifier =
     Operating System :: OS Independent
     Programming Language :: Python
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
+    Programming Language :: Python :: 3.12
     Programming Language :: Python :: Implementation :: CPython
     Topic :: Software Development :: Libraries
 
@@ -31,7 +31,6 @@ install_requires =
     cloudpickle>=1.5.0
     psutil>=5.9.0
     tblib>=1.7.0
-    pickle5; python_version<"3.8"
     uvloop>=0.14.0; sys_platform!="win32"
     packaging
 
@@ -53,7 +52,6 @@ dev =
     sphinx
     pydata-sphinx-theme>=0.3.0
     sphinx-intl>=0.9.9
-    mock>=4.0.0; python_version<"3.8"
     flake8>=3.8.0
     black
 doc =
@@ -82,6 +80,8 @@ omit =
     xoscar/nvutils.py
     *.pxd
     */tests/*
+disable_warnings =
+    include-ignored
 
 [coverage:report]
 exclude_lines =

python/setup.py

@@ -17,6 +17,7 @@
 import re
 import subprocess
 import sys
+import sysconfig
 from distutils.command.build_ext import build_ext as _du_build_ext
 from distutils.file_util import copy_file, move_file
 from pathlib import Path
@@ -28,6 +29,7 @@
 from packaging.version import Version
 from setuptools import Extension, setup
 from setuptools.command.build_ext import build_ext
+from setuptools.command.install_lib import install_lib
 from setuptools.extension import Library
 
 try:
@@ -143,6 +145,45 @@ def build_long_description():
     "win-arm64": "ARM64",
 }
 
+TARGET_TO_PLAT = {
+    'x86': 'win32',
+    'x64': 'win-amd64',
+    'arm': 'win-arm32',
+    'arm64': 'win-arm64',
+}
+
+
+# Copied from https://github.com/pypa/setuptools/blob/main/setuptools/_distutils/util.py#L50
+def get_host_platform():
+    """
+    Return a string that identifies the current platform. Use this
+    function to distinguish platform-specific build directories and
+    platform-specific built distributions.
+    """
+
+    # This function initially exposed platforms as defined in Python 3.9
+    # even with older Python versions when distutils was split out.
+    # Now it delegates to stdlib sysconfig, but maintains compatibility.
+    return sysconfig.get_platform()
+
+
+def get_platform():
+    if os.name == 'nt':
+        target = os.environ.get('VSCMD_ARG_TGT_ARCH')
+        return TARGET_TO_PLAT.get(target) or get_host_platform()
+    return get_host_platform()
+
+
+plat_specifier = ".{}-{}".format(get_platform(), sys.implementation.cache_tag)
+
+
+def get_build_lib():
+    return os.path.join("build", "lib" + plat_specifier)
+
+
+def get_build_temp():
+    return os.path.join("build", 'temp' + plat_specifier)
+
 
 # A CMakeExtension needs a sourcedir instead of a file list.
 # The name must be the _single_ output extension from the CMake build.
@@ -154,6 +195,18 @@ def __init__(self, name: str, sourcedir: str = "") -> None:
 
 
 class CMakeBuild(build_ext):
+    def finalize_options(self):
+        """
+        For python 3.12, the build_temp and build_lib dirs are temp dirs which are depended on your OS,
+        which leads to that cannot find the copy directory during C++ compiled process.
+        However, for Python < 3.12, these two dirs can be automatically located in the `build` directory of the project directory.
+        Therefore, in order to be compatible with all Python versions,
+        directly using fixed dirs here by coping source codes from `setuptools`.
+        """
+        self.build_temp = get_build_temp()
+        self.build_lib = get_build_lib()
+        super().finalize_options()
+
     def copy_extensions_to_source(self):
         build_py = self.get_finalized_command('build_py')
         for ext in self.extensions:

python/xoscar/aio/__init__.py

@@ -11,15 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-import asyncio
-import sys
-
 from .file import AioFileObject
 from .lru import alru_cache
 from .parallelism import AioEvent
-
-if sys.version_info[:2] < (3, 9):
-    from ._threads import to_thread
-
-    asyncio.to_thread = to_thread