Merge branch 'main' into force-static-libs

codebuild/cd/manylinux-x64-build.yml

@@ -12,8 +12,6 @@ phases:
   build:
     commands:
       - echo Build started on `date`
-      - /opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-      - auditwheel repair --plat manylinux1_x86_64 dist/awscrt-*cp37-cp37m-linux_x86_64.whl
       - /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
       - auditwheel repair --plat manylinux1_x86_64 dist/awscrt-*cp38-cp38-linux_x86_64.whl
       - /opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel

codebuild/cd/manylinux-x86-build.yml

@@ -12,8 +12,6 @@ phases:
   build:
     commands:
       - echo Build started on `date`
-      - /opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-      - auditwheel repair --plat manylinux1_i686 dist/awscrt-*cp37-cp37m-linux_i686.whl
       - /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
       - auditwheel repair --plat manylinux1_i686 dist/awscrt-*cp38-cp38-linux_i686.whl
       - /opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel

continuous-delivery/build-wheels-manylinux2014-aarch64.sh

@@ -4,9 +4,6 @@ set -ex
 
 /opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py
 
-/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp37*.whl
-
 /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp38*.whl
 
@@ -19,9 +16,17 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp310*.whl
 /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl
 
-# Don't need to build wheels for Python 3.12 and later.
+# Don't need to build wheels for Python 3.12.
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
+# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+# Manylinux images don't contain setuptools from Python 3.13, so we need to install it.
+# Install in a custom location due to access issues.
+/opt/python/cp313-cp313/bin/python -m pip install --target ./local -r requirements-dev.txt
+export PYTHONPATH=./local:$PYTHONPATH
+/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp313*.whl
+
 rm dist/*.whl
 cp -rv wheelhouse/* dist/
 

continuous-delivery/build-wheels-manylinux2014-x86_64.sh

@@ -4,9 +4,6 @@ set -ex
 
 /opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py
 
-/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp37*.whl
-
 /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp38*.whl
 
@@ -19,9 +16,17 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp310*.whl
 /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl
 
-# Don't need to build wheels for Python 3.12 and later.
+# Don't need to build wheels for Python 3.12.
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
+# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+# Manylinux images don't contain setuptools from Python 3.13, so we need to install it.
+# Install in a custom location due to access issues.
+/opt/python/cp313-cp313/bin/python -m pip install --target ./local -r requirements-dev.txt
+export PYTHONPATH=./local:$PYTHONPATH
+/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp313*.whl
+
 rm dist/*.whl
 cp -rv wheelhouse/* dist/
 

continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh

@@ -4,9 +4,6 @@ set -ex
 
 /opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py
 
-/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp37*.whl
-
 /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp38*.whl
 
@@ -19,9 +16,17 @@ auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp310*.whl
 /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp311*.whl
 
-# Don't need to build wheels for Python 3.12 and later.
+# Don't need to build wheels for Python 3.12.
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
+# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+# Manylinux images don't contain setuptools from Python 3.13, so we need to install it.
+# Install in a custom location due to access issues.
+/opt/python/cp313-cp313/bin/python -m pip install --target ./local -r requirements-dev.txt
+export PYTHONPATH=./local:$PYTHONPATH
+/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp313*.whl
+
 rm dist/*.whl
 cp -rv wheelhouse/* dist/
 

continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh

@@ -4,9 +4,6 @@ set -ex
 
 /opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py
 
-/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp37*.whl
-
 /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp38*.whl
 
@@ -19,9 +16,17 @@ auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp310*.whl
 /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp311*.whl
 
-# Don't need to build wheels for Python 3.12 and later.
+# Don't need to build wheels for Python 3.12.
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
+# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+# Manylinux images don't contain setuptools from Python 3.13, so we need to install it.
+# Install in a custom location due to access issues.
+/opt/python/cp313-cp313/bin/python -m pip install --target ./local -r requirements-dev.txt
+export PYTHONPATH=./local:$PYTHONPATH
+/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp313*.whl
+
 rm dist/*.whl
 cp -rv wheelhouse/* dist/
 

continuous-delivery/build-wheels-osx.sh

@@ -5,13 +5,15 @@ set -ex
 
 /Library/Frameworks/Python.framework/Versions/3.9/bin/python3 ./continuous-delivery/update-version.py
 
-/Library/Frameworks/Python.framework/Versions/3.7/bin/python3 setup.py sdist bdist_wheel
 /Library/Frameworks/Python.framework/Versions/3.8/bin/python3 setup.py sdist bdist_wheel
 /Library/Frameworks/Python.framework/Versions/3.9/bin/python3 setup.py sdist bdist_wheel
 /Library/Frameworks/Python.framework/Versions/3.10/bin/python3 setup.py sdist bdist_wheel
 /Library/Frameworks/Python.framework/Versions/3.11/bin/python3 setup.py sdist bdist_wheel
 
-# Don't need to build wheels for Python 3.12 and later.
+# Don't need to build wheels for Python 3.12.
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
+# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+/Library/Frameworks/Python.framework/Versions/3.13/bin/python3 setup.py sdist bdist_wheel
+
 #now you just need to run twine (that's in a different script)

continuous-delivery/build-wheels-win32.bat

@@ -1,12 +1,17 @@
 
 "C:\Program Files (x86)\Python39-32\python.exe" .\continuous-delivery\update-version.py || goto error
 
-"C:\Program Files (x86)\Python37-32\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files (x86)\Python38-32\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files (x86)\Python39-32\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files (x86)\Python310-32\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files (x86)\Python311-32\python.exe" setup.py sdist bdist_wheel || goto error
 
+:: Don't need to build wheels for Python 3.12.
+:: The 3.11 wheel uses the stable ABI, so it works with newer versions too.
+
+:: We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions.
+"C:\Program Files (x86)\Python313-32\python.exe" setup.py sdist bdist_wheel || goto error
+
 goto :EOF
 
 :error

continuous-delivery/build-wheels-win64.bat

@@ -1,11 +1,16 @@
 "C:\Program Files\Python39\python.exe" continuous-delivery\update-version.py || goto error
 
-"C:\Program Files\Python37\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files\Python38\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files\Python39\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files\Python310\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files\Python311\python.exe" setup.py sdist bdist_wheel || goto error
 
+:: Don't need to build wheels for Python 3.12.
+:: The 3.11 wheel uses the stable ABI, so it works with newer versions too.
+
+:: We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions.
+"C:\Program Files\Python313\python.exe" setup.py sdist bdist_wheel || goto error
+
 goto :EOF
 
 :error

setup.py

@@ -1,8 +1,6 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0.
-
 import codecs
-import distutils.ccompiler
 import glob
 import os
 import os.path
@@ -78,6 +76,7 @@ def determine_generator_args():
     if sys.platform == 'win32':
         try:
             # See which compiler python picks
+            import distutils.ccompiler
             compiler = distutils.ccompiler.new_compiler()
             compiler.initialize()
 
@@ -308,8 +307,12 @@ def run(self):
 class bdist_wheel_abi3(bdist_wheel):
     def get_tag(self):
         python, abi, plat = super().get_tag()
-        if python.startswith("cp") and sys.version_info >= (3, 11):
-            # on CPython, our wheels are abi3 and compatible back to 3.11
+        # on CPython, our wheels are abi3 and compatible back to 3.11
+        if python.startswith("cp") and sys.version_info >= (3, 13):
+            # 3.13 deprecates PyWeakref_GetObject(), adds alternative
+            return "cp313", "abi3", plat
+        elif python.startswith("cp") and sys.version_info >= (3, 11):
+            # 3.11 is the first stable ABI that has everything we need
             return "cp311", "abi3", plat
 
         return python, abi, plat
@@ -329,6 +332,11 @@ def awscrt_ext():
     libraries.reverse()
 
     if sys.platform == 'win32':
+        # distutils is deprecated in Python 3.10 and removed in 3.12. However, it still works because Python defines a compatibility interface as long as setuptools is installed.
+        # We don't have an official alternative for distutils.ccompiler as of September 2024. See: https://github.com/pypa/setuptools/issues/2806
+        # Once that issue is resolved, we can migrate to the official solution.
+        # For now, restrict distutils to Windows only, where it's needed.
+        import distutils.ccompiler
         # the windows apis being used under the hood. Since we're static linking we have to follow the entire chain down
         libraries += ['Secur32', 'Crypt32', 'Advapi32', 'NCrypt', 'BCrypt', 'Kernel32', 'Ws2_32', 'Shlwapi']
         # Ensure that debug info is in the obj files, and that it is linked into the .pyd so that
@@ -379,7 +387,7 @@ def awscrt_ext():
         # rare cases where that didn't happen, so let's be explicit.
         extra_link_args += ['-pthread']
 
-    if distutils.ccompiler.get_default_compiler() != 'msvc':
+    if sys.platform != 'win32' or distutils.ccompiler.get_default_compiler() != 'msvc':
         extra_compile_args += ['-Wno-strict-aliasing', '-std=gnu99']
 
         # treat warnings as errors in development mode
@@ -408,7 +416,13 @@ def awscrt_ext():
                 else:
                     extra_link_args += ['-Wl,--fatal-warnings']
 
-    if sys.version_info >= (3, 11):
+    # prefer building with stable ABI, so a wheel can work with multiple major versions
+    if sys.version_info >= (3, 13):
+        # 3.13 deprecates PyWeakref_GetObject(), adds alternative
+        define_macros.append(('Py_LIMITED_API', '0x030D0000'))
+        py_limited_api = True
+    elif sys.version_info >= (3, 11):
+        # 3.11 is the first stable ABI that has everything we need
         define_macros.append(('Py_LIMITED_API', '0x030B0000'))
         py_limited_api = True
 

source/http_stream.c

@@ -203,7 +203,10 @@ static void s_on_stream_complete(struct aws_http_stream *native_stream, int erro
     }
 
     /* DECREF python self, we don't need to force it to stay alive any longer. */
-    Py_DECREF(PyWeakref_GetObject(stream->self_proxy));
+    PyObject *self = aws_py_weakref_get_ref(stream->self_proxy);
+    /* DECREF twice because `aws_py_weakref_get_ref` returns a strong reference */
+    Py_XDECREF(self);
+    Py_XDECREF(self);
 
     PyGILState_Release(state);
     /*************** GIL RELEASE ***************/

source/module.c

@@ -516,6 +516,39 @@ PyObject *aws_py_memory_view_from_byte_buffer(struct aws_byte_buf *buf) {
     return PyMemoryView_FromMemory(mem_start, mem_size, PyBUF_WRITE);
 }
 
+PyObject *aws_py_weakref_get_ref(PyObject *ref) {
+    /* If Python >= 3.13 */
+#if PY_VERSION_HEX >= 0x030D0000
+    /* Use PyWeakref_GetRef() (new in Python 3.13), which gets you:
+     * a new strong reference,
+     * or NULL because ref is dead,
+     * or -1 because you called it wrong */
+    PyObject *obj = NULL;
+    if (PyWeakref_GetRef(ref, &obj) == -1) {
+        PyErr_WriteUnraisable(PyErr_Occurred());
+        AWS_ASSERT(0 && "expected a weakref");
+    }
+    return obj;
+
+#else
+    /* Use PyWeakref_GetObject() (deprecated as of Python 3.13), which gets you:
+     * a borrowed reference,
+     * or Py_None because ref is dead,
+     * or NULL because you called it wrong */
+    PyObject *obj = PyWeakref_GetObject(ref); /* borrowed reference */
+    if (obj == NULL) {
+        PyErr_WriteUnraisable(PyErr_Occurred());
+        AWS_ASSERT(0 && "expected a weakref");
+    } else if (obj == Py_None) {
+        obj = NULL;
+    } else {
+        /* Be like PyWeakref_GetRef() and make it new strong reference */
+        Py_INCREF(obj);
+    }
+    return obj;
+#endif
+}
+
 int aws_py_gilstate_ensure(PyGILState_STATE *out_state) {
     if (AWS_LIKELY(Py_IsInitialized())) {
         *out_state = PyGILState_Ensure();

source/module.h

@@ -106,6 +106,27 @@ PyObject *aws_py_get_error_message(PyObject *self, PyObject *args);
 /* Create a write-only memoryview from the remaining free space in an aws_byte_buf */
 PyObject *aws_py_memory_view_from_byte_buffer(struct aws_byte_buf *buf);
 
+/* Python 3.13+ changed the function to get a reference from WeakRef. This function is an abstraction over two different
+ * APIs since we support Python versions before 3.13. Returns a strong reference if non-null, which you must release. */
+
+/**
+ * Given a weak reference, returns a NEW strong reference to the referenced object,
+ * or NULL if the reference is dead (this function NEVER raises a python exception or AWS Error).
+ *
+ * You MUST NOT call this if ref came from a user, or ref is NULL.
+ *
+ * This is a simplified version of PyWeakref_GetRef() / PyWeakref_GetObject().
+ * Simpler because:
+ * - Python 3.13 adds PyWeakref_GetRef() and deprecates PyWeakref_GetObject().
+ *   This function calls the appropriate one.
+ *
+ * - This functions has 2 outcomes instead of 3:
+ *   The 3rd being a Python exception for calling it incorrectly.
+ *   If that happens, this function calls PyErr_WriteUnraisable() to clear the exception,
+ *   which is what you would have done anyway.
+ */
+PyObject *aws_py_weakref_get_ref(PyObject *ref);
+
 /* Allocator that calls into PyObject_[Malloc|Free|Realloc] */
 struct aws_allocator *aws_py_get_allocator(void);