[python3] add features for readline and extensions (#41183)

Author: Neumann-A

ports/python3/0019-fix-ssl-linkage.patch

@@ -0,0 +1,22 @@
+diff --git a/configure.ac b/configure.ac
+index ef899b881d..9ed1836608 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -6712,7 +6712,7 @@ done
+ 
+ # check if OpenSSL libraries work as expected
+ WITH_SAVE_ENV([
+-  LIBS="$LIBS $OPENSSL_LIBS"
++  LIBS="$OPENSSL_LIBS $LIBS"
+   CFLAGS="$CFLAGS $OPENSSL_INCLUDES"
+   LDFLAGS="$LDFLAGS $OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH"
+ 
+@@ -6737,7 +6737,7 @@ WITH_SAVE_ENV([
+ ])
+ 
+ WITH_SAVE_ENV([
+-  LIBS="$LIBS $LIBCRYPTO_LIBS"
++  LIBS="$LIBCRYPTO_LIBS $LIBS"
+   CFLAGS="$CFLAGS $OPENSSL_INCLUDES"
+   LDFLAGS="$LDFLAGS $OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH"
+ 

ports/python3/portfile.cmake

@@ -3,6 +3,15 @@ if (VCPKG_LIBRARY_LINKAGE STREQUAL dynamic AND VCPKG_CRT_LINKAGE STREQUAL static
     set(VCPKG_LIBRARY_LINKAGE static)
 endif()
 
+if("extensions" IN_LIST FEATURES)
+    if(VCPKG_TARGET_IS_WINDOWS)
+        vcpkg_check_linkage(ONLY_DYNAMIC_LIBRARY)
+    endif()
+    set(PYTHON_HAS_EXTENSIONS ON)
+else()
+    set(PYTHON_HAS_EXTENSIONS OFF)
+endif()
+
 if(NOT VCPKG_HOST_IS_WINDOWS)
     message(WARNING "${PORT} currently requires the following programs from the system package manager:
     autoconf automake autoconf-archive
@@ -35,6 +44,7 @@ set(PATCHES
     0015-dont-use-WINDOWS-def.patch
     0016-undup-ffi-symbols.patch # Required for lld-link.
     0018-fix-sysconfig-include.patch
+    0019-fix-ssl-linkage.patch
 )
 
 if(VCPKG_LIBRARY_LINKAGE STREQUAL "static")
@@ -57,6 +67,10 @@ endif()
 
 if(VCPKG_TARGET_IS_WINDOWS)
     string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "dynamic" PYTHON_ALLOW_EXTENSIONS)
+    if(PYTHON_HAS_EXTENSIONS AND NOT PYTHON_ALLOW_EXTENSIONS)
+        # This should never be reached due to vcpkg_check_linkage above
+        message(FATAL_ERROR "Cannot build python extensions! Python extensions on windows can only be built if python is a dynamic library!")
+    endif()
     # The Windows 11 SDK has a problem that causes it to error on the resource files, so we patch that.
     vcpkg_get_windows_sdk(WINSDK_VERSION)
     if("${WINSDK_VERSION}" VERSION_GREATER_EQUAL "10.0.22000")
@@ -102,7 +116,7 @@ endfunction()
 if(VCPKG_TARGET_IS_WINDOWS)
     # Due to the way Python handles C extension modules on Windows, a static python core cannot
     # load extension modules.
-    if(PYTHON_ALLOW_EXTENSIONS)
+    if(PYTHON_HAS_EXTENSIONS)
         find_library(BZ2_RELEASE NAMES bz2 PATHS "${CURRENT_INSTALLED_DIR}/lib" NO_DEFAULT_PATH)
         find_library(BZ2_DEBUG NAMES bz2d PATHS "${CURRENT_INSTALLED_DIR}/debug/lib" NO_DEFAULT_PATH)
         find_library(CRYPTO_RELEASE NAMES libcrypto PATHS "${CURRENT_INSTALLED_DIR}/lib" NO_DEFAULT_PATH)
@@ -120,7 +134,7 @@ if(VCPKG_TARGET_IS_WINDOWS)
         list(APPEND add_libs_rel "${BZ2_RELEASE};${EXPAT_RELEASE};${FFI_RELEASE};${LZMA_RELEASE};${SQLITE_RELEASE}")
         list(APPEND add_libs_dbg "${BZ2_DEBUG};${EXPAT_DEBUG};${FFI_DEBUG};${LZMA_DEBUG};${SQLITE_DEBUG}")
     else()
-        message(STATUS "WARNING: Static builds of Python will not have C extension modules available.")
+        message(STATUS "WARNING: Extensions have been disabled. No C extension modules will be available.")
     endif()
     find_library(ZLIB_RELEASE NAMES zlib PATHS "${CURRENT_INSTALLED_DIR}/lib" NO_DEFAULT_PATH)
     find_library(ZLIB_DEBUG NAMES zlib zlibd PATHS "${CURRENT_INSTALLED_DIR}/debug/lib" NO_DEFAULT_PATH)
@@ -136,7 +150,7 @@ if(VCPKG_TARGET_IS_WINDOWS)
     )
 
     list(APPEND VCPKG_CMAKE_CONFIGURE_OPTIONS "-DVCPKG_SET_CHARSET_FLAG=OFF")
-    if(PYTHON_ALLOW_EXTENSIONS)
+    if(PYTHON_HAS_EXTENSIONS)
         set(OPTIONS
             "/p:IncludeExtensions=true"
             "/p:IncludeExternals=true"
@@ -189,7 +203,7 @@ if(VCPKG_TARGET_IS_WINDOWS)
     endif()
 
     # The extension modules must be placed in the DLLs directory, so we can't use vcpkg_copy_tools()
-    if(PYTHON_ALLOW_EXTENSIONS)
+    if(PYTHON_HAS_EXTENSIONS)
         file(GLOB_RECURSE PYTHON_EXTENSIONS_RELEASE "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/*.pyd")
         file(COPY ${PYTHON_EXTENSIONS_RELEASE} DESTINATION "${CURRENT_PACKAGES_DIR}/bin")
         file(COPY ${PYTHON_EXTENSIONS_RELEASE} DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${PORT}/DLLs")
@@ -257,13 +271,18 @@ else()
         "--without-ensurepip"
         "--with-suffix="
         "--with-system-expat"
-        "--without-readline"
         "--disable-test-modules"
     )
     if(VCPKG_TARGET_IS_OSX)
         list(APPEND OPTIONS "LIBS=-liconv -lintl")
     endif()
 
+    if("readline" IN_LIST FEATURES)
+        list(APPEND OPTIONS "--with-readline")
+    else()
+        list(APPEND OPTIONS "--without-readline")
+    endif()
+
     # The version of the build Python must match the version of the cross compiled host Python.
     # https://docs.python.org/3/using/configure.html#cross-compiling-options
     if(VCPKG_CROSSCOMPILING)
@@ -329,7 +348,7 @@ vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE")
 
 file(READ "${CMAKE_CURRENT_LIST_DIR}/usage" usage)
 if(VCPKG_TARGET_IS_WINDOWS)
-    if(PYTHON_ALLOW_EXTENSIONS)
+    if(PYTHON_HAS_EXTENSIONS)
         file(READ "${CMAKE_CURRENT_LIST_DIR}/usage.win" usage_extra)
     else()
         set(usage_extra "")
@@ -384,10 +403,12 @@ else()
   file(COPY_FILE "${CURRENT_PACKAGES_DIR}/tools/python3/python3.${PYTHON_VERSION_MINOR}" "${CURRENT_PACKAGES_DIR}/tools/python3/python3")
 endif()
 
-configure_file("${CMAKE_CURRENT_LIST_DIR}/vcpkg-port-config.cmake" "${CURRENT_PACKAGES_DIR}/share/python3/vcpkg-port-config.cmake" @ONLY)
+configure_file("${CMAKE_CURRENT_LIST_DIR}/vcpkg-port-config.cmake" "${CURRENT_PACKAGES_DIR}/share/${PORT}/vcpkg-port-config.cmake" @ONLY)
 
 # For testing
 block()
+  include("${CURRENT_PACKAGES_DIR}/share/${PORT}/vcpkg-port-config.cmake")
   set(CURRENT_HOST_INSTALLED_DIR "${CURRENT_PACKAGES_DIR}")
+  set(CURRENT_INSTALLED_DIR "${CURRENT_PACKAGES_DIR}")
   vcpkg_get_vcpkg_installed_python(VCPKG_PYTHON3)
-endblocK()
+endblock()

ports/python3/vcpkg-port-config.cmake

@@ -1,7 +1,9 @@
+include_guard(GLOBAL)
 set(PYTHON3_VERSION "@VERSION@")
 set(PYTHON3_VERSION_MAJOR "@PYTHON_VERSION_MAJOR@")
 set(PYTHON3_VERSION_MINOR "@PYTHON_VERSION_MINOR@")
 set(PYTHON3_INCLUDE "include/python${PYTHON3_VERSION_MAJOR}.${PYTHON3_VERSION_MINOR}")
+set(PYTHON3_HAS_EXTENSIONS "@PYTHON_HAS_EXTENSIONS@")
 set(site_base "")
 if(VCPKG_TARGET_IS_WINDOWS)
   set(site_base "tools/python${PYTHON3_VERSION_MAJOR}/Lib")

ports/python3/vcpkg.json

@@ -1,48 +1,35 @@
 {
   "name": "python3",
   "version": "3.11.8",
-  "port-version": 4,
+  "port-version": 5,
   "description": "The Python programming language",
   "homepage": "https://github.com/python/cpython",
   "license": "Python-2.0",
   "supports": "!uwp & !mingw",
   "dependencies": [
-    {
-      "name": "bzip2",
-      "platform": "!(windows & static)"
-    },
-    "expat",
     {
       "name": "gettext",
       "platform": "osx"
     },
-    {
-      "name": "libffi",
-      "platform": "!(windows & static)"
-    },
     {
       "name": "libiconv",
       "platform": "osx"
     },
-    {
-      "name": "liblzma",
-      "platform": "!(windows & static)"
-    },
     {
       "name": "libuuid",
       "platform": "!osx & !windows"
     },
-    {
-      "name": "openssl",
-      "platform": "!(windows & static)"
-    },
     {
       "name": "python3",
-      "host": true
+      "host": true,
+      "default-features": false
     },
     {
-      "name": "sqlite3",
-      "platform": "!(windows & static)"
+      "name": "python3",
+      "features": [
+        "extensions"
+      ],
+      "platform": "!windows"
     },
     {
       "name": "vcpkg-get-python",
@@ -55,9 +42,49 @@
     },
     "zlib"
   ],
+  "default-features": [
+    {
+      "name": "extensions",
+      "platform": "!(staticcrt & windows)"
+    }
+  ],
   "features": {
     "deprecated-win7-support": {
       "description": "Deprecated support for the Windows 7 platform -- may be removed at any time."
+    },
+    "extensions": {
+      "description": "Allow the build and usage of python extensions. On windows this requires python to be a dynamic library!",
+      "supports": "!(staticcrt & windows)",
+      "dependencies": [
+        {
+          "name": "bzip2",
+          "default-features": false
+        },
+        {
+          "name": "expat",
+          "default-features": false
+        },
+        {
+          "name": "libffi",
+          "default-features": false
+        },
+        {
+          "name": "liblzma",
+          "default-features": false
+        },
+        {
+          "name": "openssl",
+          "default-features": false
+        },
+        {
+          "name": "sqlite3",
+          "default-features": false
+        }
+      ]
+    },
+    "readline": {
+      "description": "Build with readline. Requires system readline to be installed",
+      "supports": "!windows"
     }
   }
 }

ports/vcpkg-get-python/vcpkg-port-config.cmake

@@ -1,9 +1,15 @@
+include_guard(GLOBAL)
+
 function(vcpkg_get_vcpkg_installed_python out_python)
   if(NOT VCPKG_TARGET_IS_WINDOWS)
     # vcpkg installed python on !windows works as normal python would work.
     set(${out_python} "${CURRENT_HOST_INSTALLED_DIR}/tools/python3/python3" PARENT_SCOPE)
     return()
   endif()
+  if(DEFINED CACHE{z_vcpkg_get_vcpkg_installed_python})
+    set(${out_python} "${z_vcpkg_get_vcpkg_installed_python}" PARENT_SCOPE)
+    return()
+  endif()
 
   # On windows python is unable to lookup DLLs, so a manual venv is created
   set(python_home "${CURRENT_HOST_INSTALLED_DIR}/tools/python3")
@@ -42,5 +48,10 @@ if vcpkg_bin_path.is_dir():
 "
 )
 
+ file(COPY "${CURRENT_INSTALLED_DIR}/${PYTHON3_INCLUDE}/" DESTINATION "${python_base}/include")
+ set(suffix "PCBuild/AMD64") # TODO: ask python for the correct suffix.
+ file(COPY "${CURRENT_INSTALLED_DIR}/lib/python${PYTHON3_VERSION_MAJOR}${PYTHON3_VERSION_MINOR}.lib" DESTINATION "${python_base}/${suffix}")
+
  set(${out_python} "${python_base}/Scripts/python.exe" PARENT_SCOPE)
+ set(z_vcpkg_get_vcpkg_installed_python "${python_base}/Scripts/python.exe" CACHE INTERNAL "")
 endfunction()

ports/vcpkg-get-python/vcpkg.json

@@ -1,6 +1,6 @@
 {
   "name": "vcpkg-get-python",
-  "version-date": "2024-06-08",
+  "version-date": "2024-06-22",
   "license": "MIT",
   "supports": "native"
 }

versions/baseline.json

@@ -7258,7 +7258,7 @@
     },
     "python3": {
       "baseline": "3.11.8",
-      "port-version": 4
+      "port-version": 5
     },
     "qca": {
       "baseline": "2.3.7",
@@ -9293,7 +9293,7 @@
       "port-version": 0
     },
     "vcpkg-get-python": {
-      "baseline": "2024-06-08",
+      "baseline": "2024-06-22",
       "port-version": 0
     },
     "vcpkg-get-python-packages": {

versions/p-/python3.json

@@ -1,5 +1,10 @@
 {
   "versions": [
+    {
+      "git-tree": "cd869d19271e7b865248145bb3e14093faa1b687",
+      "version": "3.11.8",
+      "port-version": 5
+    },
     {
       "git-tree": "42da794facada8d85273d1efcc53f1af7e8cb243",
       "version": "3.11.8",

versions/v-/vcpkg-get-python.json

@@ -1,5 +1,10 @@
 {
   "versions": [
+    {
+      "git-tree": "e1993fbd0925b052b31f62967690a2634cb952a2",
+      "version-date": "2024-06-22",
+      "port-version": 0
+    },
     {
       "git-tree": "829be9eea1ad30190a8bb66e4c8a46c9c6d5b0f5",
       "version-date": "2024-06-08",