1From ffe5a060a1b24bedc2aac5d60d6b3b7e40f55adc Mon Sep 17 00:00:00 2001 2From: Vincent Fazio <5265893+vfazio@users.noreply.github.com> 3Date: Wed, 28 Feb 2024 13:55:04 -0600 4Subject: [PATCH] gh-115382: Fix cross compiles when host and target use same 5 SOABI 6 7Previously, when a build was configured to use a host interpreter via 8--with-build-python, the PYTHON_FOR_BUILD config value included a path 9in PYTHONPATH that pointed to the target's built external modules. 10 11For "normal" foreign architecture cross compiles, when loading compiled 12external libraries, the target libraries were processed first due to 13their precedence in sys.path. These libraries were then ruled out due to 14a mismatch in the SOABI so the import mechanism continued searching 15until it found the host's native modules. 16 17However, if the host interpreter and the target python were on the same 18version + SOABI combination, the host interpreter would attempt to load 19the target's external modules due to their precedence in sys.path. 20 21Despite the "match", the target build may have been linked against a 22different libc or may include unsupported instructions so loading or 23executing the target's external modules can lead to crashes. 24 25Now, the path to the target's external modules is no longer defined in 26PYTHONPATH to prevent accidentally loading these foreign modules. 27 28One caveat is that during certain build stages, the target's sysconfig 29module requires higher precedence than the host's version in order to 30accurately query the target build's configuration. 31 32This worked previously due to the target's sysconfig data module having 33precedence over the host's (see above). In order to keep this desired 34behavior, a new environment variable, _PYTHON_SYSCONFIGDATA_PATH, has 35been defined so sysconfig can search this directory for the target's 36sysconfig data. 37 38Signed-off-by: Vincent Fazio <vfazio@gmail.com> 39Upstream-issue: https://github.com/python/cpython/issues/115382 40Upstream: https://github.com/python/cpython/pull/116294 41--- 42 Lib/sysconfig.py | 15 ++++++++++++++- 43 Lib/test/libregrtest/main.py | 1 + 44 Lib/test/pythoninfo.py | 1 + 45 Tools/scripts/run_tests.py | 1 + 46 configure | 2 +- 47 configure.ac | 2 +- 48 6 files changed, 19 insertions(+), 3 deletions(-) 49 50diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py 51index ebe37118274..ba424236e11 100644 52--- a/Lib/sysconfig.py 53+++ b/Lib/sysconfig.py 54@@ -528,7 +528,20 @@ def _init_posix(vars): 55 """Initialize the module as appropriate for POSIX systems.""" 56 # _sysconfigdata is generated at build time, see _generate_posix_vars() 57 name = _get_sysconfigdata_name() 58- _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) 59+ 60+ # For cross builds, the path to the target's sysconfigdata must be specified 61+ # so it can be imported. It cannot be in PYTHONPATH, as foreign modules in 62+ # sys.path can cause crashes when loaded by the host interpreter. 63+ # Rely on truthiness as a valueless env variable is still an empty string. 64+ # See OS X note in _generate_posix_vars re _sysconfigdata. 65+ if (path := os.environ.get('_PYTHON_SYSCONFIGDATA_PATH')): 66+ from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES 67+ from importlib.util import module_from_spec 68+ spec = FileFinder(path, (SourceFileLoader, SOURCE_SUFFIXES)).find_spec(name) 69+ _temp = module_from_spec(spec) 70+ spec.loader.exec_module(_temp) 71+ else: 72+ _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) 73 build_time_vars = _temp.build_time_vars 74 vars.update(build_time_vars) 75 76diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py 77index a9725fa9673..121e2e73938 100644 78--- a/Lib/test/libregrtest/main.py 79+++ b/Lib/test/libregrtest/main.py 80@@ -519,6 +519,7 @@ def _add_cross_compile_opts(self, regrtest_opts): 81 '_PYTHON_PROJECT_BASE', 82 '_PYTHON_HOST_PLATFORM', 83 '_PYTHON_SYSCONFIGDATA_NAME', 84+ "_PYTHON_SYSCONFIGDATA_PATH", 85 'PYTHONPATH' 86 } 87 old_environ = os.environ 88diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py 89index 74ebb5e5b8a..fa7fbca34e1 100644 90--- a/Lib/test/pythoninfo.py 91+++ b/Lib/test/pythoninfo.py 92@@ -326,6 +326,7 @@ def format_groups(groups): 93 "_PYTHON_HOST_PLATFORM", 94 "_PYTHON_PROJECT_BASE", 95 "_PYTHON_SYSCONFIGDATA_NAME", 96+ "_PYTHON_SYSCONFIGDATA_PATH", 97 "__PYVENV_LAUNCHER__", 98 99 # Sanitizer options 100diff --git a/Tools/scripts/run_tests.py b/Tools/scripts/run_tests.py 101index 445a34ae3e8..4077a834245 100644 102--- a/Tools/scripts/run_tests.py 103+++ b/Tools/scripts/run_tests.py 104@@ -42,6 +42,7 @@ def main(regrtest_args): 105 '_PYTHON_PROJECT_BASE', 106 '_PYTHON_HOST_PLATFORM', 107 '_PYTHON_SYSCONFIGDATA_NAME', 108+ "_PYTHON_SYSCONFIGDATA_PATH", 109 'PYTHONPATH' 110 } 111 environ = { 112diff --git a/configure b/configure 113index cb3db60f9c2..5fc0cffa977 100755 114--- a/configure 115+++ b/configure 116@@ -3262,7 +3262,7 @@ fi 117 fi 118 ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python 119 PYTHON_FOR_FREEZE="$with_build_python" 120- PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$with_build_python 121+ PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) _PYTHON_SYSCONFIGDATA_PATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`) '$with_build_python 122 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_build_python" >&5 123 $as_echo "$with_build_python" >&6; } 124 125diff --git a/configure.ac b/configure.ac 126index 852cbaa6e4c..5ca316a948b 100644 127--- a/configure.ac 128+++ b/configure.ac 129@@ -162,7 +162,7 @@ AC_ARG_WITH( 130 dnl Build Python interpreter is used for regeneration and freezing. 131 ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python 132 PYTHON_FOR_FREEZE="$with_build_python" 133- PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$with_build_python 134+ PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) _PYTHON_SYSCONFIGDATA_PATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`) '$with_build_python 135 AC_MSG_RESULT([$with_build_python]) 136 ], [ 137 AS_VAR_IF([cross_compiling], [yes], 138-- 1392.44.0 140 141