From e8c91d90ba8fab410a27fad4f709cc73f6ffcbf4 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 11 Jul 2024 16:21:37 -0400 Subject: [PATCH] gh-121103: Put free-threaded libraries in `lib/python3.14t` (#121293) On POSIX systems, excluding macOS framework installs, the lib directory for the free-threaded build now includes a "t" suffix to avoid conflicts with a co-located default build installation. --- Lib/site.py | 12 +++++-- Lib/sysconfig/__init__.py | 33 +++++++++++-------- Lib/test/test_embed.py | 14 ++++---- Lib/test/test_getpath.py | 1 + Lib/test/test_site.py | 4 +-- Lib/test/test_sysconfig.py | 2 +- Lib/test/test_venv.py | 5 +-- Makefile.pre.in | 3 +- ...-07-02-20-16-09.gh-issue-121103.TMef9j.rst | 3 ++ Modules/getpath.c | 5 +++ Modules/getpath.py | 8 +++-- configure | 14 +++++--- configure.ac | 13 +++++--- 13 files changed, 77 insertions(+), 40 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-07-02-20-16-09.gh-issue-121103.TMef9j.rst diff --git a/Lib/site.py b/Lib/site.py index daa56e15894..460269433f0 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -312,6 +312,10 @@ def _getuserbase(): # Same to sysconfig.get_path('purelib', os.name+'_user') def _get_path(userbase): version = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' implementation = _get_implementation() implementation_lower = implementation.lower() @@ -322,7 +326,7 @@ def _get_path(userbase): if sys.platform == 'darwin' and sys._framework: return f'{userbase}/lib/{implementation_lower}/site-packages' - return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages' + return f'{userbase}/lib/python{version[0]}.{version[1]}{abi_thread}/site-packages' def getuserbase(): @@ -390,6 +394,10 @@ def getsitepackages(prefixes=None): implementation = _get_implementation().lower() ver = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' if os.sep == '/': libdirs = [sys.platlibdir] if sys.platlibdir != "lib": @@ -397,7 +405,7 @@ def getsitepackages(prefixes=None): for libdir in libdirs: path = os.path.join(prefix, libdir, - f"{implementation}{ver[0]}.{ver[1]}", + f"{implementation}{ver[0]}.{ver[1]}{abi_thread}", "site-packages") sitepackages.append(path) else: diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 98a14e5d3a3..83e057c177f 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -27,10 +27,10 @@ _ALWAYS_STR = { _INSTALL_SCHEMES = { 'posix_prefix': { - 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}', - 'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', 'include': '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': @@ -77,10 +77,10 @@ _INSTALL_SCHEMES = { # Downstream distributors who patch posix_prefix/nt scheme are encouraged to # leave the following schemes unchanged 'posix_venv': { - 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}', - 'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', 'include': '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': @@ -148,11 +148,11 @@ if _HAS_USER_BASE: 'data': '{userbase}', }, 'posix_user': { - 'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}', - 'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}', - 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages', - 'include': '{userbase}/include/{implementation_lower}{py_version_short}', + 'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}{abi_thread}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, @@ -487,6 +487,9 @@ def _init_config_vars(): # the init-function. _CONFIG_VARS['userbase'] = _getuserbase() + # e.g., 't' for free-threaded or '' for default build + _CONFIG_VARS['abi_thread'] = 't' if _CONFIG_VARS.get('Py_GIL_DISABLED') else '' + # Always convert srcdir to an absolute path srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) if os.name == 'posix': @@ -655,6 +658,10 @@ def get_python_version(): return _PY_VERSION_SHORT +def _get_python_version_abi(): + return _PY_VERSION_SHORT + get_config_var("abi_thread") + + def expand_makefile_vars(s, vars): """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in 'string' according to 'vars' (a dictionary mapping variable names to diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 634513ec7a5..30dab1fbaa4 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -48,6 +48,8 @@ API_ISOLATED = 3 INIT_LOOPS = 4 MAX_HASH_SEED = 4294967295 +ABI_THREAD = 't' if sysconfig.get_config_var('Py_GIL_DISABLED') else '' + # If we are running from a build dir, but the stdlib has been installed, # some tests need to expect different results. @@ -1285,11 +1287,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): ver = sys.version_info return [ os.path.join(prefix, sys.platlibdir, - f'python{ver.major}{ver.minor}.zip'), + f'python{ver.major}{ver.minor}{ABI_THREAD}.zip'), os.path.join(prefix, sys.platlibdir, - f'python{ver.major}.{ver.minor}'), + f'python{ver.major}.{ver.minor}{ABI_THREAD}'), os.path.join(exec_prefix, sys.platlibdir, - f'python{ver.major}.{ver.minor}', 'lib-dynload'), + f'python{ver.major}.{ver.minor}{ABI_THREAD}', 'lib-dynload'), ] @contextlib.contextmanager @@ -1343,7 +1345,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] else: version = f'{sys.version_info.major}.{sys.version_info.minor}' - stdlib = os.path.join(home, sys.platlibdir, f'python{version}') + stdlib = os.path.join(home, sys.platlibdir, f'python{version}{ABI_THREAD}') expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) config = { @@ -1384,7 +1386,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] else: version = f'{sys.version_info.major}.{sys.version_info.minor}' - stdlib = os.path.join(home, sys.platlibdir, f'python{version}') + stdlib = os.path.join(home, sys.platlibdir, f'python{version}{ABI_THREAD}') expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) config = { @@ -1515,7 +1517,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): if not MS_WINDOWS: lib_dynload = os.path.join(pyvenv_home, sys.platlibdir, - f'python{ver.major}.{ver.minor}', + f'python{ver.major}.{ver.minor}{ABI_THREAD}', 'lib-dynload') os.makedirs(lib_dynload) else: diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 2f7aa69efc1..6c86c3d1c8c 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -844,6 +844,7 @@ DEFAULT_NAMESPACE = dict( PYDEBUGEXT="", VERSION_MAJOR=9, # fixed version number for ease VERSION_MINOR=8, # of testing + ABI_THREAD="", PYWINVER=None, EXE_SUFFIX=None, diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index bcdc232c712..035913cdd05 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -328,13 +328,13 @@ class HelperFunctionsTests(unittest.TestCase): if sys.platlibdir != "lib": self.assertEqual(len(dirs), 2) wanted = os.path.join('xoxo', sys.platlibdir, - 'python%d.%d' % sys.version_info[:2], + f'python{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[0], wanted) else: self.assertEqual(len(dirs), 1) wanted = os.path.join('xoxo', 'lib', - 'python%d.%d' % sys.version_info[:2], + f'python{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[-1], wanted) else: diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 9233304c6a5..37cee927686 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -157,7 +157,7 @@ class TestSysConfig(unittest.TestCase): binpath = 'bin' incpath = 'include' libpath = os.path.join('lib', - 'python%d.%d' % sys.version_info[:2], + f'python{sysconfig._get_python_version_abi()}', 'site-packages') # Resolve the paths in an imaginary venv/ directory diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 1769ed61b94..2b7d297f011 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -75,7 +75,7 @@ class BaseTest(unittest.TestCase): self.include = 'Include' else: self.bindir = 'bin' - self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) + self.lib = ('lib', f'python{sysconfig._get_python_version_abi()}') self.include = 'include' executable = sys._base_executable self.exe = os.path.split(executable)[-1] @@ -593,7 +593,8 @@ class BasicTest(BaseTest): libdir = os.path.join(non_installed_dir, platlibdir, self.lib[1]) os.makedirs(libdir) landmark = os.path.join(libdir, "os.py") - stdlib_zip = "python%d%d.zip" % sys.version_info[:2] + abi_thread = "t" if sysconfig.get_config_var("Py_GIL_DISABLED") else "" + stdlib_zip = f"python{sys.version_info.major}{sys.version_info.minor}{abi_thread}" zip_landmark = os.path.join(non_installed_dir, platlibdir, stdlib_zip) diff --git a/Makefile.pre.in b/Makefile.pre.in index 0bece8717ef..d380c422714 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -41,6 +41,7 @@ AR= @AR@ READELF= @READELF@ SOABI= @SOABI@ ABIFLAGS= @ABIFLAGS@ +ABI_THREAD= @ABI_THREAD@ LDVERSION= @LDVERSION@ MODULE_LDFLAGS=@MODULE_LDFLAGS@ GITVERSION= @GITVERSION@ @@ -158,7 +159,7 @@ WHEEL_PKG_DIR= @WHEEL_PKG_DIR@ # Detailed destination directories BINLIBDEST= @BINLIBDEST@ -LIBDEST= $(SCRIPTDIR)/python$(VERSION) +LIBDEST= $(SCRIPTDIR)/python$(VERSION)$(ABI_THREAD) INCLUDEPY= $(INCLUDEDIR)/python$(LDVERSION) CONFINCLUDEPY= $(CONFINCLUDEDIR)/python$(LDVERSION) diff --git a/Misc/NEWS.d/next/Build/2024-07-02-20-16-09.gh-issue-121103.TMef9j.rst b/Misc/NEWS.d/next/Build/2024-07-02-20-16-09.gh-issue-121103.TMef9j.rst new file mode 100644 index 00000000000..4bc8c6de0b7 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-07-02-20-16-09.gh-issue-121103.TMef9j.rst @@ -0,0 +1,3 @@ +On POSIX systems, excluding macOS framework installs, the lib directory +for the free-threaded build now includes a "t" suffix to avoid conflicts +with a co-located default build installation. diff --git a/Modules/getpath.c b/Modules/getpath.c index abed1390282..d0128b20fae 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -951,6 +951,11 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) !wchar_to_dict(dict, "executable_dir", NULL) || !wchar_to_dict(dict, "py_setpath", _PyPathConfig_GetGlobalModuleSearchPath()) || !funcs_to_dict(dict, config->pathconfig_warnings) || +#ifdef Py_GIL_DISABLED + !decode_to_dict(dict, "ABI_THREAD", "t") || +#else + !decode_to_dict(dict, "ABI_THREAD", "") || +#endif #ifndef MS_WINDOWS PyDict_SetItemString(dict, "winreg", Py_None) < 0 || #endif diff --git a/Modules/getpath.py b/Modules/getpath.py index bc7053224aa..1f1bfcb4f64 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -40,6 +40,7 @@ # EXE_SUFFIX -- [in, opt] '.exe' on Windows/Cygwin/similar # VERSION_MAJOR -- [in] sys.version_info.major # VERSION_MINOR -- [in] sys.version_info.minor +# ABI_THREAD -- [in] either 't' for free-threaded builds or '' # PYWINVER -- [in] the Windows platform-specific version (e.g. 3.8-32) # ** Values read from the environment ** @@ -172,17 +173,18 @@ # ****************************************************************************** platlibdir = config.get('platlibdir') or PLATLIBDIR +ABI_THREAD = ABI_THREAD or '' if os_name == 'posix' or os_name == 'darwin': BUILDDIR_TXT = 'pybuilddir.txt' BUILD_LANDMARK = 'Modules/Setup.local' DEFAULT_PROGRAM_NAME = f'python{VERSION_MAJOR}' - STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}' + STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}{ABI_THREAD}' STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}/os.py', f'{STDLIB_SUBDIR}/os.pyc'] - PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}/lib-dynload' + PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}{ABI_THREAD}/lib-dynload' BUILDSTDLIB_LANDMARKS = ['Lib/os.py'] VENV_LANDMARK = 'pyvenv.cfg' - ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}.zip' + ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}{ABI_THREAD}.zip' DELIM = ':' SEP = '/' diff --git a/configure b/configure index bbfa805883c..19786f18e61 100755 --- a/configure +++ b/configure @@ -928,6 +928,7 @@ DEF_MAKE_RULE DEF_MAKE_ALL_RULE JIT_STENCILS_H REGEN_JIT_COMMAND +ABI_THREAD ABIFLAGS LN MKDIR_P @@ -8160,7 +8161,9 @@ fi # For calculating the .so ABI tag. + ABIFLAGS="" +ABI_THREAD="" # Check for --disable-gil # --disable-gil @@ -8190,6 +8193,7 @@ printf "%s\n" "#define Py_GIL_DISABLED 1" >>confdefs.h # Add "t" for "threaded" ABIFLAGS="${ABIFLAGS}t" + ABI_THREAD="t" fi # Check for --with-pydebug @@ -24738,11 +24742,11 @@ fi -BINLIBDEST='$(LIBDIR)/python$(VERSION)' +BINLIBDEST='$(LIBDIR)/python$(VERSION)$(ABI_THREAD)' # Check for --with-platlibdir -# /usr/$LIDIRNAME/python$VERSION +# /usr/$PLATLIBDIR/python$(VERSION)$(ABI_THREAD) PLATLIBDIR="lib" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-platlibdir" >&5 @@ -24761,7 +24765,7 @@ then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } PLATLIBDIR="$withval" - BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)' + BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)$(ABI_THREAD)' else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } @@ -24775,9 +24779,9 @@ fi if test x$PLATFORM_TRIPLET = x; then - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}" else - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}-${PLATFORM_TRIPLET}" fi diff --git a/configure.ac b/configure.ac index 87c4df20818..df146cc9cf0 100644 --- a/configure.ac +++ b/configure.ac @@ -1739,7 +1739,9 @@ fi # For calculating the .so ABI tag. AC_SUBST([ABIFLAGS]) +AC_SUBST([ABI_THREAD]) ABIFLAGS="" +ABI_THREAD="" # Check for --disable-gil # --disable-gil @@ -1756,6 +1758,7 @@ then [Define if you want to disable the GIL]) # Add "t" for "threaded" ABIFLAGS="${ABIFLAGS}t" + ABI_THREAD="t" fi # Check for --with-pydebug @@ -6234,11 +6237,11 @@ fi AC_SUBST([BINLIBDEST]) -BINLIBDEST='$(LIBDIR)/python$(VERSION)' +BINLIBDEST='$(LIBDIR)/python$(VERSION)$(ABI_THREAD)' # Check for --with-platlibdir -# /usr/$LIDIRNAME/python$VERSION +# /usr/$PLATLIBDIR/python$(VERSION)$(ABI_THREAD) AC_SUBST([PLATLIBDIR]) PLATLIBDIR="lib" AC_MSG_CHECKING([for --with-platlibdir]) @@ -6257,7 +6260,7 @@ if test -n "$withval" -a "$withval" != yes -a "$withval" != no then AC_MSG_RESULT([yes]) PLATLIBDIR="$withval" - BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)' + BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)$(ABI_THREAD)' else AC_MSG_RESULT([no]) fi], @@ -6267,9 +6270,9 @@ fi], dnl define LIBPL after ABIFLAGS and LDVERSION is defined. AC_SUBST([PY_ENABLE_SHARED]) if test x$PLATFORM_TRIPLET = x; then - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}" else - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}-${PLATFORM_TRIPLET}" fi AC_SUBST([LIBPL])