From cf6f23b0e3cdef33f23967cf954a2ca4d1fa6528 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 4 Oct 2023 22:50:29 +0000 Subject: [PATCH] gh-88402: Add new sysconfig variables on Windows (GH-110049) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Filipe Laíns --- Include/internal/pycore_importdl.h | 57 +++++++++++ Lib/sysconfig.py | 18 ++-- Lib/test/test_sysconfig.py | 24 ++++- ...3-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst | 2 + Modules/Setup | 1 + Modules/Setup.bootstrap.in | 1 + Modules/_sysconfig.c | 98 +++++++++++++++++++ Modules/clinic/_sysconfig.c.h | 22 +++++ PC/config.c | 2 + PCbuild/pythoncore.vcxproj | 3 +- PCbuild/pythoncore.vcxproj.filters | 9 +- Python/dynload_hpux.c | 2 +- Python/dynload_shlib.c | 2 +- Python/dynload_stub.c | 2 +- Python/dynload_win.c | 22 +---- Python/import.c | 2 +- Python/importdl.c | 2 +- Python/importdl.h | 29 ------ Python/stdlib_module_names.h | 1 + Tools/c-analyzer/cpython/ignored.tsv | 2 +- 20 files changed, 231 insertions(+), 70 deletions(-) create mode 100644 Include/internal/pycore_importdl.h create mode 100644 Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst create mode 100644 Modules/_sysconfig.c create mode 100644 Modules/clinic/_sysconfig.c.h delete mode 100644 Python/importdl.h diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h new file mode 100644 index 00000000000..dee64241c76 --- /dev/null +++ b/Include/internal/pycore_importdl.h @@ -0,0 +1,57 @@ +#ifndef Py_INTERNAL_IMPORTDL_H +#define Py_INTERNAL_IMPORTDL_H + +#include "patchlevel.h" // PY_MAJOR_VERSION + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +extern const char *_PyImport_DynLoadFiletab[]; + +extern PyObject *_PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *); + +typedef PyObject *(*PyModInitFunction)(void); + +/* Max length of module suffix searched for -- accommodates "module.slb" */ +#define MAXSUFFIXSIZE 12 + +#ifdef MS_WINDOWS +#include +typedef FARPROC dl_funcptr; + +#ifdef _DEBUG +# define PYD_DEBUG_SUFFIX "_d" +#else +# define PYD_DEBUG_SUFFIX "" +#endif + +#ifdef Py_NOGIL +# define PYD_THREADING_TAG "t" +#else +# define PYD_THREADING_TAG "" +#endif + +#ifdef PYD_PLATFORM_TAG +# define PYD_SOABI "cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG "-" PYD_PLATFORM_TAG +#else +# define PYD_SOABI "cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG +#endif + +#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX "." PYD_SOABI ".pyd" +#define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" + +#else +typedef void (*dl_funcptr)(void); +#endif + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_IMPORTDL_H */ diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index a8b5c5f7dfb..edfe451a86b 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -544,16 +544,20 @@ def _init_posix(vars): def _init_non_posix(vars): """Initialize the module as appropriate for NT""" # set basic install directories - import _imp + import _winapi + import _sysconfig vars['LIBDEST'] = get_path('stdlib') vars['BINLIBDEST'] = get_path('platstdlib') vars['INCLUDEPY'] = get_path('include') - try: - # GH-99201: _imp.extension_suffixes may be empty when - # HAVE_DYNAMIC_LOADING is not set. In this case, don't set EXT_SUFFIX. - vars['EXT_SUFFIX'] = _imp.extension_suffixes()[0] - except IndexError: - pass + + # Add EXT_SUFFIX, SOABI, and Py_NOGIL + vars.update(_sysconfig.config_vars()) + + vars['LIBDIR'] = _safe_realpath(os.path.join(get_config_var('installed_base'), 'libs')) + if hasattr(sys, 'dllhandle'): + dllhandle = _winapi.GetModuleFileName(sys.dllhandle) + vars['LIBRARY'] = os.path.basename(_safe_realpath(dllhandle)) + vars['LDLIBRARY'] = vars['LIBRARY'] vars['EXE'] = '.exe' vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index b6dbf3d52cb..a077ac5349f 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -17,7 +17,9 @@ from sysconfig import (get_paths, get_platform, get_config_vars, get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, _expand_vars, _get_preferred_schemes, _main) +import _imp import _osx_support +import _sysconfig HAS_USER_BASE = sysconfig._HAS_USER_BASE @@ -394,6 +396,24 @@ class TestSysConfig(unittest.TestCase): self.assertIn(ldflags, ldshared) + @unittest.skipIf(not _imp.extension_suffixes(), "stub loader has no suffixes") + def test_soabi(self): + soabi = sysconfig.get_config_var('SOABI') + self.assertIn(soabi, _imp.extension_suffixes()[0]) + + def test_library(self): + library = sysconfig.get_config_var('LIBRARY') + ldlibrary = sysconfig.get_config_var('LDLIBRARY') + major, minor = sys.version_info[:2] + if sys.platform == 'win32': + self.assertTrue(library.startswith(f'python{major}{minor}')) + self.assertTrue(library.endswith('.dll')) + self.assertEqual(library, ldlibrary) + else: + self.assertTrue(library.startswith(f'libpython{major}.{minor}')) + self.assertTrue(library.endswith('.a')) + self.assertTrue(ldlibrary.startswith(f'libpython{major}.{minor}')) + @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX") @requires_subprocess() def test_platform_in_subprocess(self): @@ -472,10 +492,8 @@ class TestSysConfig(unittest.TestCase): @unittest.skipIf(sysconfig.get_config_var('EXT_SUFFIX') is None, 'EXT_SUFFIX required for this test') + @unittest.skipIf(not _imp.extension_suffixes(), "stub loader has no suffixes") def test_EXT_SUFFIX_in_vars(self): - import _imp - if not _imp.extension_suffixes(): - self.skipTest("stub loader has no suffixes") vars = sysconfig.get_config_vars() self.assertEqual(vars['EXT_SUFFIX'], _imp.extension_suffixes()[0]) diff --git a/Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst b/Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst new file mode 100644 index 00000000000..80ec65081c0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst @@ -0,0 +1,2 @@ +Add new variables to :py:meth:`sysconfig.get_config_vars` on Windows: +``LIBRARY``, ``LDLIBRARY``, ``LIBDIR``, ``SOABI``, and ``Py_NOGIL``. diff --git a/Modules/Setup b/Modules/Setup index 8676f9ddce4..1367f0ef4fa 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -155,6 +155,7 @@ PYTHONPATH=$(COREPYTHONPATH) #math mathmodule.c #mmap mmapmodule.c #select selectmodule.c +#_sysconfig _sysconfig.c # XML #_elementtree _elementtree.c diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index 8ef0f203a82..cd12c1bd0df 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -19,6 +19,7 @@ errno errnomodule.c _io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c itertools itertoolsmodule.c _sre _sre/sre.c +_sysconfig _sysconfig.c _thread _threadmodule.c time timemodule.c _typing _typingmodule.c diff --git a/Modules/_sysconfig.c b/Modules/_sysconfig.c new file mode 100644 index 00000000000..6f1cc16b584 --- /dev/null +++ b/Modules/_sysconfig.c @@ -0,0 +1,98 @@ +// _sysconfig provides data for the Python sysconfig module + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#include "Python.h" + +#include "pycore_importdl.h" // _PyImport_DynLoadFiletab +#include "pycore_long.h" // _PyLong_GetZero, _PyLong_GetOne + + +/*[clinic input] +module _sysconfig +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=0a7c02d3e212ac97]*/ + +#include "clinic/_sysconfig.c.h" + +#ifdef MS_WINDOWS +static int +add_string_value(PyObject *dict, const char *key, const char *str_value) +{ + PyObject *value = PyUnicode_FromString(str_value); + if (value == NULL) { + return -1; + } + int err = PyDict_SetItemString(dict, key, value); + Py_DECREF(value); + return err; +} +#endif + +/*[clinic input] +_sysconfig.config_vars + +Returns a dictionary containing build variables intended to be exposed by sysconfig. +[clinic start generated code]*/ + +static PyObject * +_sysconfig_config_vars_impl(PyObject *module) +/*[clinic end generated code: output=9c41cdee63ea9487 input=391ff42f3af57d01]*/ +{ + PyObject *config = PyDict_New(); + if (config == NULL) { + return NULL; + } + +#ifdef MS_WINDOWS + if (add_string_value(config, "EXT_SUFFIX", PYD_TAGGED_SUFFIX) < 0) { + Py_DECREF(config); + return NULL; + } + if (add_string_value(config, "SOABI", PYD_SOABI) < 0) { + Py_DECREF(config); + return NULL; + } +#endif + +#ifdef Py_NOGIL + PyObject *py_nogil = _PyLong_GetOne(); +#else + PyObject *py_nogil = _PyLong_GetZero(); +#endif + if (PyDict_SetItemString(config, "Py_NOGIL", py_nogil) < 0) { + Py_DECREF(config); + return NULL; + } + + return config; +} + +PyDoc_STRVAR(sysconfig__doc__, +"A helper for the sysconfig module."); + +static struct PyMethodDef sysconfig_methods[] = { + _SYSCONFIG_CONFIG_VARS_METHODDEF + {NULL, NULL} +}; + +static PyModuleDef_Slot sysconfig_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef sysconfig_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_sysconfig", + .m_doc = sysconfig__doc__, + .m_methods = sysconfig_methods, + .m_slots = sysconfig_slots, +}; + +PyMODINIT_FUNC +PyInit__sysconfig(void) +{ + return PyModuleDef_Init(&sysconfig_module); +} diff --git a/Modules/clinic/_sysconfig.c.h b/Modules/clinic/_sysconfig.c.h new file mode 100644 index 00000000000..eb3d396298b --- /dev/null +++ b/Modules/clinic/_sysconfig.c.h @@ -0,0 +1,22 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(_sysconfig_config_vars__doc__, +"config_vars($module, /)\n" +"--\n" +"\n" +"Returns a dictionary containing build variables intended to be exposed by sysconfig."); + +#define _SYSCONFIG_CONFIG_VARS_METHODDEF \ + {"config_vars", (PyCFunction)_sysconfig_config_vars, METH_NOARGS, _sysconfig_config_vars__doc__}, + +static PyObject * +_sysconfig_config_vars_impl(PyObject *module); + +static PyObject * +_sysconfig_config_vars(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _sysconfig_config_vars_impl(module); +} +/*[clinic end generated code: output=25d395cf02eced1f input=a9049054013a1b77]*/ diff --git a/PC/config.c b/PC/config.c index 88f69758aac..da2bde64096 100644 --- a/PC/config.c +++ b/PC/config.c @@ -22,6 +22,7 @@ extern PyObject* PyInit__sha1(void); extern PyObject* PyInit__sha2(void); extern PyObject* PyInit__sha3(void); extern PyObject* PyInit__statistics(void); +extern PyObject* PyInit__sysconfig(void); extern PyObject* PyInit__typing(void); extern PyObject* PyInit__blake2(void); extern PyObject* PyInit_time(void); @@ -102,6 +103,7 @@ struct _inittab _PyImport_Inittab[] = { {"_sha2", PyInit__sha2}, {"_sha3", PyInit__sha3}, {"_blake2", PyInit__blake2}, + {"_sysconfig", PyInit__sysconfig}, {"time", PyInit_time}, {"_thread", PyInit__thread}, {"_tokenize", PyInit__tokenize}, diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 1ec106777db..43a79fd5938 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -240,6 +240,7 @@ + @@ -367,7 +368,6 @@ - @@ -438,6 +438,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index f381120c9b0..59159ed6099 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -300,9 +300,6 @@ PC - - Python - Python @@ -633,6 +630,9 @@ Include\internal + + Include\internal + Include\internal @@ -959,6 +959,9 @@ Modules + + Modules + Modules diff --git a/Python/dynload_hpux.c b/Python/dynload_hpux.c index a53373038ed..1c44722ff9a 100644 --- a/Python/dynload_hpux.c +++ b/Python/dynload_hpux.c @@ -5,7 +5,7 @@ #include #include "Python.h" -#include "importdl.h" +#include "pycore_importdl.h" #if defined(__hp9000s300) #define FUNCNAME_PATTERN "_%.20s_%.200s" diff --git a/Python/dynload_shlib.c b/Python/dynload_shlib.c index 6761bba4579..5a37a83805b 100644 --- a/Python/dynload_shlib.c +++ b/Python/dynload_shlib.c @@ -4,7 +4,7 @@ #include "Python.h" #include "pycore_interp.h" // _PyInterpreterState.dlopenflags #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "importdl.h" +#include "pycore_importdl.h" #include #include diff --git a/Python/dynload_stub.c b/Python/dynload_stub.c index 59160483caa..11f7e5f643f 100644 --- a/Python/dynload_stub.c +++ b/Python/dynload_stub.c @@ -3,7 +3,7 @@ not present. */ #include "Python.h" -#include "importdl.h" +#include "pycore_importdl.h" const char *_PyImport_DynLoadFiletab[] = {NULL}; diff --git a/Python/dynload_win.c b/Python/dynload_win.c index fcb3cb74404..a0ac31c80a5 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -5,30 +5,10 @@ #include "pycore_fileutils.h" // _Py_add_relfile() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "importdl.h" // dl_funcptr +#include "pycore_importdl.h" // dl_funcptr #include "patchlevel.h" // PY_MAJOR_VERSION #include -#ifdef _DEBUG -#define PYD_DEBUG_SUFFIX "_d" -#else -#define PYD_DEBUG_SUFFIX "" -#endif - -#ifdef Py_NOGIL -# define PYD_THREADING_TAG "t" -#else -# define PYD_THREADING_TAG "" -#endif - -#ifdef PYD_PLATFORM_TAG -#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG "-" PYD_PLATFORM_TAG ".pyd" -#else -#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG ".pyd" -#endif - -#define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" - const char *_PyImport_DynLoadFiletab[] = { PYD_TAGGED_SUFFIX, PYD_UNTAGGED_SUFFIX, diff --git a/Python/import.c b/Python/import.c index 5636968ed9e..cafdd834502 100644 --- a/Python/import.c +++ b/Python/import.c @@ -17,7 +17,7 @@ #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include "marshal.h" // PyMarshal_ReadObjectFromString() -#include "importdl.h" // _PyImport_DynLoadFiletab +#include "pycore_importdl.h" // _PyImport_DynLoadFiletab #include "pydtrace.h" // PyDTrace_IMPORT_FIND_LOAD_START_ENABLED() #include // bool diff --git a/Python/importdl.c b/Python/importdl.c index 9ab0a5ad33a..7dfd301d77e 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -15,7 +15,7 @@ */ #ifdef HAVE_DYNAMIC_LOADING -#include "importdl.h" +#include "pycore_importdl.h" #ifdef MS_WINDOWS extern dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, diff --git a/Python/importdl.h b/Python/importdl.h deleted file mode 100644 index 9171adc2770..00000000000 --- a/Python/importdl.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef Py_IMPORTDL_H -#define Py_IMPORTDL_H - -#ifdef __cplusplus -extern "C" { -#endif - - -extern const char *_PyImport_DynLoadFiletab[]; - -extern PyObject *_PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *); - -typedef PyObject *(*PyModInitFunction)(void); - -/* Max length of module suffix searched for -- accommodates "module.slb" */ -#define MAXSUFFIXSIZE 12 - -#ifdef MS_WINDOWS -#include -typedef FARPROC dl_funcptr; -#else -typedef void (*dl_funcptr)(void); -#endif - - -#ifdef __cplusplus -} -#endif -#endif /* !Py_IMPORTDL_H */ diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 13b1764f088..701bfc35cc8 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -77,6 +77,7 @@ static const char* _Py_stdlib_module_names[] = { "_strptime", "_struct", "_symtable", +"_sysconfig", "_thread", "_threading_local", "_tkinter", diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index c6c69a3e222..f9911643332 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -607,6 +607,7 @@ Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput AST_LITERAL_EVAL_INITIALIZED # XXX Fix the analyzer. ## forward/extern references +Include/internal/pycore_importdl.h - _PyImport_DynLoadFiletab - Include/py_curses.h - PyCurses_API - Include/pydecimal.h - _decimal_api - Modules/_blake2/blake2module.c - blake2b_type_spec - @@ -668,7 +669,6 @@ Objects/object.c - _PyLineIterator - Objects/object.c - _PyPositionsIterator - Python/perf_trampoline.c - _Py_trampoline_func_start - Python/perf_trampoline.c - _Py_trampoline_func_end - -Python/importdl.h - _PyImport_DynLoadFiletab - Modules/expat/xmlrole.c - prolog0 - Modules/expat/xmlrole.c - prolog1 - Modules/expat/xmlrole.c - prolog2 -