diff --git a/.gitignore b/.gitignore index 2d380a441d2..8872e9d5508 100644 --- a/.gitignore +++ b/.gitignore @@ -69,7 +69,7 @@ Lib/test/data/* /_bootstrap_python /Makefile /Makefile.pre -iOSTestbed.* +/iOSTestbed.* iOS/Frameworks/ iOS/Resources/Info.plist iOS/testbed/build diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index d92bb2f8e5c..b58ef359378 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1241,6 +1241,69 @@ find and load modules. and how the module's :attr:`__file__` is populated. +.. class:: AppleFrameworkLoader(name, path) + + A specialization of :class:`importlib.machinery.ExtensionFileLoader` that + is able to load extension modules in Framework format. + + For compatibility with the iOS App Store, *all* binary modules in an iOS app + must be dynamic libraries, contained in a framework with appropriate + metadata, stored in the ``Frameworks`` folder of the packaged app. There can + be only a single binary per framework, and there can be no executable binary + material outside the Frameworks folder. + + To accomodate this requirement, when running on iOS, extension module + binaries are *not* packaged as ``.so`` files on ``sys.path``, but as + individual standalone frameworks. To discover those frameworks, this loader + is be registered against the ``.fwork`` file extension, with a ``.fwork`` + file acting as a placeholder in the original location of the binary on + ``sys.path``. The ``.fwork`` file contains the path of the actual binary in + the ``Frameworks`` folder, relative to the app bundle. To allow for + resolving a framework-packaged binary back to the original location, the + framework is expected to contain a ``.origin`` file that contains the + location of the ``.fwork`` file, relative to the app bundle. + + For example, consider the case of an import ``from foo.bar import _whiz``, + where ``_whiz`` is implemented with the binary module + ``sources/foo/bar/_whiz.abi3.so``, with ``sources`` being the location + registered on ``sys.path``, relative to the application bundle. This module + *must* be distributed as + ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz`` (creating the framework + name from the full import path of the module), with an ``Info.plist`` file + in the ``.framework`` directory identifying the binary as a framework. The + ``foo.bar._whiz`` module would be represented in the original location with + a ``sources/foo/bar/_whiz.abi3.fwork`` marker file, containing the path + ``Frameworks/foo.bar._whiz/foo.bar._whiz``. The framework would also contain + ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz.origin``, containing the + path to the ``.fwork`` file. + + When a module is loaded with this loader, the ``__file__`` for the module + will report as the location of the ``.fwork`` file. This allows code to use + the ``__file__`` of a module as an anchor for file system traveral. + However, the spec origin will reference the location of the *actual* binary + in the ``.framework`` folder. + + The Xcode project building the app is responsible for converting any ``.so`` + files from wherever they exist in the ``PYTHONPATH`` into frameworks in the + ``Frameworks`` folder (including stripping extensions from the module file, + the addition of framework metadata, and signing the resulting framework), + and creating the ``.fwork`` and ``.origin`` files. This will usually be done + with a build step in the Xcode project; see the iOS documentation for + details on how to construct this build step. + + .. versionadded:: 3.13 + + .. availability:: iOS. + + .. attribute:: name + + Name of the module the loader supports. + + .. attribute:: path + + Path to the ``.fwork`` file for the extension module. + + :mod:`importlib.util` -- Utility code for importers --------------------------------------------------- diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index cd441836f62..9709c4f4dc5 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -133,7 +133,7 @@ class Availability(SphinxDirective): known_platforms = frozenset({ "AIX", "Android", "BSD", "DragonFlyBSD", "Emscripten", "FreeBSD", "GNU/kFreeBSD", "Linux", "NetBSD", "OpenBSD", "POSIX", "Solaris", - "Unix", "VxWorks", "WASI", "Windows", "macOS", + "Unix", "VxWorks", "WASI", "Windows", "macOS", "iOS", # libc "BSD libc", "glibc", "musl", # POSIX platforms with pthreads diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index f63e31a3fb0..36b2af7f2a0 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -348,6 +348,17 @@ class CDLL(object): winmode=None): if name: name = _os.fspath(name) + + # If the filename that has been provided is an iOS/tvOS/watchOS + # .fwork file, dereference the location to the true origin of the + # binary. + if name.endswith(".fwork"): + with open(name) as f: + name = _os.path.join( + _os.path.dirname(_sys.executable), + f.read().strip() + ) + self._name = name flags = self._func_flags_ if use_errno: diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index c550883e7c7..12d7428fe9a 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -67,7 +67,7 @@ if os.name == "nt": return fname return None -elif os.name == "posix" and sys.platform == "darwin": +elif os.name == "posix" and sys.platform in {"darwin", "ios", "tvos", "watchos"}: from ctypes.macholib.dyld import dyld_find as _dyld_find def find_library(name): possible = ['lib%s.dylib' % name, diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index b26be8583d0..4749a627c50 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -52,7 +52,7 @@ _pathseps_with_colon = {f':{s}' for s in path_separators} # Bootstrap-related code ###################################################### _CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win', -_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin' +_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin', 'ios', 'tvos', 'watchos' _CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY + _CASE_INSENSITIVE_PLATFORMS_STR_KEY) @@ -1714,6 +1714,46 @@ class FileFinder: return f'FileFinder({self.path!r})' +class AppleFrameworkLoader(ExtensionFileLoader): + """A loader for modules that have been packaged as frameworks for + compatibility with Apple's iOS App Store policies. + """ + def create_module(self, spec): + # If the ModuleSpec has been created by the FileFinder, it will have + # been created with an origin pointing to the .fwork file. We need to + # redirect this to the location in the Frameworks folder, using the + # content of the .fwork file. + if spec.origin.endswith(".fwork"): + with _io.FileIO(spec.origin, 'r') as file: + framework_binary = file.read().decode().strip() + bundle_path = _path_split(sys.executable)[0] + spec.origin = _path_join(bundle_path, framework_binary) + + # If the loader is created based on the spec for a loaded module, the + # path will be pointing at the Framework location. If this occurs, + # get the original .fwork location to use as the module's __file__. + if self.path.endswith(".fwork"): + path = self.path + else: + with _io.FileIO(self.path + ".origin", 'r') as file: + origin = file.read().decode().strip() + bundle_path = _path_split(sys.executable)[0] + path = _path_join(bundle_path, origin) + + module = _bootstrap._call_with_frames_removed(_imp.create_dynamic, spec) + + _bootstrap._verbose_message( + "Apple framework extension module {!r} loaded from {!r} (path {!r})", + spec.name, + spec.origin, + path, + ) + + # Ensure that the __file__ points at the .fwork location + module.__file__ = path + + return module + # Import setup ############################################################### def _fix_up_module(ns, name, pathname, cpathname=None): @@ -1746,10 +1786,17 @@ def _get_supported_file_loaders(): Each item is a tuple (loader, suffixes). """ - extensions = ExtensionFileLoader, _imp.extension_suffixes() + if sys.platform in {"ios", "tvos", "watchos"}: + extension_loaders = [(AppleFrameworkLoader, [ + suffix.replace(".so", ".fwork") + for suffix in _imp.extension_suffixes() + ])] + else: + extension_loaders = [] + extension_loaders.append((ExtensionFileLoader, _imp.extension_suffixes())) source = SourceFileLoader, SOURCE_SUFFIXES bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES - return [extensions, source, bytecode] + return extension_loaders + [source, bytecode] def _set_bootstrap_module(_bootstrap_module): diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index b56fa94eb9c..37fef357fe2 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -180,7 +180,11 @@ class ExecutionLoader(InspectLoader): else: return self.source_to_code(source, path) -_register(ExecutionLoader, machinery.ExtensionFileLoader) +_register( + ExecutionLoader, + machinery.ExtensionFileLoader, + machinery.AppleFrameworkLoader, +) class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader): diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index d9a19a13f7b..fbd30b159fb 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -12,6 +12,7 @@ from ._bootstrap_external import FileFinder from ._bootstrap_external import SourceFileLoader from ._bootstrap_external import SourcelessFileLoader from ._bootstrap_external import ExtensionFileLoader +from ._bootstrap_external import AppleFrameworkLoader from ._bootstrap_external import NamespaceLoader diff --git a/Lib/inspect.py b/Lib/inspect.py index 8a2b2c96e99..7336cea0dc3 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -954,6 +954,10 @@ def getsourcefile(object): elif any(filename.endswith(s) for s in importlib.machinery.EXTENSION_SUFFIXES): return None + elif filename.endswith(".fwork"): + # Apple mobile framework markers are another type of non-source file + return None + # return a filename found in the linecache even if it doesn't exist on disk if filename in linecache.cache: return filename @@ -984,6 +988,7 @@ def getmodule(object, _filename=None): return object if hasattr(object, '__module__'): return sys.modules.get(object.__module__) + # Try the filename to modulename cache if _filename is not None and _filename in modulesbyfile: return sys.modules.get(modulesbyfile[_filename]) @@ -1119,7 +1124,7 @@ def findsource(object): # Allow filenames in form of "" to pass through. # `doctest` monkeypatches `linecache` module to enable # inspection, so let `linecache.getlines` to be called. - if not (file.startswith('<') and file.endswith('>')): + if (not (file.startswith('<') and file.endswith('>'))) or file.endswith('.fwork'): raise OSError('source code not available') module = getmodule(object, file) diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index a0a020f9eeb..ac478ee7f51 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -72,7 +72,12 @@ def _find_module(name, path=None): if isinstance(spec.loader, importlib.machinery.SourceFileLoader): kind = _PY_SOURCE - elif isinstance(spec.loader, importlib.machinery.ExtensionFileLoader): + elif isinstance( + spec.loader, ( + importlib.machinery.ExtensionFileLoader, + importlib.machinery.AppleFrameworkLoader, + ) + ): kind = _C_EXTENSION elif isinstance(spec.loader, importlib.machinery.SourcelessFileLoader): diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index d3fcd0b59df..7365ead1c47 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2056,6 +2056,13 @@ class SubinterpreterTest(unittest.TestCase): self.addCleanup(os.close, r) self.addCleanup(os.close, w) + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + script = textwrap.dedent(f""" import importlib.machinery import importlib.util @@ -2063,7 +2070,7 @@ class SubinterpreterTest(unittest.TestCase): fullname = '_test_module_state_shared' origin = importlib.util.find_spec('_testmultiphase').origin - loader = importlib.machinery.ExtensionFileLoader(fullname, origin) + loader = importlib.machinery.{loader}(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) attr_id = str(id(module.Error)).encode() @@ -2371,7 +2378,12 @@ class Test_ModuleStateAccess(unittest.TestCase): def setUp(self): fullname = '_testmultiphase_meth_state_access' # XXX origin = importlib.util.find_spec('_testmultiphase').origin - loader = importlib.machinery.ExtensionFileLoader(fullname, origin) + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = importlib.machinery.AppleFrameworkLoader(fullname, origin) + else: + loader = importlib.machinery.ExtensionFileLoader(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 7b0126226c4..4deed7f3ba2 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -5,7 +5,11 @@ import json import importlib.util from importlib._bootstrap_external import _get_sourcefile from importlib.machinery import ( - BuiltinImporter, ExtensionFileLoader, FrozenImporter, SourceFileLoader, + AppleFrameworkLoader, + BuiltinImporter, + ExtensionFileLoader, + FrozenImporter, + SourceFileLoader, ) import marshal import os @@ -25,7 +29,7 @@ import _imp from test.support import os_helper from test.support import ( - STDLIB_DIR, swap_attr, swap_item, cpython_only, is_emscripten, + STDLIB_DIR, swap_attr, swap_item, cpython_only, is_apple_mobile, is_emscripten, is_wasi, run_in_subinterp, run_in_subinterp_with_config, Py_TRACE_REFS) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, ready_to_import, @@ -66,6 +70,7 @@ def _require_loader(module, loader, skip): MODULE_KINDS = { BuiltinImporter: 'built-in', ExtensionFileLoader: 'extension', + AppleFrameworkLoader: 'framework extension', FrozenImporter: 'frozen', SourceFileLoader: 'pure Python', } @@ -91,7 +96,12 @@ def require_builtin(module, *, skip=False): assert module.__spec__.origin == 'built-in', module.__spec__ def require_extension(module, *, skip=False): - _require_loader(module, ExtensionFileLoader, skip) + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + _require_loader(module, AppleFrameworkLoader, skip) + else: + _require_loader(module, ExtensionFileLoader, skip) def require_frozen(module, *, skip=True): module = _require_loader(module, FrozenImporter, skip) @@ -134,7 +144,8 @@ if _testsinglephase is not None: # it to its nominal state. sys.modules.pop('_testsinglephase', None) _orig._clear_globals() - _testinternalcapi.clear_extension('_testsinglephase', _orig.__file__) + origin = _orig.__spec__.origin + _testinternalcapi.clear_extension('_testsinglephase', origin) import _testsinglephase @@ -360,7 +371,7 @@ class ImportTests(unittest.TestCase): self.assertEqual(cm.exception.path, _testcapi.__file__) self.assertRegex( str(cm.exception), - r"cannot import name 'i_dont_exist' from '_testcapi' \(.*\.(so|pyd)\)" + r"cannot import name 'i_dont_exist' from '_testcapi' \(.*\.(so|fwork|pyd)\)" ) else: self.assertEqual( @@ -1689,6 +1700,14 @@ class SubinterpImportTests(unittest.TestCase): os.set_blocking(r, False) return (r, w) + def create_extension_loader(self, modname, filename): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + return AppleFrameworkLoader(modname, filename) + else: + return ExtensionFileLoader(modname, filename) + def import_script(self, name, fd, filename=None, check_override=None): override_text = '' if check_override is not None: @@ -1697,12 +1716,19 @@ class SubinterpImportTests(unittest.TestCase): _imp._override_multi_interp_extensions_check({check_override}) ''' if filename: + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + return textwrap.dedent(f''' from importlib.util import spec_from_loader, module_from_spec - from importlib.machinery import ExtensionFileLoader + from importlib.machinery import {loader} import os, sys {override_text} - loader = ExtensionFileLoader({name!r}, {filename!r}) + loader = {loader}({name!r}, {filename!r}) spec = spec_from_loader({name!r}, loader) try: module = module_from_spec(spec) @@ -1883,7 +1909,7 @@ class SubinterpImportTests(unittest.TestCase): def test_multi_init_extension_non_isolated_compat(self): modname = '_test_non_isolated' filename = _testmultiphase.__file__ - loader = ExtensionFileLoader(modname, filename) + loader = self.create_extension_loader(modname, filename) spec = importlib.util.spec_from_loader(modname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -1901,7 +1927,7 @@ class SubinterpImportTests(unittest.TestCase): def test_multi_init_extension_per_interpreter_gil_compat(self): modname = '_test_shared_gil_only' filename = _testmultiphase.__file__ - loader = ExtensionFileLoader(modname, filename) + loader = self.create_extension_loader(modname, filename) spec = importlib.util.spec_from_loader(modname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -2034,10 +2060,25 @@ class SinglephaseInitTests(unittest.TestCase): @classmethod def setUpClass(cls): spec = importlib.util.find_spec(cls.NAME) - from importlib.machinery import ExtensionFileLoader - cls.FILE = spec.origin cls.LOADER = type(spec.loader) - assert cls.LOADER is ExtensionFileLoader + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader, and we need to differentiate between the + # spec.origin and the original file location. + if is_apple_mobile: + assert cls.LOADER is AppleFrameworkLoader + + cls.ORIGIN = spec.origin + with open(spec.origin + ".origin", "r") as f: + cls.FILE = os.path.join( + os.path.dirname(sys.executable), + f.read().strip() + ) + else: + assert cls.LOADER is ExtensionFileLoader + + cls.ORIGIN = spec.origin + cls.FILE = spec.origin # Start fresh. cls.clean_up() @@ -2053,14 +2094,15 @@ class SinglephaseInitTests(unittest.TestCase): @classmethod def clean_up(cls): name = cls.NAME - filename = cls.FILE if name in sys.modules: if hasattr(sys.modules[name], '_clear_globals'): - assert sys.modules[name].__file__ == filename + assert sys.modules[name].__file__ == cls.FILE, \ + f"{sys.modules[name].__file__} != {cls.FILE}" + sys.modules[name]._clear_globals() del sys.modules[name] # Clear all internally cached data for the extension. - _testinternalcapi.clear_extension(name, filename) + _testinternalcapi.clear_extension(name, cls.ORIGIN) ######################### # helpers @@ -2068,7 +2110,7 @@ class SinglephaseInitTests(unittest.TestCase): def add_module_cleanup(self, name): def clean_up(): # Clear all internally cached data for the extension. - _testinternalcapi.clear_extension(name, self.FILE) + _testinternalcapi.clear_extension(name, self.ORIGIN) self.addCleanup(clean_up) def _load_dynamic(self, name, path): @@ -2091,7 +2133,7 @@ class SinglephaseInitTests(unittest.TestCase): except AttributeError: already_loaded = self.already_loaded = {} assert name not in already_loaded - mod = self._load_dynamic(name, self.FILE) + mod = self._load_dynamic(name, self.ORIGIN) self.assertNotIn(mod, already_loaded.values()) already_loaded[name] = mod return types.SimpleNamespace( @@ -2103,7 +2145,7 @@ class SinglephaseInitTests(unittest.TestCase): def re_load(self, name, mod): assert sys.modules[name] is mod assert mod.__dict__ == mod.__dict__ - reloaded = self._load_dynamic(name, self.FILE) + reloaded = self._load_dynamic(name, self.ORIGIN) return types.SimpleNamespace( name=name, module=reloaded, @@ -2129,7 +2171,7 @@ class SinglephaseInitTests(unittest.TestCase): name = {self.NAME!r} if name in sys.modules: sys.modules.pop(name)._clear_globals() - _testinternalcapi.clear_extension(name, {self.FILE!r}) + _testinternalcapi.clear_extension(name, {self.ORIGIN!r}) ''')) _interpreters.destroy(interpid) self.addCleanup(clean_up) @@ -2146,7 +2188,7 @@ class SinglephaseInitTests(unittest.TestCase): postcleanup = f''' {import_} mod._clear_globals() - _testinternalcapi.clear_extension(name, {self.FILE!r}) + _testinternalcapi.clear_extension(name, {self.ORIGIN!r}) ''' try: @@ -2184,7 +2226,7 @@ class SinglephaseInitTests(unittest.TestCase): # mod.__name__ might not match, but the spec will. self.assertEqual(mod.__spec__.name, loaded.name) self.assertEqual(mod.__file__, self.FILE) - self.assertEqual(mod.__spec__.origin, self.FILE) + self.assertEqual(mod.__spec__.origin, self.ORIGIN) if not isolated: self.assertTrue(issubclass(mod.error, Exception)) self.assertEqual(mod.int_const, 1969) @@ -2578,7 +2620,7 @@ class SinglephaseInitTests(unittest.TestCase): # First, load in the main interpreter but then completely clear it. loaded_main = self.load(self.NAME) loaded_main.module._clear_globals() - _testinternalcapi.clear_extension(self.NAME, self.FILE) + _testinternalcapi.clear_extension(self.NAME, self.ORIGIN) # At this point: # * alive in 0 interpreters diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py index 3de120958fd..cdc8884d668 100644 --- a/Lib/test/test_importlib/extension/test_finder.py +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -1,3 +1,4 @@ +from test.support import is_apple_mobile from test.test_importlib import abc, util machinery = util.import_importlib('importlib.machinery') @@ -19,9 +20,27 @@ class FinderTests(abc.FinderTests): ) def find_spec(self, fullname): - importer = self.machinery.FileFinder(util.EXTENSIONS.path, - (self.machinery.ExtensionFileLoader, - self.machinery.EXTENSION_SUFFIXES)) + if is_apple_mobile: + # Apple mobile platforms require a specialist loader that uses + # .fwork files as placeholders for the true `.so` files. + loaders = [ + ( + self.machinery.AppleFrameworkLoader, + [ + ext.replace(".so", ".fwork") + for ext in self.machinery.EXTENSION_SUFFIXES + ] + ) + ] + else: + loaders = [ + ( + self.machinery.ExtensionFileLoader, + self.machinery.EXTENSION_SUFFIXES + ) + ] + + importer = self.machinery.FileFinder(util.EXTENSIONS.path, *loaders) return importer.find_spec(fullname) diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index f4879e75847..7607f0e0857 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -1,3 +1,4 @@ +from test.support import is_apple_mobile from test.test_importlib import abc, util machinery = util.import_importlib('importlib.machinery') @@ -23,8 +24,15 @@ class LoaderTests: raise unittest.SkipTest( f"{util.EXTENSIONS.name} is a builtin module" ) - self.loader = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + + self.loader = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path) def load_module(self, fullname): with warnings.catch_warnings(): @@ -32,13 +40,11 @@ class LoaderTests: return self.loader.load_module(fullname) def test_equality(self): - other = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + other = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path) self.assertEqual(self.loader, other) def test_inequality(self): - other = self.machinery.ExtensionFileLoader('_' + util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + other = self.LoaderClass('_' + util.EXTENSIONS.name, util.EXTENSIONS.file_path) self.assertNotEqual(self.loader, other) def test_load_module_API(self): @@ -58,8 +64,7 @@ class LoaderTests: ('__package__', '')]: self.assertEqual(getattr(module, attr), value) self.assertIn(util.EXTENSIONS.name, sys.modules) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) # No extension module as __init__ available for testing. test_package = None @@ -86,7 +91,7 @@ class LoaderTests: self.assertFalse(self.loader.is_package(util.EXTENSIONS.name)) for suffix in self.machinery.EXTENSION_SUFFIXES: path = os.path.join('some', 'path', 'pkg', '__init__' + suffix) - loader = self.machinery.ExtensionFileLoader('pkg', path) + loader = self.LoaderClass('pkg', path) self.assertTrue(loader.is_package('pkg')) @@ -101,6 +106,14 @@ class SinglePhaseExtensionModuleTests(abc.LoaderTests): def setUp(self): if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + self.name = '_testsinglephase' if self.name in sys.builtin_module_names: raise unittest.SkipTest( @@ -109,8 +122,8 @@ class SinglePhaseExtensionModuleTests(abc.LoaderTests): finder = self.machinery.FileFinder(None) self.spec = importlib.util.find_spec(self.name) assert self.spec - self.loader = self.machinery.ExtensionFileLoader( - self.name, self.spec.origin) + + self.loader = self.LoaderClass(self.name, self.spec.origin) def load_module(self): with warnings.catch_warnings(): @@ -120,7 +133,7 @@ class SinglePhaseExtensionModuleTests(abc.LoaderTests): def load_module_by_name(self, fullname): # Load a module from the test extension by name. origin = self.spec.origin - loader = self.machinery.ExtensionFileLoader(fullname, origin) + loader = self.LoaderClass(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -137,8 +150,7 @@ class SinglePhaseExtensionModuleTests(abc.LoaderTests): with self.assertRaises(AttributeError): module.__path__ self.assertIs(module, sys.modules[self.name]) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) # No extension module as __init__ available for testing. test_package = None @@ -182,6 +194,14 @@ class MultiPhaseExtensionModuleTests(abc.LoaderTests): def setUp(self): if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + self.name = '_testmultiphase' if self.name in sys.builtin_module_names: raise unittest.SkipTest( @@ -190,8 +210,7 @@ class MultiPhaseExtensionModuleTests(abc.LoaderTests): finder = self.machinery.FileFinder(None) self.spec = importlib.util.find_spec(self.name) assert self.spec - self.loader = self.machinery.ExtensionFileLoader( - self.name, self.spec.origin) + self.loader = self.LoaderClass(self.name, self.spec.origin) def load_module(self): # Load the module from the test extension. @@ -202,7 +221,7 @@ class MultiPhaseExtensionModuleTests(abc.LoaderTests): def load_module_by_name(self, fullname): # Load a module from the test extension by name. origin = self.spec.origin - loader = self.machinery.ExtensionFileLoader(fullname, origin) + loader = self.LoaderClass(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -228,8 +247,7 @@ class MultiPhaseExtensionModuleTests(abc.LoaderTests): with self.assertRaises(AttributeError): module.__path__ self.assertIs(module, sys.modules[self.name]) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) def test_functionality(self): # Test basic functionality of stuff defined in an extension module. diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index a09286806e5..a6a76e58976 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -707,13 +707,20 @@ class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase): @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") def test_incomplete_multi_phase_init_module(self): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + prescript = textwrap.dedent(f''' from importlib.util import spec_from_loader, module_from_spec - from importlib.machinery import ExtensionFileLoader + from importlib.machinery import {loader} name = '_test_shared_gil_only' filename = {_testmultiphase.__file__!r} - loader = ExtensionFileLoader(name, filename) + loader = {loader}(name, filename) spec = spec_from_loader(name, loader) ''') diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index a900cc1dddf..89272484009 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -8,6 +8,7 @@ import os import os.path from test import support from test.support import import_helper +from test.support import is_apple_mobile from test.support import os_helper import unittest import sys @@ -43,6 +44,11 @@ else: global EXTENSIONS for path in sys.path: for ext in machinery.EXTENSION_SUFFIXES: + # Apple mobile platforms mechanically load .so files, + # but the findable files are labelled .fwork + if is_apple_mobile: + ext = ext.replace(".so", ".fwork") + filename = EXTENSIONS.name + ext file_path = os.path.join(path, filename) if os.path.exists(file_path): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst new file mode 100644 index 00000000000..5405a3bdc36 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst @@ -0,0 +1,2 @@ +Added a Loader that can discover extension modules in an iOS-style Frameworks +folder. diff --git a/configure b/configure index 07dce38c927..229f0d32d32 100755 --- a/configure +++ b/configure @@ -12740,7 +12740,6 @@ if test -z "$SHLIB_SUFFIX"; then esac ;; CYGWIN*) SHLIB_SUFFIX=.dll;; - iOS) SHLIB_SUFFIX=.dylib;; *) SHLIB_SUFFIX=.so;; esac fi diff --git a/configure.ac b/configure.ac index 3e676c56693..cd179777384 100644 --- a/configure.ac +++ b/configure.ac @@ -3285,7 +3285,6 @@ if test -z "$SHLIB_SUFFIX"; then esac ;; CYGWIN*) SHLIB_SUFFIX=.dll;; - iOS) SHLIB_SUFFIX=.dylib;; *) SHLIB_SUFFIX=.so;; esac fi diff --git a/iOS/Resources/Info.plist.in b/iOS/Resources/Info.plist.in index 3ecdc894f0a..52c0a6e7fd7 100644 --- a/iOS/Resources/Info.plist.in +++ b/iOS/Resources/Info.plist.in @@ -17,7 +17,7 @@ CFBundlePackageType FMWK CFBundleShortVersionString - %VERSION% + @VERSION@ CFBundleLongVersionString %VERSION%, (c) 2001-2024 Python Software Foundation. CFBundleSignature diff --git a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj index 4f138a4e7cc..4389c08ac19 100644 --- a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj +++ b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj @@ -273,7 +273,7 @@ ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_DYLIB=$2\n\n # The name of the .dylib file\n DYLIB=$(basename \"$FULL_DYLIB\")\n # The name of the .dylib file, relative to the install base\n RELATIVE_DYLIB=${FULL_DYLIB#$CODESIGNING_FOLDER_PATH/$INSTALL_BASE/}\n # The full dotted name of the binary module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $RELATIVE_DYLIB | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_DYLIB\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$DYLIB\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\" \n fi\n \n echo \"Installing binary for $RELATIVE_DYLIB\" \n mv \"$FULL_DYLIB\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library dylibs...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.dylib\" | while read FULL_DYLIB; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload \"$FULL_DYLIB\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; + shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The location of the extension file, relative to the bundle\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n # The path to the extension file, relative to the install base\n PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n # The full dotted name of the extension module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n # Create a back reference to the .so file location in the framework\n echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\" \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; }; /* End PBXShellScriptBuildPhase section */ diff --git a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m index 53ea107db4a..e6a919c304e 100644 --- a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m +++ b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m @@ -15,7 +15,7 @@ const char *argv[] = { "iOSTestbed", // argv[0] is the process that is running. "-uall", // Enable all resources - "-v", // run in verbose mode so we get test failure information + "-W", // Display test output on failure // To run a subset of tests, add the test names below; e.g., // "test_os", // "test_sys",