From 4454057269b995341b04d13f0bf97f96080f27d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Batuhan=20Ta=C5=9Fkaya?= Date: Wed, 22 Apr 2020 19:09:03 +0300 Subject: [PATCH] bpo-39562: Prevent collision of future and compiler flags (GH-19230) The constant values of future flags in the __future__ module is updated in order to prevent collision with compiler flags. Previously PyCF_ALLOW_TOP_LEVEL_AWAIT was clashing with CO_FUTURE_DIVISION. --- Doc/whatsnew/3.9.rst | 4 ++++ Include/code.h | 24 ++++++++++++------------ Include/compile.h | 6 ++++++ Lib/__future__.py | 16 ++++++++-------- Lib/test/test_future.py | 17 +++++++++++++++++ Python/bltinmodule.c | 2 +- 6 files changed, 48 insertions(+), 21 deletions(-) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 20ebe92865a..8064785178c 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -832,6 +832,10 @@ Changes in the Python API inherit from it should have this method defined. (Contributed by Kyle Stanley in :issue:`34037`.) +* The constant values of future flags in the :mod:`__future__` module + is updated in order to prevent collision with compiler flags. Previously + ``PyCF_ALLOW_TOP_LEVEL_AWAIT`` was clashing with ``CO_FUTURE_DIVISION``. + (Contributed by Batuhan Taskaya in :issue:`39562`) CPython bytecode changes ------------------------ diff --git a/Include/code.h b/Include/code.h index 107eba4b9c4..b268a6aedff 100644 --- a/Include/code.h +++ b/Include/code.h @@ -88,19 +88,19 @@ typedef struct { #define CO_ITERABLE_COROUTINE 0x0100 #define CO_ASYNC_GENERATOR 0x0200 -/* These are no longer used. */ -#if 0 -#define CO_GENERATOR_ALLOWED 0x1000 -#endif -#define CO_FUTURE_DIVISION 0x2000 -#define CO_FUTURE_ABSOLUTE_IMPORT 0x4000 /* do absolute imports by default */ -#define CO_FUTURE_WITH_STATEMENT 0x8000 -#define CO_FUTURE_PRINT_FUNCTION 0x10000 -#define CO_FUTURE_UNICODE_LITERALS 0x20000 +/* bpo-39562: These constant values are changed in Python 3.9 + to prevent collision with compiler flags. CO_FUTURE_ and PyCF_ + constants must be kept unique. PyCF_ constants can use bits from + 0x0100 to 0x10000. CO_FUTURE_ constants use bits starting at 0x20000. */ +#define CO_FUTURE_DIVISION 0x20000 +#define CO_FUTURE_ABSOLUTE_IMPORT 0x40000 /* do absolute imports by default */ +#define CO_FUTURE_WITH_STATEMENT 0x80000 +#define CO_FUTURE_PRINT_FUNCTION 0x100000 +#define CO_FUTURE_UNICODE_LITERALS 0x200000 -#define CO_FUTURE_BARRY_AS_BDFL 0x40000 -#define CO_FUTURE_GENERATOR_STOP 0x80000 -#define CO_FUTURE_ANNOTATIONS 0x100000 +#define CO_FUTURE_BARRY_AS_BDFL 0x400000 +#define CO_FUTURE_GENERATOR_STOP 0x800000 +#define CO_FUTURE_ANNOTATIONS 0x1000000 /* This value is found in the co_cell2arg array when the associated cell variable does not correspond to an argument. */ diff --git a/Include/compile.h b/Include/compile.h index c0c73b29e4d..a2db65d47f0 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -18,12 +18,18 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *); CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \ CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS) #define PyCF_MASK_OBSOLETE (CO_NESTED) + +/* bpo-39562: CO_FUTURE_ and PyCF_ constants must be kept unique. + PyCF_ constants can use bits from 0x0100 to 0x10000. + CO_FUTURE_ constants use bits starting at 0x20000. */ #define PyCF_SOURCE_IS_UTF8 0x0100 #define PyCF_DONT_IMPLY_DEDENT 0x0200 #define PyCF_ONLY_AST 0x0400 #define PyCF_IGNORE_COOKIE 0x0800 #define PyCF_TYPE_COMMENTS 0x1000 #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000 +#define PyCF_COMPILE_MASK (PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | \ + PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT) #ifndef Py_LIMITED_API typedef struct { diff --git a/Lib/__future__.py b/Lib/__future__.py index e1135685d84..d7cb8ac5f49 100644 --- a/Lib/__future__.py +++ b/Lib/__future__.py @@ -68,14 +68,14 @@ __all__ = ["all_feature_names"] + all_feature_names # this module. CO_NESTED = 0x0010 # nested_scopes CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000) -CO_FUTURE_DIVISION = 0x2000 # division -CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default -CO_FUTURE_WITH_STATEMENT = 0x8000 # with statement -CO_FUTURE_PRINT_FUNCTION = 0x10000 # print function -CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals -CO_FUTURE_BARRY_AS_BDFL = 0x40000 -CO_FUTURE_GENERATOR_STOP = 0x80000 # StopIteration becomes RuntimeError in generators -CO_FUTURE_ANNOTATIONS = 0x100000 # annotations become strings at runtime +CO_FUTURE_DIVISION = 0x20000 # division +CO_FUTURE_ABSOLUTE_IMPORT = 0x40000 # perform absolute imports by default +CO_FUTURE_WITH_STATEMENT = 0x80000 # with statement +CO_FUTURE_PRINT_FUNCTION = 0x100000 # print function +CO_FUTURE_UNICODE_LITERALS = 0x200000 # unicode string literals +CO_FUTURE_BARRY_AS_BDFL = 0x400000 +CO_FUTURE_GENERATOR_STOP = 0x800000 # StopIteration becomes RuntimeError in generators +CO_FUTURE_ANNOTATIONS = 0x1000000 # annotations become strings at runtime class _Feature: def __init__(self, optionalRelease, mandatoryRelease, compiler_flag): diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py index fdca2312fab..56b7ac68655 100644 --- a/Lib/test/test_future.py +++ b/Lib/test/test_future.py @@ -1,5 +1,7 @@ # Test various flavors of legal and illegal future statements +import __future__ +import ast import unittest from test import support from textwrap import dedent @@ -75,6 +77,21 @@ class FutureTest(unittest.TestCase): from test import badsyntax_future10 self.check_syntax_error(cm.exception, "badsyntax_future10", 3) + def test_ensure_flags_dont_clash(self): + # bpo-39562: test that future flags and compiler flags doesn't clash + + # obtain future flags (CO_FUTURE_***) from the __future__ module + flags = { + f"CO_FUTURE_{future.upper()}": getattr(__future__, future).compiler_flag + for future in __future__.all_feature_names + } + # obtain some of the exported compiler flags (PyCF_***) from the ast module + flags |= { + flag: getattr(ast, flag) + for flag in dir(ast) if flag.startswith("PyCF_") + } + self.assertCountEqual(set(flags.values()), flags.values()) + def test_parserhack(self): # test that the parser.c::future_hack function works as expected # Note: although this test must pass, it's not testing the original diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index a9fc21f1087..22ee5969473 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -739,7 +739,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, } if (flags & - ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT | PyCF_ONLY_AST | PyCF_TYPE_COMMENTS)) + ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_COMPILE_MASK)) { PyErr_SetString(PyExc_ValueError, "compile(): unrecognised flags");