From a4fd7aa4a6420cef1c22ec64eab54d8aea41cc57 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 21 Aug 2024 15:52:04 +0100 Subject: [PATCH] GH-115776: Allow any fixed sized object to have inline values (GH-123192) --- Include/internal/pycore_object.h | 5 +++-- ...-08-21-08-53-00.gh-issue-115776.9A7Dv_.rst | 2 ++ Objects/object_layout.md | 4 ++++ Objects/typeobject.c | 2 +- Python/bytecodes.c | 16 ++++++++------- Python/executor_cases.c.h | 20 +++++++++++-------- Python/gc.c | 3 +++ Python/gc_free_threading.c | 3 +++ Python/generated_cases.c.h | 15 ++++++++------ Python/optimizer_bytecodes.c | 4 ++-- Python/optimizer_cases.c.h | 4 ++-- Python/specialize.c | 16 +++++++++------ Tools/gdb/libpython.py | 5 +---- 13 files changed, 61 insertions(+), 38 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-08-21-08-53-00.gh-issue-115776.9A7Dv_.rst diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index ee33da77f01..0f2de6fd28f 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -803,10 +803,11 @@ _PyObject_GetManagedDict(PyObject *obj) static inline PyDictValues * _PyObject_InlineValues(PyObject *obj) { + PyTypeObject *tp = Py_TYPE(obj); + assert(tp->tp_basicsize > 0 && tp->tp_basicsize % sizeof(PyObject *) == 0); assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - assert(Py_TYPE(obj)->tp_basicsize == sizeof(PyObject)); - return (PyDictValues *)((char *)obj + sizeof(PyObject)); + return (PyDictValues *)((char *)obj + tp->tp_basicsize); } extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-21-08-53-00.gh-issue-115776.9A7Dv_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-21-08-53-00.gh-issue-115776.9A7Dv_.rst new file mode 100644 index 00000000000..953ebd72382 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-21-08-53-00.gh-issue-115776.9A7Dv_.rst @@ -0,0 +1,2 @@ +Enables inline values (Python's equivalent of hidden classes) on any class +who's instances are of a fixed size. diff --git a/Objects/object_layout.md b/Objects/object_layout.md index 352409425ee..4a781668636 100644 --- a/Objects/object_layout.md +++ b/Objects/object_layout.md @@ -28,6 +28,10 @@ So the pre-header is these two fields: If the object has no physical dictionary, then the ``dict_pointer`` is set to `NULL`. +In 3.13 only objects with no additional data could have inline values. +That is, instances of classes with `tp_basicsize == sizeof(PyObject)`. +In 3.14, any object whose class has `tp_itemsize == 0` can have inline values. +In both versions, the inline values starts `tp_basicsize` bytes after the object.
3.12 diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 0d7009ac57b..f74d51222b7 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8340,7 +8340,7 @@ type_ready_managed_dict(PyTypeObject *type) return -1; } } - if (type->tp_itemsize == 0 && type->tp_basicsize == sizeof(PyObject)) { + if (type->tp_itemsize == 0) { type->tp_flags |= Py_TPFLAGS_INLINE_VALUES; } return 0; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5adcd77b4d5..250e2d12a9d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2012,9 +2012,10 @@ dummy_func( DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid); } - split op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { + split op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, null if (oparg & 1))) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index]; + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = *value_ptr; DEOPT_IF(attr_o == NULL); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); @@ -2196,16 +2197,17 @@ dummy_func( EXIT_IF(_PyObject_InlineValues(owner_o)->valid == 0); } - op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) { + op(_STORE_ATTR_INSTANCE_VALUE, (offset/1, value, owner --)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); assert(_PyObject_GetManagedDict(owner_o) == NULL); - PyDictValues *values = _PyObject_InlineValues(owner_o); - - PyObject *old_value = values->values[index]; - values->values[index] = PyStackRef_AsPyObjectSteal(value); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + *value_ptr = PyStackRef_AsPyObjectSteal(value); if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + int index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } else { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1db8e506732..55b06a0e235 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2277,9 +2277,10 @@ _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t offset = (uint16_t)CURRENT_OPERAND(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index]; + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = *value_ptr; if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2299,9 +2300,10 @@ _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t offset = (uint16_t)CURRENT_OPERAND(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index]; + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = *value_ptr; if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2583,14 +2585,16 @@ _PyStackRef value; owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); + uint16_t offset = (uint16_t)CURRENT_OPERAND(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); assert(_PyObject_GetManagedDict(owner_o) == NULL); - PyDictValues *values = _PyObject_InlineValues(owner_o); - PyObject *old_value = values->values[index]; - values->values[index] = PyStackRef_AsPyObjectSteal(value); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + *value_ptr = PyStackRef_AsPyObjectSteal(value); if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + int index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } else { diff --git a/Python/gc.c b/Python/gc.c index 923a79299ca..f920743b921 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -2055,6 +2055,9 @@ _PyObject_GC_New(PyTypeObject *tp) return NULL; } _PyObject_Init(op, tp); + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + _PyObject_InitInlineValues(op, tp); + } return op; } diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index b95456519dc..54de0c2671a 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1810,6 +1810,9 @@ _PyObject_GC_New(PyTypeObject *tp) return NULL; } _PyObject_Init(op, tp); + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + _PyObject_InitInlineValues(op, tp); + } return op; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 42a58bb7285..13bbff29d09 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4999,9 +4999,10 @@ } // _LOAD_ATTR_INSTANCE_VALUE { - uint16_t index = read_u16(&this_instr[4].cache); + uint16_t offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index]; + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = *value_ptr; DEOPT_IF(attr_o == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); @@ -6829,14 +6830,16 @@ // _STORE_ATTR_INSTANCE_VALUE value = stack_pointer[-2]; { - uint16_t index = read_u16(&this_instr[4].cache); + uint16_t offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); assert(_PyObject_GetManagedDict(owner_o) == NULL); - PyDictValues *values = _PyObject_InlineValues(owner_o); - PyObject *old_value = values->values[index]; - values->values[index] = PyStackRef_AsPyObjectSteal(value); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + *value_ptr = PyStackRef_AsPyObjectSteal(value); if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + int index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } else { diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 6e46d9bed11..9a1b9da52f4 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -452,10 +452,10 @@ dummy_func(void) { top, unused[oparg-2], bottom)) { } - op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { + op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, null if (oparg & 1))) { attr = sym_new_not_null(ctx); null = sym_new_null(ctx); - (void)index; + (void)offset; (void)owner; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index e5be9d0e3b5..672fec3946f 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1064,10 +1064,10 @@ _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)this_instr->operand; + uint16_t offset = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); - (void)index; + (void)offset; (void)owner; stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; diff --git a/Python/specialize.c b/Python/specialize.c index b3a2e07c3bb..db794bea0be 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -849,15 +849,19 @@ specialize_dict_access( assert(PyUnicode_CheckExact(name)); Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); assert (index != DKIX_ERROR); - if (index != (uint16_t)index) { - SPECIALIZATION_FAIL(base_op, - index == DKIX_EMPTY ? - SPEC_FAIL_ATTR_NOT_IN_KEYS : - SPEC_FAIL_OUT_OF_RANGE); + if (index == DKIX_EMPTY) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS); + return 0; + } + assert(index >= 0); + char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index]; + Py_ssize_t offset = value_addr - (char *)owner; + if (offset != (uint16_t)offset) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); return 0; } write_u32(cache->version, type->tp_version_tag); - cache->index = (uint16_t)index; + cache->index = (uint16_t)offset; instr->op.code = values_op; } else { diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 8aa74635aed..cf03788d037 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -69,9 +69,6 @@ def _type_unsigned_int_ptr(): def _sizeof_void_p(): return gdb.lookup_type('void').pointer().sizeof -def _sizeof_pyobject(): - return gdb.lookup_type('PyObject').sizeof - def _managed_dict_offset(): # See pycore_object.h pyobj = gdb.lookup_type("PyObject") @@ -505,7 +502,7 @@ class HeapTypeObjectPtr(PyObjectPtr): dict_ptr = dict_ptr_ptr.cast(_type_char_ptr().pointer()).dereference() if int(dict_ptr): return None - char_ptr = obj_ptr + _sizeof_pyobject() + char_ptr = obj_ptr + typeobj.field('tp_basicsize') values_ptr = char_ptr.cast(gdb.lookup_type("PyDictValues").pointer()) values = values_ptr['values'] return PyKeysValuesPair(self.get_cached_keys(), values)