mirror of
https://github.com/python/cpython.git
synced 2024-11-24 17:47:13 +01:00
2e95c5ba3b
Each thread specializes a thread-local copy of the bytecode, created on the first RESUME, in free-threaded builds. All copies of the bytecode for a code object are stored in the co_tlbc array on the code object. Threads reserve a globally unique index identifying its copy of the bytecode in all co_tlbc arrays at thread creation and release the index at thread destruction. The first entry in every co_tlbc array always points to the "main" copy of the bytecode that is stored at the end of the code object. This ensures that no bytecode is copied for programs that do not use threads. Thread-local bytecode can be disabled at runtime by providing either -X tlbc=0 or PYTHON_TLBC=0. Disabling thread-local bytecode also disables specialization. Concurrent modifications to the bytecode made by the specializing interpreter and instrumentation use atomics, with specialization taking care not to overwrite an instruction that was instrumented concurrently.
2855 lines
92 KiB
C
2855 lines
92 KiB
C
#include "Python.h"
|
|
|
|
#include "opcode_ids.h"
|
|
|
|
#include "pycore_bitutils.h" // _Py_popcount32
|
|
#include "pycore_call.h"
|
|
#include "pycore_ceval.h" // _PY_EVAL_EVENTS_BITS
|
|
#include "pycore_code.h" // _PyCode_Clear_Executors()
|
|
#include "pycore_critical_section.h"
|
|
#include "pycore_frame.h"
|
|
#include "pycore_interp.h"
|
|
#include "pycore_long.h"
|
|
#include "pycore_modsupport.h" // _PyModule_CreateInitialized()
|
|
#include "pycore_namespace.h"
|
|
#include "pycore_object.h"
|
|
#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE, _PyOpcode_Caches
|
|
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_UINTPTR_RELEASE
|
|
#include "pycore_pyerrors.h"
|
|
#include "pycore_pystate.h" // _PyInterpreterState_GET()
|
|
|
|
/* Uncomment this to dump debugging output when assertions fail */
|
|
// #define INSTRUMENT_DEBUG 1
|
|
|
|
#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)
|
|
|
|
#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj) \
|
|
if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \
|
|
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); \
|
|
}
|
|
#define ASSERT_WORLD_STOPPED() assert(_PyInterpreterState_GET()->stoptheworld.world_stopped);
|
|
|
|
#else
|
|
|
|
#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj)
|
|
#define ASSERT_WORLD_STOPPED()
|
|
|
|
#endif
|
|
|
|
#ifdef Py_GIL_DISABLED
|
|
|
|
#define LOCK_CODE(code) \
|
|
assert(!_PyInterpreterState_GET()->stoptheworld.world_stopped); \
|
|
Py_BEGIN_CRITICAL_SECTION(code)
|
|
|
|
#define UNLOCK_CODE() Py_END_CRITICAL_SECTION()
|
|
|
|
#define MODIFY_BYTECODE(code, func, ...) \
|
|
do { \
|
|
PyCodeObject *co = (code); \
|
|
for (Py_ssize_t i = 0; i < code->co_tlbc->size; i++) { \
|
|
char *bc = co->co_tlbc->entries[i]; \
|
|
if (bc == NULL) { \
|
|
continue; \
|
|
} \
|
|
(func)((_Py_CODEUNIT *)bc, __VA_ARGS__); \
|
|
} \
|
|
} while (0)
|
|
|
|
#else
|
|
|
|
#define LOCK_CODE(code)
|
|
#define UNLOCK_CODE()
|
|
#define MODIFY_BYTECODE(code, func, ...) \
|
|
(func)(_PyCode_CODE(code), __VA_ARGS__)
|
|
|
|
#endif
|
|
|
|
PyObject _PyInstrumentation_DISABLE = _PyObject_HEAD_INIT(&PyBaseObject_Type);
|
|
|
|
PyObject _PyInstrumentation_MISSING = _PyObject_HEAD_INIT(&PyBaseObject_Type);
|
|
|
|
static const int8_t EVENT_FOR_OPCODE[256] = {
|
|
[RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN,
|
|
[INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN,
|
|
[CALL] = PY_MONITORING_EVENT_CALL,
|
|
[INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL,
|
|
[CALL_KW] = PY_MONITORING_EVENT_CALL,
|
|
[INSTRUMENTED_CALL_KW] = PY_MONITORING_EVENT_CALL,
|
|
[CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL,
|
|
[INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL,
|
|
[LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL,
|
|
[INSTRUMENTED_LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL,
|
|
[RESUME] = -1,
|
|
[YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD,
|
|
[INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD,
|
|
[JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP,
|
|
[JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP,
|
|
[POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH,
|
|
[POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH,
|
|
[POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH,
|
|
[POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH,
|
|
[INSTRUMENTED_JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP,
|
|
[INSTRUMENTED_JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP,
|
|
[INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH,
|
|
[INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH,
|
|
[INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH,
|
|
[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH,
|
|
[FOR_ITER] = PY_MONITORING_EVENT_BRANCH,
|
|
[INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH,
|
|
[END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION,
|
|
[INSTRUMENTED_END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION,
|
|
[END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION,
|
|
[INSTRUMENTED_END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION,
|
|
};
|
|
|
|
static const uint8_t DE_INSTRUMENT[256] = {
|
|
[INSTRUMENTED_RESUME] = RESUME,
|
|
[INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE,
|
|
[INSTRUMENTED_CALL] = CALL,
|
|
[INSTRUMENTED_CALL_KW] = CALL_KW,
|
|
[INSTRUMENTED_CALL_FUNCTION_EX] = CALL_FUNCTION_EX,
|
|
[INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE,
|
|
[INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD,
|
|
[INSTRUMENTED_JUMP_BACKWARD] = JUMP_BACKWARD,
|
|
[INSTRUMENTED_POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE,
|
|
[INSTRUMENTED_POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE,
|
|
[INSTRUMENTED_POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE,
|
|
[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE,
|
|
[INSTRUMENTED_FOR_ITER] = FOR_ITER,
|
|
[INSTRUMENTED_END_FOR] = END_FOR,
|
|
[INSTRUMENTED_END_SEND] = END_SEND,
|
|
[INSTRUMENTED_LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR,
|
|
};
|
|
|
|
static const uint8_t INSTRUMENTED_OPCODES[256] = {
|
|
[RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE,
|
|
[INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE,
|
|
[CALL] = INSTRUMENTED_CALL,
|
|
[INSTRUMENTED_CALL] = INSTRUMENTED_CALL,
|
|
[CALL_KW] = INSTRUMENTED_CALL_KW,
|
|
[INSTRUMENTED_CALL_KW] = INSTRUMENTED_CALL_KW,
|
|
[CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX,
|
|
[INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX,
|
|
[YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE,
|
|
[INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE,
|
|
[RESUME] = INSTRUMENTED_RESUME,
|
|
[INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME,
|
|
[JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD,
|
|
[INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD,
|
|
[JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD,
|
|
[INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD,
|
|
[POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE,
|
|
[INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE,
|
|
[POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE,
|
|
[INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE,
|
|
[POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE,
|
|
[INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE,
|
|
[POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE,
|
|
[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE,
|
|
[END_FOR] = INSTRUMENTED_END_FOR,
|
|
[INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR,
|
|
[END_SEND] = INSTRUMENTED_END_SEND,
|
|
[INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND,
|
|
[FOR_ITER] = INSTRUMENTED_FOR_ITER,
|
|
[INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER,
|
|
[LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR,
|
|
[INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR,
|
|
|
|
[INSTRUMENTED_LINE] = INSTRUMENTED_LINE,
|
|
[INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION,
|
|
};
|
|
|
|
static inline bool
|
|
opcode_has_event(int opcode)
|
|
{
|
|
return (
|
|
opcode != INSTRUMENTED_LINE &&
|
|
INSTRUMENTED_OPCODES[opcode] > 0
|
|
);
|
|
}
|
|
|
|
static inline bool
|
|
is_instrumented(int opcode)
|
|
{
|
|
assert(opcode != 0);
|
|
assert(opcode != RESERVED);
|
|
assert(opcode != ENTER_EXECUTOR);
|
|
return opcode >= MIN_INSTRUMENTED_OPCODE;
|
|
}
|
|
|
|
#ifndef NDEBUG
|
|
static inline bool
|
|
monitors_equals(_Py_LocalMonitors a, _Py_LocalMonitors b)
|
|
{
|
|
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
|
|
if (a.tools[i] != b.tools[i]) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
#endif
|
|
|
|
static inline _Py_LocalMonitors
|
|
monitors_sub(_Py_LocalMonitors a, _Py_LocalMonitors b)
|
|
{
|
|
_Py_LocalMonitors res;
|
|
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
|
|
res.tools[i] = a.tools[i] & ~b.tools[i];
|
|
}
|
|
return res;
|
|
}
|
|
|
|
#ifndef NDEBUG
|
|
static inline _Py_LocalMonitors
|
|
monitors_and(_Py_LocalMonitors a, _Py_LocalMonitors b)
|
|
{
|
|
_Py_LocalMonitors res;
|
|
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
|
|
res.tools[i] = a.tools[i] & b.tools[i];
|
|
}
|
|
return res;
|
|
}
|
|
#endif
|
|
|
|
/* The union of the *local* events in a and b.
|
|
* Global events like RAISE are ignored.
|
|
* Used for instrumentation, as only local
|
|
* events get instrumented.
|
|
*/
|
|
static inline _Py_LocalMonitors
|
|
local_union(_Py_GlobalMonitors a, _Py_LocalMonitors b)
|
|
{
|
|
_Py_LocalMonitors res;
|
|
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
|
|
res.tools[i] = a.tools[i] | b.tools[i];
|
|
}
|
|
return res;
|
|
}
|
|
|
|
static inline bool
|
|
monitors_are_empty(_Py_LocalMonitors m)
|
|
{
|
|
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
|
|
if (m.tools[i]) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static inline bool
|
|
multiple_tools(_Py_LocalMonitors *m)
|
|
{
|
|
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
|
|
if (_Py_popcount32(m->tools[i]) > 1) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static inline _PyMonitoringEventSet
|
|
get_local_events(_Py_LocalMonitors *m, int tool_id)
|
|
{
|
|
_PyMonitoringEventSet result = 0;
|
|
for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) {
|
|
if ((m->tools[e] >> tool_id) & 1) {
|
|
result |= (1 << e);
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
static inline _PyMonitoringEventSet
|
|
get_events(_Py_GlobalMonitors *m, int tool_id)
|
|
{
|
|
_PyMonitoringEventSet result = 0;
|
|
for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) {
|
|
if ((m->tools[e] >> tool_id) & 1) {
|
|
result |= (1 << e);
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
/* Line delta.
|
|
* 8 bit value.
|
|
* if line_delta == -128:
|
|
* line = None # represented as -1
|
|
* elif line_delta == -127 or line_delta == -126:
|
|
* line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT));
|
|
* else:
|
|
* line = first_line + (offset >> OFFSET_SHIFT) + line_delta;
|
|
*/
|
|
|
|
#define NO_LINE -128
|
|
#define COMPUTED_LINE_LINENO_CHANGE -127
|
|
#define COMPUTED_LINE -126
|
|
|
|
#define OFFSET_SHIFT 4
|
|
|
|
static int8_t
|
|
compute_line_delta(PyCodeObject *code, int offset, int line)
|
|
{
|
|
if (line < 0) {
|
|
return NO_LINE;
|
|
}
|
|
int delta = line - code->co_firstlineno - (offset >> OFFSET_SHIFT);
|
|
if (delta <= INT8_MAX && delta > COMPUTED_LINE) {
|
|
return delta;
|
|
}
|
|
return COMPUTED_LINE;
|
|
}
|
|
|
|
static int
|
|
compute_line(PyCodeObject *code, int offset, int8_t line_delta)
|
|
{
|
|
if (line_delta > COMPUTED_LINE) {
|
|
return code->co_firstlineno + (offset >> OFFSET_SHIFT) + line_delta;
|
|
}
|
|
if (line_delta == NO_LINE) {
|
|
|
|
return -1;
|
|
}
|
|
assert(line_delta == COMPUTED_LINE || line_delta == COMPUTED_LINE_LINENO_CHANGE);
|
|
/* Look it up */
|
|
return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT));
|
|
}
|
|
|
|
int
|
|
_PyInstruction_GetLength(PyCodeObject *code, int offset)
|
|
{
|
|
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
|
|
|
|
int opcode =
|
|
FT_ATOMIC_LOAD_UINT8_RELAXED(_PyCode_CODE(code)[offset].op.code);
|
|
assert(opcode != 0);
|
|
assert(opcode != RESERVED);
|
|
if (opcode == INSTRUMENTED_LINE) {
|
|
opcode = code->_co_monitoring->lines[offset].original_opcode;
|
|
}
|
|
if (opcode == INSTRUMENTED_INSTRUCTION) {
|
|
opcode = code->_co_monitoring->per_instruction_opcodes[offset];
|
|
}
|
|
int deinstrumented = DE_INSTRUMENT[opcode];
|
|
if (deinstrumented) {
|
|
opcode = deinstrumented;
|
|
}
|
|
else {
|
|
opcode = _PyOpcode_Deopt[opcode];
|
|
}
|
|
assert(opcode != 0);
|
|
if (opcode == ENTER_EXECUTOR) {
|
|
int exec_index = _PyCode_CODE(code)[offset].op.arg;
|
|
_PyExecutorObject *exec = code->co_executors->executors[exec_index];
|
|
opcode = _PyOpcode_Deopt[exec->vm_data.opcode];
|
|
}
|
|
assert(!is_instrumented(opcode));
|
|
assert(opcode != ENTER_EXECUTOR);
|
|
assert(opcode == _PyOpcode_Deopt[opcode]);
|
|
return 1 + _PyOpcode_Caches[opcode];
|
|
}
|
|
|
|
#ifdef INSTRUMENT_DEBUG
|
|
|
|
static void
|
|
dump_instrumentation_data_tools(PyCodeObject *code, uint8_t *tools, int i, FILE*out)
|
|
{
|
|
if (tools == NULL) {
|
|
fprintf(out, "tools = NULL");
|
|
}
|
|
else {
|
|
fprintf(out, "tools = %d", tools[i]);
|
|
}
|
|
}
|
|
|
|
static void
|
|
dump_instrumentation_data_lines(PyCodeObject *code, _PyCoLineInstrumentationData *lines, int i, FILE*out)
|
|
{
|
|
if (lines == NULL) {
|
|
fprintf(out, ", lines = NULL");
|
|
}
|
|
else if (lines[i].original_opcode == 0) {
|
|
fprintf(out, ", lines = {original_opcode = No LINE (0), line_delta = %d)", lines[i].line_delta);
|
|
}
|
|
else {
|
|
fprintf(out, ", lines = {original_opcode = %s, line_delta = %d)", _PyOpcode_OpName[lines[i].original_opcode], lines[i].line_delta);
|
|
}
|
|
}
|
|
|
|
static void
|
|
dump_instrumentation_data_line_tools(PyCodeObject *code, uint8_t *line_tools, int i, FILE*out)
|
|
{
|
|
if (line_tools == NULL) {
|
|
fprintf(out, ", line_tools = NULL");
|
|
}
|
|
else {
|
|
fprintf(out, ", line_tools = %d", line_tools[i]);
|
|
}
|
|
}
|
|
|
|
static void
|
|
dump_instrumentation_data_per_instruction(PyCodeObject *code, _PyCoMonitoringData *data, int i, FILE*out)
|
|
{
|
|
if (data->per_instruction_opcodes == NULL) {
|
|
fprintf(out, ", per-inst opcode = NULL");
|
|
}
|
|
else {
|
|
fprintf(out, ", per-inst opcode = %s", _PyOpcode_OpName[data->per_instruction_opcodes[i]]);
|
|
}
|
|
if (data->per_instruction_tools == NULL) {
|
|
fprintf(out, ", per-inst tools = NULL");
|
|
}
|
|
else {
|
|
fprintf(out, ", per-inst tools = %d", data->per_instruction_tools[i]);
|
|
}
|
|
}
|
|
|
|
static void
|
|
dump_global_monitors(const char *prefix, _Py_GlobalMonitors monitors, FILE*out)
|
|
{
|
|
fprintf(out, "%s monitors:\n", prefix);
|
|
for (int event = 0; event < _PY_MONITORING_UNGROUPED_EVENTS; event++) {
|
|
fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]);
|
|
}
|
|
}
|
|
|
|
static void
|
|
dump_local_monitors(const char *prefix, _Py_LocalMonitors monitors, FILE*out)
|
|
{
|
|
fprintf(out, "%s monitors:\n", prefix);
|
|
for (int event = 0; event < _PY_MONITORING_LOCAL_EVENTS; event++) {
|
|
fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]);
|
|
}
|
|
}
|
|
|
|
/* No error checking -- Don't use this for anything but experimental debugging */
|
|
static void
|
|
dump_instrumentation_data(PyCodeObject *code, int star, FILE*out)
|
|
{
|
|
_PyCoMonitoringData *data = code->_co_monitoring;
|
|
fprintf(out, "\n");
|
|
PyObject_Print(code->co_name, out, Py_PRINT_RAW);
|
|
fprintf(out, "\n");
|
|
if (data == NULL) {
|
|
fprintf(out, "NULL\n");
|
|
return;
|
|
}
|
|
dump_global_monitors("Global", _PyInterpreterState_GET()->monitors, out);
|
|
dump_local_monitors("Code", data->local_monitors, out);
|
|
dump_local_monitors("Active", data->active_monitors, out);
|
|
int code_len = (int)Py_SIZE(code);
|
|
bool starred = false;
|
|
for (int i = 0; i < code_len; i += _PyInstruction_GetLength(code, i)) {
|
|
_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];
|
|
int opcode = instr->op.code;
|
|
if (i == star) {
|
|
fprintf(out, "** ");
|
|
starred = true;
|
|
}
|
|
fprintf(out, "Offset: %d, line: %d %s: ", i, PyCode_Addr2Line(code, i*2), _PyOpcode_OpName[opcode]);
|
|
dump_instrumentation_data_tools(code, data->tools, i, out);
|
|
dump_instrumentation_data_lines(code, data->lines, i, out);
|
|
dump_instrumentation_data_line_tools(code, data->line_tools, i, out);
|
|
dump_instrumentation_data_per_instruction(code, data, i, out);
|
|
fprintf(out, "\n");
|
|
;
|
|
}
|
|
if (!starred && star >= 0) {
|
|
fprintf(out, "Error offset not at valid instruction offset: %d\n", star);
|
|
fprintf(out, " ");
|
|
dump_instrumentation_data_tools(code, data->tools, star, out);
|
|
dump_instrumentation_data_lines(code, data->lines, star, out);
|
|
dump_instrumentation_data_line_tools(code, data->line_tools, star, out);
|
|
dump_instrumentation_data_per_instruction(code, data, star, out);
|
|
fprintf(out, "\n");
|
|
}
|
|
}
|
|
|
|
#define CHECK(test) do { \
|
|
if (!(test)) { \
|
|
dump_instrumentation_data(code, i, stderr); \
|
|
} \
|
|
assert(test); \
|
|
} while (0)
|
|
|
|
static bool
|
|
valid_opcode(int opcode)
|
|
{
|
|
if (opcode == INSTRUMENTED_LINE) {
|
|
return true;
|
|
}
|
|
if (IS_VALID_OPCODE(opcode) &&
|
|
opcode != CACHE &&
|
|
opcode != RESERVED &&
|
|
opcode < 255)
|
|
{
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static void
|
|
sanity_check_instrumentation(PyCodeObject *code)
|
|
{
|
|
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
|
|
|
|
_PyCoMonitoringData *data = code->_co_monitoring;
|
|
if (data == NULL) {
|
|
return;
|
|
}
|
|
_Py_GlobalMonitors global_monitors = _PyInterpreterState_GET()->monitors;
|
|
_Py_LocalMonitors active_monitors;
|
|
if (code->_co_monitoring) {
|
|
_Py_LocalMonitors local_monitors = code->_co_monitoring->local_monitors;
|
|
active_monitors = local_union(global_monitors, local_monitors);
|
|
}
|
|
else {
|
|
_Py_LocalMonitors empty = (_Py_LocalMonitors) { 0 };
|
|
active_monitors = local_union(global_monitors, empty);
|
|
}
|
|
assert(monitors_equals(
|
|
code->_co_monitoring->active_monitors,
|
|
active_monitors));
|
|
int code_len = (int)Py_SIZE(code);
|
|
for (int i = 0; i < code_len;) {
|
|
_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];
|
|
int opcode = instr->op.code;
|
|
int base_opcode = _Py_GetBaseCodeUnit(code, offset).op.code;
|
|
CHECK(valid_opcode(opcode));
|
|
CHECK(valid_opcode(base_opcode));
|
|
if (opcode == INSTRUMENTED_INSTRUCTION) {
|
|
opcode = data->per_instruction_opcodes[i];
|
|
if (!is_instrumented(opcode)) {
|
|
CHECK(_PyOpcode_Deopt[opcode] == opcode);
|
|
}
|
|
}
|
|
if (opcode == INSTRUMENTED_LINE) {
|
|
CHECK(data->lines);
|
|
CHECK(valid_opcode(data->lines[i].original_opcode));
|
|
opcode = data->lines[i].original_opcode;
|
|
CHECK(opcode != END_FOR);
|
|
CHECK(opcode != RESUME);
|
|
CHECK(opcode != RESUME_CHECK);
|
|
CHECK(opcode != INSTRUMENTED_RESUME);
|
|
if (!is_instrumented(opcode)) {
|
|
CHECK(_PyOpcode_Deopt[opcode] == opcode);
|
|
}
|
|
CHECK(opcode != INSTRUMENTED_LINE);
|
|
}
|
|
else if (data->lines) {
|
|
/* If original_opcode is INSTRUMENTED_INSTRUCTION
|
|
* *and* we are executing a INSTRUMENTED_LINE instruction
|
|
* that has de-instrumented itself, then we will execute
|
|
* an invalid INSTRUMENTED_INSTRUCTION */
|
|
CHECK(data->lines[i].original_opcode != INSTRUMENTED_INSTRUCTION);
|
|
}
|
|
if (opcode == INSTRUMENTED_INSTRUCTION) {
|
|
CHECK(data->per_instruction_opcodes[i] != 0);
|
|
opcode = data->per_instruction_opcodes[i];
|
|
}
|
|
if (is_instrumented(opcode)) {
|
|
CHECK(DE_INSTRUMENT[opcode] == base_opcode);
|
|
int event = EVENT_FOR_OPCODE[DE_INSTRUMENT[opcode]];
|
|
if (event < 0) {
|
|
/* RESUME fixup */
|
|
event = instr->op.arg ? 1: 0;
|
|
}
|
|
CHECK(active_monitors.tools[event] != 0);
|
|
}
|
|
if (data->lines && base_opcode != END_FOR) {
|
|
int line1 = compute_line(code, i, data->lines[i].line_delta);
|
|
int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT));
|
|
CHECK(line1 == line2);
|
|
}
|
|
CHECK(valid_opcode(opcode));
|
|
if (data->tools) {
|
|
uint8_t local_tools = data->tools[i];
|
|
if (opcode_has_event(base_opcode)) {
|
|
int event = EVENT_FOR_OPCODE[base_opcode];
|
|
if (event == -1) {
|
|
/* RESUME fixup */
|
|
event = _PyCode_CODE(code)[i].op.arg;
|
|
}
|
|
CHECK((active_monitors.tools[event] & local_tools) == local_tools);
|
|
}
|
|
else {
|
|
CHECK(local_tools == 0xff);
|
|
}
|
|
}
|
|
i += _PyInstruction_GetLength(code, i);
|
|
assert(i <= code_len);
|
|
}
|
|
}
|
|
#else
|
|
|
|
#define CHECK(test) assert(test)
|
|
|
|
#endif
|
|
|
|
/* Get the underlying code unit, stripping instrumentation and ENTER_EXECUTOR */
|
|
_Py_CODEUNIT
|
|
_Py_GetBaseCodeUnit(PyCodeObject *code, int i)
|
|
{
|
|
_Py_CODEUNIT *src_instr = _PyCode_CODE(code) + i;
|
|
_Py_CODEUNIT inst = {
|
|
.cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t *)src_instr)};
|
|
int opcode = inst.op.code;
|
|
if (opcode < MIN_INSTRUMENTED_OPCODE) {
|
|
inst.op.code = _PyOpcode_Deopt[opcode];
|
|
assert(inst.op.code <= RESUME);
|
|
return inst;
|
|
}
|
|
if (opcode == ENTER_EXECUTOR) {
|
|
_PyExecutorObject *exec = code->co_executors->executors[inst.op.arg];
|
|
opcode = _PyOpcode_Deopt[exec->vm_data.opcode];
|
|
inst.op.code = opcode;
|
|
assert(opcode <= RESUME);
|
|
inst.op.arg = exec->vm_data.oparg;
|
|
assert(inst.op.code <= RESUME);
|
|
return inst;
|
|
}
|
|
if (opcode == INSTRUMENTED_LINE) {
|
|
opcode = code->_co_monitoring->lines[i].original_opcode;
|
|
}
|
|
if (opcode == INSTRUMENTED_INSTRUCTION) {
|
|
opcode = code->_co_monitoring->per_instruction_opcodes[i];
|
|
}
|
|
CHECK(opcode != INSTRUMENTED_INSTRUCTION);
|
|
CHECK(opcode != INSTRUMENTED_LINE);
|
|
int deinstrumented = DE_INSTRUMENT[opcode];
|
|
if (deinstrumented) {
|
|
inst.op.code = deinstrumented;
|
|
}
|
|
else {
|
|
inst.op.code = _PyOpcode_Deopt[opcode];
|
|
}
|
|
assert(inst.op.code < MIN_SPECIALIZED_OPCODE);
|
|
return inst;
|
|
}
|
|
|
|
static void
|
|
de_instrument(_Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i,
|
|
int event)
|
|
{
|
|
assert(event != PY_MONITORING_EVENT_INSTRUCTION);
|
|
assert(event != PY_MONITORING_EVENT_LINE);
|
|
|
|
_Py_CODEUNIT *instr = &bytecode[i];
|
|
uint8_t *opcode_ptr = &instr->op.code;
|
|
int opcode = *opcode_ptr;
|
|
assert(opcode != ENTER_EXECUTOR);
|
|
if (opcode == INSTRUMENTED_LINE) {
|
|
opcode_ptr = &monitoring->lines[i].original_opcode;
|
|
opcode = *opcode_ptr;
|
|
}
|
|
if (opcode == INSTRUMENTED_INSTRUCTION) {
|
|
opcode_ptr = &monitoring->per_instruction_opcodes[i];
|
|
opcode = *opcode_ptr;
|
|
}
|
|
int deinstrumented = DE_INSTRUMENT[opcode];
|
|
if (deinstrumented == 0) {
|
|
return;
|
|
}
|
|
CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented);
|
|
FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, deinstrumented);
|
|
if (_PyOpcode_Caches[deinstrumented]) {
|
|
FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff,
|
|
adaptive_counter_warmup().value_and_backoff);
|
|
}
|
|
}
|
|
|
|
static void
|
|
de_instrument_line(_Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring,
|
|
int i)
|
|
{
|
|
_Py_CODEUNIT *instr = &bytecode[i];
|
|
int opcode = instr->op.code;
|
|
if (opcode != INSTRUMENTED_LINE) {
|
|
return;
|
|
}
|
|
_PyCoLineInstrumentationData *lines = &monitoring->lines[i];
|
|
int original_opcode = lines->original_opcode;
|
|
if (original_opcode == INSTRUMENTED_INSTRUCTION) {
|
|
lines->original_opcode = monitoring->per_instruction_opcodes[i];
|
|
}
|
|
CHECK(original_opcode != 0);
|
|
CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]);
|
|
FT_ATOMIC_STORE_UINT8(instr->op.code, original_opcode);
|
|
if (_PyOpcode_Caches[original_opcode]) {
|
|
FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff,
|
|
adaptive_counter_warmup().value_and_backoff);
|
|
}
|
|
assert(instr->op.code != INSTRUMENTED_LINE);
|
|
}
|
|
|
|
static void
|
|
de_instrument_per_instruction(_Py_CODEUNIT *bytecode,
|
|
_PyCoMonitoringData *monitoring, int i)
|
|
{
|
|
_Py_CODEUNIT *instr = &bytecode[i];
|
|
uint8_t *opcode_ptr = &instr->op.code;
|
|
int opcode = *opcode_ptr;
|
|
if (opcode == INSTRUMENTED_LINE) {
|
|
opcode_ptr = &monitoring->lines[i].original_opcode;
|
|
opcode = *opcode_ptr;
|
|
}
|
|
if (opcode != INSTRUMENTED_INSTRUCTION) {
|
|
return;
|
|
}
|
|
int original_opcode = monitoring->per_instruction_opcodes[i];
|
|
CHECK(original_opcode != 0);
|
|
CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]);
|
|
FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, original_opcode);
|
|
if (_PyOpcode_Caches[original_opcode]) {
|
|
FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff,
|
|
adaptive_counter_warmup().value_and_backoff);
|
|
}
|
|
assert(*opcode_ptr != INSTRUMENTED_INSTRUCTION);
|
|
assert(instr->op.code != INSTRUMENTED_INSTRUCTION);
|
|
}
|
|
|
|
static void
|
|
instrument(_Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i)
|
|
{
|
|
_Py_CODEUNIT *instr = &bytecode[i];
|
|
uint8_t *opcode_ptr = &instr->op.code;
|
|
int opcode =*opcode_ptr;
|
|
if (opcode == INSTRUMENTED_LINE) {
|
|
_PyCoLineInstrumentationData *lines = &monitoring->lines[i];
|
|
opcode_ptr = &lines->original_opcode;
|
|
opcode = *opcode_ptr;
|
|
}
|
|
if (opcode == INSTRUMENTED_INSTRUCTION) {
|
|
opcode_ptr = &monitoring->per_instruction_opcodes[i];
|
|
opcode = *opcode_ptr;
|
|
CHECK(opcode != INSTRUMENTED_INSTRUCTION && opcode != INSTRUMENTED_LINE);
|
|
CHECK(opcode == _PyOpcode_Deopt[opcode]);
|
|
}
|
|
CHECK(opcode != 0);
|
|
if (!is_instrumented(opcode)) {
|
|
int deopt = _PyOpcode_Deopt[opcode];
|
|
int instrumented = INSTRUMENTED_OPCODES[deopt];
|
|
assert(instrumented);
|
|
FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, instrumented);
|
|
if (_PyOpcode_Caches[deopt]) {
|
|
FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff,
|
|
adaptive_counter_warmup().value_and_backoff);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void
|
|
instrument_line(_Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i)
|
|
{
|
|
uint8_t *opcode_ptr = &bytecode[i].op.code;
|
|
int opcode = *opcode_ptr;
|
|
if (opcode == INSTRUMENTED_LINE) {
|
|
return;
|
|
}
|
|
_PyCoLineInstrumentationData *lines = &monitoring->lines[i];
|
|
lines->original_opcode = _PyOpcode_Deopt[opcode];
|
|
CHECK(lines->original_opcode > 0);
|
|
FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, INSTRUMENTED_LINE);
|
|
}
|
|
|
|
static void
|
|
instrument_per_instruction(_Py_CODEUNIT *bytecode,
|
|
_PyCoMonitoringData *monitoring, int i)
|
|
{
|
|
_Py_CODEUNIT *instr = &bytecode[i];
|
|
uint8_t *opcode_ptr = &instr->op.code;
|
|
int opcode = *opcode_ptr;
|
|
if (opcode == INSTRUMENTED_LINE) {
|
|
_PyCoLineInstrumentationData *lines = &monitoring->lines[i];
|
|
opcode_ptr = &lines->original_opcode;
|
|
opcode = *opcode_ptr;
|
|
}
|
|
if (opcode == INSTRUMENTED_INSTRUCTION) {
|
|
assert(monitoring->per_instruction_opcodes[i] > 0);
|
|
return;
|
|
}
|
|
CHECK(opcode != 0);
|
|
if (is_instrumented(opcode)) {
|
|
monitoring->per_instruction_opcodes[i] = opcode;
|
|
}
|
|
else {
|
|
assert(opcode != 0);
|
|
assert(_PyOpcode_Deopt[opcode] != 0);
|
|
assert(_PyOpcode_Deopt[opcode] != RESUME);
|
|
monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode];
|
|
}
|
|
assert(monitoring->per_instruction_opcodes[i] > 0);
|
|
FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, INSTRUMENTED_INSTRUCTION);
|
|
}
|
|
|
|
static void
|
|
remove_tools(PyCodeObject * code, int offset, int event, int tools)
|
|
{
|
|
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
|
|
assert(event != PY_MONITORING_EVENT_LINE);
|
|
assert(event != PY_MONITORING_EVENT_INSTRUCTION);
|
|
assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event));
|
|
assert(opcode_has_event(_Py_GetBaseCodeUnit(code, offset).op.code));
|
|
_PyCoMonitoringData *monitoring = code->_co_monitoring;
|
|
bool should_de_instrument;
|
|
if (monitoring && monitoring->tools) {
|
|
monitoring->tools[offset] &= ~tools;
|
|
should_de_instrument = (monitoring->tools[offset] == 0);
|
|
}
|
|
else {
|
|
/* Single tool */
|
|
uint8_t single_tool = code->_co_monitoring->active_monitors.tools[event];
|
|
assert(_Py_popcount32(single_tool) <= 1);
|
|
should_de_instrument = ((single_tool & tools) == single_tool);
|
|
}
|
|
if (should_de_instrument) {
|
|
MODIFY_BYTECODE(code, de_instrument, monitoring, offset, event);
|
|
}
|
|
}
|
|
|
|
#ifndef NDEBUG
|
|
static bool
|
|
tools_is_subset_for_event(PyCodeObject * code, int event, int tools)
|
|
{
|
|
int global_tools = _PyInterpreterState_GET()->monitors.tools[event];
|
|
int local_tools = code->_co_monitoring->local_monitors.tools[event];
|
|
return tools == ((global_tools | local_tools) & tools);
|
|
}
|
|
#endif
|
|
|
|
static void
|
|
remove_line_tools(PyCodeObject * code, int offset, int tools)
|
|
{
|
|
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
|
|
|
|
_PyCoMonitoringData *monitoring = code->_co_monitoring;
|
|
assert(monitoring);
|
|
bool should_de_instrument;
|
|
if (monitoring->line_tools)
|
|
{
|
|
uint8_t *toolsptr = &monitoring->line_tools[offset];
|
|
*toolsptr &= ~tools;
|
|
should_de_instrument = (*toolsptr == 0);
|
|
}
|
|
else {
|
|
/* Single tool */
|
|
uint8_t single_tool = monitoring->active_monitors.tools[PY_MONITORING_EVENT_LINE];
|
|
assert(_Py_popcount32(single_tool) <= 1);
|
|
should_de_instrument = ((single_tool & tools) == single_tool);
|
|
}
|
|
if (should_de_instrument) {
|
|
MODIFY_BYTECODE(code, de_instrument_line, monitoring, offset);
|
|
}
|
|
}
|
|
|
|
static void
|
|
add_tools(PyCodeObject * code, int offset, int event, int tools)
|
|
{
|
|
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
|
|
assert(event != PY_MONITORING_EVENT_LINE);
|
|
assert(event != PY_MONITORING_EVENT_INSTRUCTION);
|
|
assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event));
|
|
assert(code->_co_monitoring);
|
|
if (code->_co_monitoring &&
|
|
code->_co_monitoring->tools
|
|
) {
|
|
code->_co_monitoring->tools[offset] |= tools;
|
|
}
|
|
else {
|
|
/* Single tool */
|
|
assert(_Py_popcount32(tools) == 1);
|
|
assert(tools_is_subset_for_event(code, event, tools));
|
|
}
|
|
MODIFY_BYTECODE(code, instrument, code->_co_monitoring, offset);
|
|
}
|
|
|
|
static void
|
|
add_line_tools(PyCodeObject * code, int offset, int tools)
|
|
{
|
|
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
|
|
|
|
assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_LINE, tools));
|
|
assert(code->_co_monitoring);
|
|
if (code->_co_monitoring->line_tools) {
|
|
code->_co_monitoring->line_tools[offset] |= tools;
|
|
}
|
|
else {
|
|
/* Single tool */
|
|
assert(_Py_popcount32(tools) == 1);
|
|
}
|
|
MODIFY_BYTECODE(code, instrument_line, code->_co_monitoring, offset);
|
|
}
|
|
|
|
|
|
static void
|
|
add_per_instruction_tools(PyCodeObject * code, int offset, int tools)
|
|
{
|
|
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
|
|
|
|
assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_INSTRUCTION, tools));
|
|
assert(code->_co_monitoring);
|
|
if (code->_co_monitoring->per_instruction_tools) {
|
|
code->_co_monitoring->per_instruction_tools[offset] |= tools;
|
|
}
|
|
else {
|
|
/* Single tool */
|
|
assert(_Py_popcount32(tools) == 1);
|
|
}
|
|
MODIFY_BYTECODE(code, instrument_per_instruction, code->_co_monitoring, offset);
|
|
}
|
|
|
|
|
|
static void
|
|
remove_per_instruction_tools(PyCodeObject * code, int offset, int tools)
|
|
{
|
|
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
|
|
|
|
_PyCoMonitoringData *monitoring = code->_co_monitoring;
|
|
assert(code->_co_monitoring);
|
|
bool should_de_instrument;
|
|
if (code->_co_monitoring->per_instruction_tools) {
|
|
uint8_t *toolsptr = &code->_co_monitoring->per_instruction_tools[offset];
|
|
*toolsptr &= ~tools;
|
|
should_de_instrument = (*toolsptr == 0);
|
|
}
|
|
else {
|
|
/* Single tool */
|
|
uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION];
|
|
assert(_Py_popcount32(single_tool) <= 1);
|
|
should_de_instrument = ((single_tool & tools) == single_tool);
|
|
}
|
|
if (should_de_instrument) {
|
|
MODIFY_BYTECODE(code, de_instrument_per_instruction, monitoring, offset);
|
|
}
|
|
}
|
|
|
|
|
|
/* Return 1 if DISABLE returned, -1 if error, 0 otherwise */
|
|
static int
|
|
call_one_instrument(
|
|
PyInterpreterState *interp, PyThreadState *tstate, PyObject **args,
|
|
size_t nargsf, int8_t tool, int event)
|
|
{
|
|
assert(0 <= tool && tool < 8);
|
|
assert(tstate->tracing == 0);
|
|
PyObject *instrument = interp->monitoring_callables[tool][event];
|
|
if (instrument == NULL) {
|
|
return 0;
|
|
}
|
|
int old_what = tstate->what_event;
|
|
tstate->what_event = event;
|
|
tstate->tracing++;
|
|
PyObject *res = _PyObject_VectorcallTstate(tstate, instrument, args, nargsf, NULL);
|
|
tstate->tracing--;
|
|
tstate->what_event = old_what;
|
|
if (res == NULL) {
|
|
return -1;
|
|
}
|
|
Py_DECREF(res);
|
|
return (res == &_PyInstrumentation_DISABLE);
|
|
}
|
|
|
|
static const int8_t MOST_SIGNIFICANT_BITS[16] = {
|
|
-1, 0, 1, 1,
|
|
2, 2, 2, 2,
|
|
3, 3, 3, 3,
|
|
3, 3, 3, 3,
|
|
};
|
|
|
|
/* We could use _Py_bit_length here, but that is designed for larger (32/64)
|
|
* bit ints, and can perform relatively poorly on platforms without the
|
|
* necessary intrinsics. */
|
|
static inline int most_significant_bit(uint8_t bits) {
|
|
assert(bits != 0);
|
|
if (bits > 15) {
|
|
return MOST_SIGNIFICANT_BITS[bits>>4]+4;
|
|
}
|
|
return MOST_SIGNIFICANT_BITS[bits];
|
|
}
|
|
|
|
static uint32_t
|
|
global_version(PyInterpreterState *interp)
|
|
{
|
|
uint32_t version = (uint32_t)_Py_atomic_load_uintptr_relaxed(
|
|
&interp->ceval.instrumentation_version);
|
|
#ifdef Py_DEBUG
|
|
PyThreadState *tstate = _PyThreadState_GET();
|
|
uint32_t thread_version =
|
|
(uint32_t)(_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) &
|
|
~_PY_EVAL_EVENTS_MASK);
|
|
assert(thread_version == version);
|
|
#endif
|
|
return version;
|
|
}
|
|
|
|
/* Atomically set the given version in the given location, without touching
|
|
anything in _PY_EVAL_EVENTS_MASK. */
|
|
static void
|
|
set_version_raw(uintptr_t *ptr, uint32_t version)
|
|
{
|
|
uintptr_t old = _Py_atomic_load_uintptr_relaxed(ptr);
|
|
uintptr_t new;
|
|
do {
|
|
new = (old & _PY_EVAL_EVENTS_MASK) | version;
|
|
} while (!_Py_atomic_compare_exchange_uintptr(ptr, &old, new));
|
|
}
|
|
|
|
static void
|
|
set_global_version(PyThreadState *tstate, uint32_t version)
|
|
{
|
|
assert((version & _PY_EVAL_EVENTS_MASK) == 0);
|
|
PyInterpreterState *interp = tstate->interp;
|
|
set_version_raw(&interp->ceval.instrumentation_version, version);
|
|
|
|
#ifdef Py_GIL_DISABLED
|
|
// Set the version on all threads in free-threaded builds.
|
|
_PyRuntimeState *runtime = &_PyRuntime;
|
|
HEAD_LOCK(runtime);
|
|
for (tstate = interp->threads.head; tstate;
|
|
tstate = PyThreadState_Next(tstate)) {
|
|
set_version_raw(&tstate->eval_breaker, version);
|
|
};
|
|
HEAD_UNLOCK(runtime);
|
|
#else
|
|
// Normal builds take the current version from instrumentation_version when
|
|
// attaching a thread, so we only have to set the current thread's version.
|
|
set_version_raw(&tstate->eval_breaker, version);
|
|
#endif
|
|
}
|
|
|
|
static bool
|
|
is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp)
|
|
{
|
|
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
|
|
return global_version(interp) == code->_co_instrumentation_version;
|
|
}
|
|
|
|
#ifndef NDEBUG
|
|
static bool
|
|
instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code)
|
|
{
|
|
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
|
|
_Py_LocalMonitors expected = local_union(
|
|
interp->monitors,
|
|
code->_co_monitoring->local_monitors);
|
|
return monitors_equals(code->_co_monitoring->active_monitors, expected);
|
|
}
|
|
|
|
static int
|
|
debug_check_sanity(PyInterpreterState *interp, PyCodeObject *code)
|
|
{
|
|
int res;
|
|
LOCK_CODE(code);
|
|
res = is_version_up_to_date(code, interp) &&
|
|
instrumentation_cross_checks(interp, code);
|
|
UNLOCK_CODE();
|
|
return res;
|
|
}
|
|
|
|
#endif
|
|
|
|
static inline uint8_t
|
|
get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i, int event)
|
|
{
|
|
uint8_t tools;
|
|
assert(event != PY_MONITORING_EVENT_LINE);
|
|
assert(event != PY_MONITORING_EVENT_INSTRUCTION);
|
|
if (event >= _PY_MONITORING_UNGROUPED_EVENTS) {
|
|
assert(event == PY_MONITORING_EVENT_C_RAISE ||
|
|
event == PY_MONITORING_EVENT_C_RETURN);
|
|
event = PY_MONITORING_EVENT_CALL;
|
|
}
|
|
if (PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) {
|
|
CHECK(debug_check_sanity(interp, code));
|
|
if (code->_co_monitoring->tools) {
|
|
tools = code->_co_monitoring->tools[i];
|
|
}
|
|
else {
|
|
tools = code->_co_monitoring->active_monitors.tools[event];
|
|
}
|
|
}
|
|
else {
|
|
tools = interp->monitors.tools[event];
|
|
}
|
|
return tools;
|
|
}
|
|
|
|
static const char *const event_names [] = {
|
|
[PY_MONITORING_EVENT_PY_START] = "PY_START",
|
|
[PY_MONITORING_EVENT_PY_RESUME] = "PY_RESUME",
|
|
[PY_MONITORING_EVENT_PY_RETURN] = "PY_RETURN",
|
|
[PY_MONITORING_EVENT_PY_YIELD] = "PY_YIELD",
|
|
[PY_MONITORING_EVENT_CALL] = "CALL",
|
|
[PY_MONITORING_EVENT_LINE] = "LINE",
|
|
[PY_MONITORING_EVENT_INSTRUCTION] = "INSTRUCTION",
|
|
[PY_MONITORING_EVENT_JUMP] = "JUMP",
|
|
[PY_MONITORING_EVENT_BRANCH] = "BRANCH",
|
|
[PY_MONITORING_EVENT_C_RETURN] = "C_RETURN",
|
|
[PY_MONITORING_EVENT_PY_THROW] = "PY_THROW",
|
|
[PY_MONITORING_EVENT_RAISE] = "RAISE",
|
|
[PY_MONITORING_EVENT_RERAISE] = "RERAISE",
|
|
[PY_MONITORING_EVENT_EXCEPTION_HANDLED] = "EXCEPTION_HANDLED",
|
|
[PY_MONITORING_EVENT_C_RAISE] = "C_RAISE",
|
|
[PY_MONITORING_EVENT_PY_UNWIND] = "PY_UNWIND",
|
|
[PY_MONITORING_EVENT_STOP_ITERATION] = "STOP_ITERATION",
|
|
};
|
|
|
|
static int
|
|
call_instrumentation_vector(
|
|
PyThreadState *tstate, int event,
|
|
_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargs, PyObject *args[])
|
|
{
|
|
if (tstate->tracing) {
|
|
return 0;
|
|
}
|
|
assert(!_PyErr_Occurred(tstate));
|
|
assert(args[0] == NULL);
|
|
PyCodeObject *code = _PyFrame_GetCode(frame);
|
|
assert(args[1] == NULL);
|
|
args[1] = (PyObject *)code;
|
|
int offset = (int)(instr - _PyFrame_GetBytecode(frame));
|
|
/* Offset visible to user should be the offset in bytes, as that is the
|
|
* convention for APIs involving code offsets. */
|
|
int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT);
|
|
PyObject *offset_obj = PyLong_FromLong(bytes_offset);
|
|
if (offset_obj == NULL) {
|
|
return -1;
|
|
}
|
|
assert(args[2] == NULL);
|
|
args[2] = offset_obj;
|
|
PyInterpreterState *interp = tstate->interp;
|
|
uint8_t tools = get_tools_for_instruction(code, interp, offset, event);
|
|
size_t nargsf = (size_t) nargs | PY_VECTORCALL_ARGUMENTS_OFFSET;
|
|
PyObject **callargs = &args[1];
|
|
int err = 0;
|
|
while (tools) {
|
|
int tool = most_significant_bit(tools);
|
|
assert(tool >= 0 && tool < 8);
|
|
assert(tools & (1 << tool));
|
|
tools ^= (1 << tool);
|
|
int res = call_one_instrument(interp, tstate, callargs, nargsf, tool, event);
|
|
if (res == 0) {
|
|
/* Nothing to do */
|
|
}
|
|
else if (res < 0) {
|
|
/* error */
|
|
err = -1;
|
|
break;
|
|
}
|
|
else {
|
|
/* DISABLE */
|
|
if (!PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) {
|
|
PyErr_Format(PyExc_ValueError,
|
|
"Cannot disable %s events. Callback removed.",
|
|
event_names[event]);
|
|
/* Clear tool to prevent infinite loop */
|
|
Py_CLEAR(interp->monitoring_callables[tool][event]);
|
|
err = -1;
|
|
break;
|
|
}
|
|
else {
|
|
LOCK_CODE(code);
|
|
remove_tools(code, offset, event, 1 << tool);
|
|
UNLOCK_CODE();
|
|
}
|
|
}
|
|
}
|
|
Py_DECREF(offset_obj);
|
|
return err;
|
|
}
|
|
|
|
int
|
|
_Py_call_instrumentation(
|
|
PyThreadState *tstate, int event,
|
|
_PyInterpreterFrame *frame, _Py_CODEUNIT *instr)
|
|
{
|
|
PyObject *args[3] = { NULL, NULL, NULL };
|
|
return call_instrumentation_vector(tstate, event, frame, instr, 2, args);
|
|
}
|
|
|
|
int
|
|
_Py_call_instrumentation_arg(
|
|
PyThreadState *tstate, int event,
|
|
_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg)
|
|
{
|
|
PyObject *args[4] = { NULL, NULL, NULL, arg };
|
|
return call_instrumentation_vector(tstate, event, frame, instr, 3, args);
|
|
}
|
|
|
|
int
|
|
_Py_call_instrumentation_2args(
|
|
PyThreadState *tstate, int event,
|
|
_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1)
|
|
{
|
|
PyObject *args[5] = { NULL, NULL, NULL, arg0, arg1 };
|
|
return call_instrumentation_vector(tstate, event, frame, instr, 4, args);
|
|
}
|
|
|
|
_Py_CODEUNIT *
|
|
_Py_call_instrumentation_jump(
|
|
PyThreadState *tstate, int event,
|
|
_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target)
|
|
{
|
|
assert(event == PY_MONITORING_EVENT_JUMP ||
|
|
event == PY_MONITORING_EVENT_BRANCH);
|
|
assert(frame->instr_ptr == instr);
|
|
int to = (int)(target - _PyFrame_GetBytecode(frame));
|
|
PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT));
|
|
if (to_obj == NULL) {
|
|
return NULL;
|
|
}
|
|
PyObject *args[4] = { NULL, NULL, NULL, to_obj };
|
|
int err = call_instrumentation_vector(tstate, event, frame, instr, 3, args);
|
|
Py_DECREF(to_obj);
|
|
if (err) {
|
|
return NULL;
|
|
}
|
|
if (frame->instr_ptr != instr) {
|
|
/* The callback has caused a jump (by setting the line number) */
|
|
return frame->instr_ptr;
|
|
}
|
|
return target;
|
|
}
|
|
|
|
static void
|
|
call_instrumentation_vector_protected(
|
|
PyThreadState *tstate, int event,
|
|
_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargs, PyObject *args[])
|
|
{
|
|
assert(_PyErr_Occurred(tstate));
|
|
PyObject *exc = _PyErr_GetRaisedException(tstate);
|
|
int err = call_instrumentation_vector(tstate, event, frame, instr, nargs, args);
|
|
if (err) {
|
|
Py_XDECREF(exc);
|
|
}
|
|
else {
|
|
_PyErr_SetRaisedException(tstate, exc);
|
|
}
|
|
assert(_PyErr_Occurred(tstate));
|
|
}
|
|
|
|
void
|
|
_Py_call_instrumentation_exc2(
|
|
PyThreadState *tstate, int event,
|
|
_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1)
|
|
{
|
|
assert(_PyErr_Occurred(tstate));
|
|
PyObject *args[5] = { NULL, NULL, NULL, arg0, arg1 };
|
|
call_instrumentation_vector_protected(tstate, event, frame, instr, 4, args);
|
|
}
|
|
|
|
|
|
int
|
|
_Py_Instrumentation_GetLine(PyCodeObject *code, int index)
|
|
{
|
|
_PyCoMonitoringData *monitoring = code->_co_monitoring;
|
|
assert(monitoring != NULL);
|
|
assert(monitoring->lines != NULL);
|
|
assert(index >= code->_co_firsttraceable);
|
|
assert(index < Py_SIZE(code));
|
|
_PyCoLineInstrumentationData *line_data = &monitoring->lines[index];
|
|
int8_t line_delta = line_data->line_delta;
|
|
int line = compute_line(code, index, line_delta);
|
|
return line;
|
|
}
|
|
|
|
int
|
|
_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev)
|
|
{
|
|
PyCodeObject *code = _PyFrame_GetCode(frame);
|
|
assert(tstate->tracing == 0);
|
|
assert(debug_check_sanity(tstate->interp, code));
|
|
_Py_CODEUNIT *bytecode = _PyFrame_GetBytecode(frame);
|
|
int i = (int)(instr - bytecode);
|
|
|
|
_PyCoMonitoringData *monitoring = code->_co_monitoring;
|
|
_PyCoLineInstrumentationData *line_data = &monitoring->lines[i];
|
|
PyInterpreterState *interp = tstate->interp;
|
|
int8_t line_delta = line_data->line_delta;
|
|
int line = 0;
|
|
|
|
if (line_delta == COMPUTED_LINE_LINENO_CHANGE) {
|
|
// We know the line number must have changed, don't need to calculate
|
|
// the line number for now because we might not need it.
|
|
line = -1;
|
|
} else {
|
|
line = compute_line(code, i, line_delta);
|
|
assert(line >= 0);
|
|
assert(prev != NULL);
|
|
int prev_index = (int)(prev - bytecode);
|
|
int prev_line = _Py_Instrumentation_GetLine(code, prev_index);
|
|
if (prev_line == line) {
|
|
int prev_opcode = bytecode[prev_index].op.code;
|
|
/* RESUME and INSTRUMENTED_RESUME are needed for the operation of
|
|
* instrumentation, so must never be hidden by an INSTRUMENTED_LINE.
|
|
*/
|
|
if (prev_opcode != RESUME && prev_opcode != INSTRUMENTED_RESUME) {
|
|
goto done;
|
|
}
|
|
}
|
|
}
|
|
|
|
uint8_t tools = code->_co_monitoring->line_tools != NULL ?
|
|
code->_co_monitoring->line_tools[i] :
|
|
(interp->monitors.tools[PY_MONITORING_EVENT_LINE] |
|
|
code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_LINE]
|
|
);
|
|
/* Special case sys.settrace to avoid boxing the line number,
|
|
* only to immediately unbox it. */
|
|
if (tools & (1 << PY_MONITORING_SYS_TRACE_ID)) {
|
|
if (tstate->c_tracefunc != NULL) {
|
|
PyFrameObject *frame_obj = _PyFrame_GetFrameObject(frame);
|
|
if (frame_obj == NULL) {
|
|
return -1;
|
|
}
|
|
if (frame_obj->f_trace_lines) {
|
|
/* Need to set tracing and what_event as if using
|
|
* the instrumentation call. */
|
|
int old_what = tstate->what_event;
|
|
tstate->what_event = PY_MONITORING_EVENT_LINE;
|
|
tstate->tracing++;
|
|
/* Call c_tracefunc directly, having set the line number. */
|
|
Py_INCREF(frame_obj);
|
|
if (line == -1 && line_delta > COMPUTED_LINE) {
|
|
/* Only assign f_lineno if it's easy to calculate, otherwise
|
|
* do lazy calculation by setting the f_lineno to 0.
|
|
*/
|
|
line = compute_line(code, i, line_delta);
|
|
}
|
|
frame_obj->f_lineno = line;
|
|
int err = tstate->c_tracefunc(tstate->c_traceobj, frame_obj, PyTrace_LINE, Py_None);
|
|
frame_obj->f_lineno = 0;
|
|
tstate->tracing--;
|
|
tstate->what_event = old_what;
|
|
Py_DECREF(frame_obj);
|
|
if (err) {
|
|
return -1;
|
|
}
|
|
}
|
|
}
|
|
tools &= (255 - (1 << PY_MONITORING_SYS_TRACE_ID));
|
|
}
|
|
if (tools == 0) {
|
|
goto done;
|
|
}
|
|
|
|
if (line == -1) {
|
|
/* Need to calculate the line number now for monitoring events */
|
|
line = compute_line(code, i, line_delta);
|
|
}
|
|
PyObject *line_obj = PyLong_FromLong(line);
|
|
if (line_obj == NULL) {
|
|
return -1;
|
|
}
|
|
PyObject *args[3] = { NULL, (PyObject *)code, line_obj };
|
|
do {
|
|
int tool = most_significant_bit(tools);
|
|
assert(tool >= 0 && tool < PY_MONITORING_SYS_PROFILE_ID);
|
|
assert(tools & (1 << tool));
|
|
tools &= ~(1 << tool);
|
|
int res = call_one_instrument(interp, tstate, &args[1],
|
|
2 | PY_VECTORCALL_ARGUMENTS_OFFSET,
|
|
tool, PY_MONITORING_EVENT_LINE);
|
|
if (res == 0) {
|
|
/* Nothing to do */
|
|
}
|
|
else if (res < 0) {
|
|
/* error */
|
|
Py_DECREF(line_obj);
|
|
return -1;
|
|
}
|
|
else {
|
|
/* DISABLE */
|
|
LOCK_CODE(code);
|
|
remove_line_tools(code, i, 1 << tool);
|
|
UNLOCK_CODE();
|
|
}
|
|
} while (tools);
|
|
Py_DECREF(line_obj);
|
|
uint8_t original_opcode;
|
|
done:
|
|
original_opcode = line_data->original_opcode;
|
|
assert(original_opcode != 0);
|
|
assert(original_opcode != INSTRUMENTED_LINE);
|
|
assert(_PyOpcode_Deopt[original_opcode] == original_opcode);
|
|
return original_opcode;
|
|
}
|
|
|
|
int
|
|
_Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr)
|
|
{
|
|
PyCodeObject *code = _PyFrame_GetCode(frame);
|
|
int offset = (int)(instr - _PyFrame_GetBytecode(frame));
|
|
_PyCoMonitoringData *instrumentation_data = code->_co_monitoring;
|
|
assert(instrumentation_data->per_instruction_opcodes);
|
|
int next_opcode = instrumentation_data->per_instruction_opcodes[offset];
|
|
if (tstate->tracing) {
|
|
return next_opcode;
|
|
}
|
|
assert(debug_check_sanity(tstate->interp, code));
|
|
PyInterpreterState *interp = tstate->interp;
|
|
uint8_t tools = instrumentation_data->per_instruction_tools != NULL ?
|
|
instrumentation_data->per_instruction_tools[offset] :
|
|
(interp->monitors.tools[PY_MONITORING_EVENT_INSTRUCTION] |
|
|
code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION]
|
|
);
|
|
int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT);
|
|
PyObject *offset_obj = PyLong_FromLong(bytes_offset);
|
|
if (offset_obj == NULL) {
|
|
return -1;
|
|
}
|
|
PyObject *args[3] = { NULL, (PyObject *)code, offset_obj };
|
|
while (tools) {
|
|
int tool = most_significant_bit(tools);
|
|
assert(tool >= 0 && tool < 8);
|
|
assert(tools & (1 << tool));
|
|
tools &= ~(1 << tool);
|
|
int res = call_one_instrument(interp, tstate, &args[1],
|
|
2 | PY_VECTORCALL_ARGUMENTS_OFFSET,
|
|
tool, PY_MONITORING_EVENT_INSTRUCTION);
|
|
if (res == 0) {
|
|
/* Nothing to do */
|
|
}
|
|
else if (res < 0) {
|
|
/* error */
|
|
Py_DECREF(offset_obj);
|
|
return -1;
|
|
}
|
|
else {
|
|
/* DISABLE */
|
|
LOCK_CODE(code);
|
|
remove_per_instruction_tools(code, offset, 1 << tool);
|
|
UNLOCK_CODE();
|
|
}
|
|
}
|
|
Py_DECREF(offset_obj);
|
|
assert(next_opcode != 0);
|
|
return next_opcode;
|
|
}
|
|
|
|
|
|
PyObject *
|
|
_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj)
|
|
{
|
|
PyInterpreterState *is = _PyInterpreterState_GET();
|
|
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
|
|
assert(0 <= event_id && event_id < _PY_MONITORING_EVENTS);
|
|
PyObject *callback = _Py_atomic_exchange_ptr(&is->monitoring_callables[tool_id][event_id],
|
|
Py_XNewRef(obj));
|
|
|
|
return callback;
|
|
}
|
|
|
|
static void
|
|
initialize_tools(PyCodeObject *code)
|
|
{
|
|
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
|
|
uint8_t* tools = code->_co_monitoring->tools;
|
|
|
|
assert(tools != NULL);
|
|
int code_len = (int)Py_SIZE(code);
|
|
for (int i = 0; i < code_len; i++) {
|
|
_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];
|
|
int opcode = instr->op.code;
|
|
assert(opcode != ENTER_EXECUTOR);
|
|
if (opcode == INSTRUMENTED_LINE) {
|
|
opcode = code->_co_monitoring->lines[i].original_opcode;
|
|
}
|
|
if (opcode == INSTRUMENTED_INSTRUCTION) {
|
|
opcode = code->_co_monitoring->per_instruction_opcodes[i];
|
|
}
|
|
bool instrumented = is_instrumented(opcode);
|
|
if (instrumented) {
|
|
opcode = DE_INSTRUMENT[opcode];
|
|
assert(opcode != 0);
|
|
}
|
|
opcode = _PyOpcode_Deopt[opcode];
|
|
if (opcode_has_event(opcode)) {
|
|
if (instrumented) {
|
|
int8_t event;
|
|
if (opcode == RESUME) {
|
|
event = instr->op.arg != 0;
|
|
}
|
|
else {
|
|
event = EVENT_FOR_OPCODE[opcode];
|
|
assert(event > 0);
|
|
}
|
|
assert(event >= 0);
|
|
assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event));
|
|
tools[i] = code->_co_monitoring->active_monitors.tools[event];
|
|
CHECK(tools[i] != 0);
|
|
}
|
|
else {
|
|
tools[i] = 0;
|
|
}
|
|
}
|
|
#ifdef Py_DEBUG
|
|
/* Initialize tools for invalid locations to all ones to try to catch errors */
|
|
else {
|
|
tools[i] = 0xff;
|
|
}
|
|
for (int j = 1; j <= _PyOpcode_Caches[opcode]; j++) {
|
|
tools[i+j] = 0xff;
|
|
}
|
|
#endif
|
|
i += _PyOpcode_Caches[opcode];
|
|
}
|
|
}
|
|
|
|
#define NO_LINE -128
|
|
|
|
static void
|
|
initialize_lines(PyCodeObject *code)
|
|
{
|
|
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
|
|
_PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines;
|
|
|
|
assert(line_data != NULL);
|
|
int code_len = (int)Py_SIZE(code);
|
|
PyCodeAddressRange range;
|
|
_PyCode_InitAddressRange(code, &range);
|
|
for (int i = 0; i < code->_co_firsttraceable && i < code_len; i++) {
|
|
line_data[i].original_opcode = 0;
|
|
line_data[i].line_delta = -127;
|
|
}
|
|
int current_line = -1;
|
|
for (int i = code->_co_firsttraceable; i < code_len; ) {
|
|
int opcode = _Py_GetBaseCodeUnit(code, i).op.code;
|
|
int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range);
|
|
line_data[i].line_delta = compute_line_delta(code, i, line);
|
|
int length = _PyInstruction_GetLength(code, i);
|
|
switch (opcode) {
|
|
case END_ASYNC_FOR:
|
|
case END_FOR:
|
|
case END_SEND:
|
|
case RESUME:
|
|
/* END_FOR cannot start a line, as it is skipped by FOR_ITER
|
|
* END_SEND cannot start a line, as it is skipped by SEND
|
|
* RESUME must not be instrumented with INSTRUMENT_LINE */
|
|
line_data[i].original_opcode = 0;
|
|
break;
|
|
default:
|
|
/* Set original_opcode to the opcode iff the instruction
|
|
* starts a line, and thus should be instrumented.
|
|
* This saves having to perform this check every time the
|
|
* we turn instrumentation on or off, and serves as a sanity
|
|
* check when debugging.
|
|
*/
|
|
if (line != current_line && line >= 0) {
|
|
line_data[i].original_opcode = opcode;
|
|
if (line_data[i].line_delta == COMPUTED_LINE) {
|
|
/* Label this line as a line with a line number change
|
|
* which could help the monitoring callback to quickly
|
|
* identify the line number change.
|
|
*/
|
|
line_data[i].line_delta = COMPUTED_LINE_LINENO_CHANGE;
|
|
}
|
|
}
|
|
else {
|
|
line_data[i].original_opcode = 0;
|
|
}
|
|
current_line = line;
|
|
}
|
|
for (int j = 1; j < length; j++) {
|
|
line_data[i+j].original_opcode = 0;
|
|
line_data[i+j].line_delta = NO_LINE;
|
|
}
|
|
i += length;
|
|
}
|
|
for (int i = code->_co_firsttraceable; i < code_len; ) {
|
|
_Py_CODEUNIT inst =_Py_GetBaseCodeUnit(code, i);
|
|
int opcode = inst.op.code;
|
|
int oparg = 0;
|
|
while (opcode == EXTENDED_ARG) {
|
|
oparg = (oparg << 8) | inst.op.arg;
|
|
i++;
|
|
inst =_Py_GetBaseCodeUnit(code, i);
|
|
opcode = inst.op.code;
|
|
}
|
|
oparg = (oparg << 8) | inst.op.arg;
|
|
i += _PyInstruction_GetLength(code, i);
|
|
int target = -1;
|
|
switch (opcode) {
|
|
case POP_JUMP_IF_FALSE:
|
|
case POP_JUMP_IF_TRUE:
|
|
case POP_JUMP_IF_NONE:
|
|
case POP_JUMP_IF_NOT_NONE:
|
|
case JUMP_FORWARD:
|
|
{
|
|
target = i + oparg;
|
|
break;
|
|
}
|
|
case FOR_ITER:
|
|
case SEND:
|
|
{
|
|
/* Skip over END_FOR/END_SEND */
|
|
target = i + oparg + 1;
|
|
break;
|
|
}
|
|
case JUMP_BACKWARD:
|
|
case JUMP_BACKWARD_NO_INTERRUPT:
|
|
{
|
|
target = i - oparg;
|
|
break;
|
|
}
|
|
default:
|
|
continue;
|
|
}
|
|
assert(target >= 0);
|
|
if (line_data[target].line_delta != NO_LINE) {
|
|
line_data[target].original_opcode = _Py_GetBaseCodeUnit(code, target).op.code;
|
|
if (line_data[target].line_delta == COMPUTED_LINE_LINENO_CHANGE) {
|
|
// If the line is a jump target, we are not sure if the line
|
|
// number changes, so we set it to COMPUTED_LINE.
|
|
line_data[target].line_delta = COMPUTED_LINE;
|
|
}
|
|
}
|
|
}
|
|
/* Scan exception table */
|
|
unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable);
|
|
unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable);
|
|
unsigned char *scan = start;
|
|
while (scan < end) {
|
|
int start_offset, size, handler;
|
|
scan = parse_varint(scan, &start_offset);
|
|
assert(start_offset >= 0 && start_offset < code_len);
|
|
scan = parse_varint(scan, &size);
|
|
assert(size >= 0 && start_offset+size <= code_len);
|
|
scan = parse_varint(scan, &handler);
|
|
assert(handler >= 0 && handler < code_len);
|
|
int depth_and_lasti;
|
|
scan = parse_varint(scan, &depth_and_lasti);
|
|
int original_opcode = _Py_GetBaseCodeUnit(code, handler).op.code;
|
|
/* Skip if not the start of a line.
|
|
* END_ASYNC_FOR is a bit special as it marks the end of
|
|
* an `async for` loop, which should not generate its own
|
|
* line event. */
|
|
if (line_data[handler].line_delta != NO_LINE &&
|
|
original_opcode != END_ASYNC_FOR) {
|
|
line_data[handler].original_opcode = original_opcode;
|
|
}
|
|
}
|
|
}
|
|
|
|
static void
|
|
initialize_line_tools(PyCodeObject *code, _Py_LocalMonitors *all_events)
|
|
{
|
|
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
|
|
uint8_t *line_tools = code->_co_monitoring->line_tools;
|
|
|
|
assert(line_tools != NULL);
|
|
int code_len = (int)Py_SIZE(code);
|
|
for (int i = 0; i < code_len; i++) {
|
|
line_tools[i] = all_events->tools[PY_MONITORING_EVENT_LINE];
|
|
}
|
|
}
|
|
|
|
static int
|
|
allocate_instrumentation_data(PyCodeObject *code)
|
|
{
|
|
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
|
|
|
|
if (code->_co_monitoring == NULL) {
|
|
code->_co_monitoring = PyMem_Malloc(sizeof(_PyCoMonitoringData));
|
|
if (code->_co_monitoring == NULL) {
|
|
PyErr_NoMemory();
|
|
return -1;
|
|
}
|
|
code->_co_monitoring->local_monitors = (_Py_LocalMonitors){ 0 };
|
|
code->_co_monitoring->active_monitors = (_Py_LocalMonitors){ 0 };
|
|
code->_co_monitoring->tools = NULL;
|
|
code->_co_monitoring->lines = NULL;
|
|
code->_co_monitoring->line_tools = NULL;
|
|
code->_co_monitoring->per_instruction_opcodes = NULL;
|
|
code->_co_monitoring->per_instruction_tools = NULL;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp)
|
|
{
|
|
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
|
|
|
|
int code_len = (int)Py_SIZE(code);
|
|
if (allocate_instrumentation_data(code)) {
|
|
return -1;
|
|
}
|
|
// If the local monitors are out of date, clear them up
|
|
_Py_LocalMonitors *local_monitors = &code->_co_monitoring->local_monitors;
|
|
for (int i = 0; i < PY_MONITORING_TOOL_IDS; i++) {
|
|
if (code->_co_monitoring->tool_versions[i] != interp->monitoring_tool_versions[i]) {
|
|
for (int j = 0; j < _PY_MONITORING_LOCAL_EVENTS; j++) {
|
|
local_monitors->tools[j] &= ~(1 << i);
|
|
}
|
|
}
|
|
}
|
|
|
|
_Py_LocalMonitors all_events = local_union(
|
|
interp->monitors,
|
|
code->_co_monitoring->local_monitors);
|
|
bool multitools = multiple_tools(&all_events);
|
|
if (code->_co_monitoring->tools == NULL && multitools) {
|
|
code->_co_monitoring->tools = PyMem_Malloc(code_len);
|
|
if (code->_co_monitoring->tools == NULL) {
|
|
PyErr_NoMemory();
|
|
return -1;
|
|
}
|
|
initialize_tools(code);
|
|
}
|
|
if (all_events.tools[PY_MONITORING_EVENT_LINE]) {
|
|
if (code->_co_monitoring->lines == NULL) {
|
|
code->_co_monitoring->lines = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData));
|
|
if (code->_co_monitoring->lines == NULL) {
|
|
PyErr_NoMemory();
|
|
return -1;
|
|
}
|
|
initialize_lines(code);
|
|
}
|
|
if (multitools && code->_co_monitoring->line_tools == NULL) {
|
|
code->_co_monitoring->line_tools = PyMem_Malloc(code_len);
|
|
if (code->_co_monitoring->line_tools == NULL) {
|
|
PyErr_NoMemory();
|
|
return -1;
|
|
}
|
|
initialize_line_tools(code, &all_events);
|
|
}
|
|
}
|
|
if (all_events.tools[PY_MONITORING_EVENT_INSTRUCTION]) {
|
|
if (code->_co_monitoring->per_instruction_opcodes == NULL) {
|
|
code->_co_monitoring->per_instruction_opcodes = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData));
|
|
if (code->_co_monitoring->per_instruction_opcodes == NULL) {
|
|
PyErr_NoMemory();
|
|
return -1;
|
|
}
|
|
// Initialize all of the instructions so if local events change while another thread is executing
|
|
// we know what the original opcode was.
|
|
for (int i = 0; i < code_len; i++) {
|
|
int opcode = _PyCode_CODE(code)[i].op.code;
|
|
code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode];
|
|
}
|
|
}
|
|
if (multitools && code->_co_monitoring->per_instruction_tools == NULL) {
|
|
code->_co_monitoring->per_instruction_tools = PyMem_Malloc(code_len);
|
|
if (code->_co_monitoring->per_instruction_tools == NULL) {
|
|
PyErr_NoMemory();
|
|
return -1;
|
|
}
|
|
for (int i = 0; i < code_len; i++) {
|
|
code->_co_monitoring->per_instruction_tools[i] = 0;
|
|
}
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp)
|
|
{
|
|
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
|
|
|
|
#ifdef _Py_TIER2
|
|
if (code->co_executors != NULL) {
|
|
_PyCode_Clear_Executors(code);
|
|
}
|
|
_Py_Executors_InvalidateDependency(interp, code, 1);
|
|
#endif
|
|
int code_len = (int)Py_SIZE(code);
|
|
/* Exit early to avoid creating instrumentation
|
|
* data for potential statically allocated code
|
|
* objects.
|
|
* See https://github.com/python/cpython/issues/108390 */
|
|
if (code->co_flags & CO_NO_MONITORING_EVENTS) {
|
|
return 0;
|
|
}
|
|
if (update_instrumentation_data(code, interp)) {
|
|
return -1;
|
|
}
|
|
_Py_LocalMonitors active_events = local_union(
|
|
interp->monitors,
|
|
code->_co_monitoring->local_monitors);
|
|
_Py_LocalMonitors new_events;
|
|
_Py_LocalMonitors removed_events;
|
|
|
|
bool restarted = interp->last_restart_version > code->_co_instrumentation_version;
|
|
if (restarted) {
|
|
removed_events = code->_co_monitoring->active_monitors;
|
|
new_events = active_events;
|
|
}
|
|
else {
|
|
removed_events = monitors_sub(code->_co_monitoring->active_monitors, active_events);
|
|
new_events = monitors_sub(active_events, code->_co_monitoring->active_monitors);
|
|
assert(monitors_are_empty(monitors_and(new_events, removed_events)));
|
|
}
|
|
code->_co_monitoring->active_monitors = active_events;
|
|
if (monitors_are_empty(new_events) && monitors_are_empty(removed_events)) {
|
|
goto done;
|
|
}
|
|
/* Insert instrumentation */
|
|
for (int i = code->_co_firsttraceable; i < code_len; i+= _PyInstruction_GetLength(code, i)) {
|
|
assert(_PyCode_CODE(code)[i].op.code != ENTER_EXECUTOR);
|
|
_Py_CODEUNIT instr = _Py_GetBaseCodeUnit(code, i);
|
|
CHECK(instr.op.code != 0);
|
|
int base_opcode = instr.op.code;
|
|
if (opcode_has_event(base_opcode)) {
|
|
int8_t event;
|
|
if (base_opcode == RESUME) {
|
|
event = instr.op.arg > 0;
|
|
}
|
|
else {
|
|
event = EVENT_FOR_OPCODE[base_opcode];
|
|
assert(event > 0);
|
|
}
|
|
uint8_t removed_tools = removed_events.tools[event];
|
|
if (removed_tools) {
|
|
remove_tools(code, i, event, removed_tools);
|
|
}
|
|
uint8_t new_tools = new_events.tools[event];
|
|
if (new_tools) {
|
|
add_tools(code, i, event, new_tools);
|
|
}
|
|
}
|
|
}
|
|
|
|
// GH-103845: We need to remove both the line and instruction instrumentation before
|
|
// adding new ones, otherwise we may remove the newly added instrumentation.
|
|
uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE];
|
|
uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION];
|
|
|
|
if (removed_line_tools) {
|
|
_PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines;
|
|
for (int i = code->_co_firsttraceable; i < code_len;) {
|
|
if (line_data[i].original_opcode) {
|
|
remove_line_tools(code, i, removed_line_tools);
|
|
}
|
|
i += _PyInstruction_GetLength(code, i);
|
|
}
|
|
}
|
|
if (removed_per_instruction_tools) {
|
|
for (int i = code->_co_firsttraceable; i < code_len;) {
|
|
int opcode = _Py_GetBaseCodeUnit(code, i).op.code;
|
|
if (opcode == RESUME || opcode == END_FOR) {
|
|
i += _PyInstruction_GetLength(code, i);
|
|
continue;
|
|
}
|
|
remove_per_instruction_tools(code, i, removed_per_instruction_tools);
|
|
i += _PyInstruction_GetLength(code, i);
|
|
}
|
|
}
|
|
#ifdef INSTRUMENT_DEBUG
|
|
sanity_check_instrumentation(code);
|
|
#endif
|
|
|
|
uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE];
|
|
uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION];
|
|
|
|
if (new_line_tools) {
|
|
_PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines;
|
|
for (int i = code->_co_firsttraceable; i < code_len;) {
|
|
if (line_data[i].original_opcode) {
|
|
add_line_tools(code, i, new_line_tools);
|
|
}
|
|
i += _PyInstruction_GetLength(code, i);
|
|
}
|
|
}
|
|
if (new_per_instruction_tools) {
|
|
for (int i = code->_co_firsttraceable; i < code_len;) {
|
|
int opcode = _Py_GetBaseCodeUnit(code, i).op.code;
|
|
if (opcode == RESUME || opcode == END_FOR) {
|
|
i += _PyInstruction_GetLength(code, i);
|
|
continue;
|
|
}
|
|
add_per_instruction_tools(code, i, new_per_instruction_tools);
|
|
i += _PyInstruction_GetLength(code, i);
|
|
}
|
|
}
|
|
|
|
done:
|
|
FT_ATOMIC_STORE_UINTPTR_RELEASE(code->_co_instrumentation_version,
|
|
global_version(interp));
|
|
|
|
#ifdef INSTRUMENT_DEBUG
|
|
sanity_check_instrumentation(code);
|
|
#endif
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp)
|
|
{
|
|
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
|
|
|
|
if (is_version_up_to_date(code, interp)) {
|
|
assert(
|
|
interp->ceval.instrumentation_version == 0 ||
|
|
instrumentation_cross_checks(interp, code)
|
|
);
|
|
return 0;
|
|
}
|
|
|
|
return force_instrument_lock_held(code, interp);
|
|
}
|
|
|
|
int
|
|
_Py_Instrument(PyCodeObject *code, PyInterpreterState *interp)
|
|
{
|
|
int res;
|
|
LOCK_CODE(code);
|
|
res = instrument_lock_held(code, interp);
|
|
UNLOCK_CODE();
|
|
return res;
|
|
}
|
|
|
|
#define C_RETURN_EVENTS \
|
|
((1 << PY_MONITORING_EVENT_C_RETURN) | \
|
|
(1 << PY_MONITORING_EVENT_C_RAISE))
|
|
|
|
#define C_CALL_EVENTS \
|
|
(C_RETURN_EVENTS | (1 << PY_MONITORING_EVENT_CALL))
|
|
|
|
|
|
static int
|
|
instrument_all_executing_code_objects(PyInterpreterState *interp) {
|
|
ASSERT_WORLD_STOPPED();
|
|
|
|
_PyRuntimeState *runtime = &_PyRuntime;
|
|
HEAD_LOCK(runtime);
|
|
PyThreadState* ts = PyInterpreterState_ThreadHead(interp);
|
|
HEAD_UNLOCK(runtime);
|
|
while (ts) {
|
|
_PyInterpreterFrame *frame = ts->current_frame;
|
|
while (frame) {
|
|
if (frame->owner != FRAME_OWNED_BY_CSTACK) {
|
|
if (instrument_lock_held(_PyFrame_GetCode(frame), interp)) {
|
|
return -1;
|
|
}
|
|
}
|
|
frame = frame->previous;
|
|
}
|
|
HEAD_LOCK(runtime);
|
|
ts = PyThreadState_Next(ts);
|
|
HEAD_UNLOCK(runtime);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static void
|
|
set_events(_Py_GlobalMonitors *m, int tool_id, _PyMonitoringEventSet events)
|
|
{
|
|
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
|
|
for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) {
|
|
uint8_t *tools = &m->tools[e];
|
|
int active = (events >> e) & 1;
|
|
*tools &= ~(1 << tool_id);
|
|
*tools |= (active << tool_id);
|
|
}
|
|
}
|
|
|
|
static void
|
|
set_local_events(_Py_LocalMonitors *m, int tool_id, _PyMonitoringEventSet events)
|
|
{
|
|
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
|
|
for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) {
|
|
uint8_t *tools = &m->tools[e];
|
|
int val = (events >> e) & 1;
|
|
*tools &= ~(1 << tool_id);
|
|
*tools |= (val << tool_id);
|
|
}
|
|
}
|
|
|
|
static int
|
|
check_tool(PyInterpreterState *interp, int tool_id)
|
|
{
|
|
if (tool_id < PY_MONITORING_SYS_PROFILE_ID &&
|
|
interp->monitoring_tool_names[tool_id] == NULL)
|
|
{
|
|
PyErr_Format(PyExc_ValueError, "tool %d is not in use", tool_id);
|
|
return -1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/* We share the eval-breaker with flags, so the monitoring
|
|
* version goes in the top 24 bits */
|
|
#define MONITORING_VERSION_INCREMENT (1 << _PY_EVAL_EVENTS_BITS)
|
|
|
|
int
|
|
_PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events)
|
|
{
|
|
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
|
|
PyThreadState *tstate = _PyThreadState_GET();
|
|
PyInterpreterState *interp = tstate->interp;
|
|
assert(events < (1 << _PY_MONITORING_UNGROUPED_EVENTS));
|
|
if (check_tool(interp, tool_id)) {
|
|
return -1;
|
|
}
|
|
|
|
int res;
|
|
_PyEval_StopTheWorld(interp);
|
|
uint32_t existing_events = get_events(&interp->monitors, tool_id);
|
|
if (existing_events == events) {
|
|
res = 0;
|
|
goto done;
|
|
}
|
|
set_events(&interp->monitors, tool_id, events);
|
|
uint32_t new_version = global_version(interp) + MONITORING_VERSION_INCREMENT;
|
|
if (new_version == 0) {
|
|
PyErr_Format(PyExc_OverflowError, "events set too many times");
|
|
res = -1;
|
|
goto done;
|
|
}
|
|
set_global_version(tstate, new_version);
|
|
#ifdef _Py_TIER2
|
|
_Py_Executors_InvalidateAll(interp, 1);
|
|
#endif
|
|
res = instrument_all_executing_code_objects(interp);
|
|
done:
|
|
_PyEval_StartTheWorld(interp);
|
|
return res;
|
|
}
|
|
|
|
int
|
|
_PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet events)
|
|
{
|
|
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
|
|
PyInterpreterState *interp = _PyInterpreterState_GET();
|
|
assert(events < (1 << _PY_MONITORING_LOCAL_EVENTS));
|
|
if (code->_co_firsttraceable >= Py_SIZE(code)) {
|
|
PyErr_Format(PyExc_SystemError, "cannot instrument shim code object '%U'", code->co_name);
|
|
return -1;
|
|
}
|
|
if (check_tool(interp, tool_id)) {
|
|
return -1;
|
|
}
|
|
|
|
int res;
|
|
_PyEval_StopTheWorld(interp);
|
|
if (allocate_instrumentation_data(code)) {
|
|
res = -1;
|
|
goto done;
|
|
}
|
|
|
|
code->_co_monitoring->tool_versions[tool_id] = interp->monitoring_tool_versions[tool_id];
|
|
|
|
_Py_LocalMonitors *local = &code->_co_monitoring->local_monitors;
|
|
uint32_t existing_events = get_local_events(local, tool_id);
|
|
if (existing_events == events) {
|
|
res = 0;
|
|
goto done;
|
|
}
|
|
set_local_events(local, tool_id, events);
|
|
|
|
res = force_instrument_lock_held(code, interp);
|
|
|
|
done:
|
|
_PyEval_StartTheWorld(interp);
|
|
return res;
|
|
}
|
|
|
|
int
|
|
_PyMonitoring_GetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet *events)
|
|
{
|
|
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
|
|
PyInterpreterState *interp = _PyInterpreterState_GET();
|
|
if (check_tool(interp, tool_id)) {
|
|
return -1;
|
|
}
|
|
if (code->_co_monitoring == NULL) {
|
|
*events = 0;
|
|
return 0;
|
|
}
|
|
_Py_LocalMonitors *local = &code->_co_monitoring->local_monitors;
|
|
*events = get_local_events(local, tool_id);
|
|
return 0;
|
|
}
|
|
|
|
int _PyMonitoring_ClearToolId(int tool_id)
|
|
{
|
|
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
|
|
PyInterpreterState *interp = _PyInterpreterState_GET();
|
|
|
|
for (int i = 0; i < _PY_MONITORING_EVENTS; i++) {
|
|
PyObject *func = _PyMonitoring_RegisterCallback(tool_id, i, NULL);
|
|
if (func != NULL) {
|
|
Py_DECREF(func);
|
|
}
|
|
}
|
|
|
|
if (_PyMonitoring_SetEvents(tool_id, 0) < 0) {
|
|
return -1;
|
|
}
|
|
|
|
_PyEval_StopTheWorld(interp);
|
|
uint32_t version = global_version(interp) + MONITORING_VERSION_INCREMENT;
|
|
if (version == 0) {
|
|
PyErr_Format(PyExc_OverflowError, "events set too many times");
|
|
_PyEval_StartTheWorld(interp);
|
|
return -1;
|
|
}
|
|
|
|
// monitoring_tool_versions[tool_id] is set to latest global version here to
|
|
// 1. invalidate local events on all existing code objects
|
|
// 2. be ready for the next call to set local events
|
|
interp->monitoring_tool_versions[tool_id] = version;
|
|
|
|
// Set the new global version so all the code objects can refresh the
|
|
// instrumentation.
|
|
set_global_version(_PyThreadState_GET(), version);
|
|
int res = instrument_all_executing_code_objects(interp);
|
|
_PyEval_StartTheWorld(interp);
|
|
return res;
|
|
}
|
|
|
|
/*[clinic input]
|
|
module monitoring
|
|
[clinic start generated code]*/
|
|
/*[clinic end generated code: output=da39a3ee5e6b4b0d input=37257f5987a360cf]*/
|
|
/*[clinic end generated code]*/
|
|
|
|
#include "clinic/instrumentation.c.h"
|
|
|
|
static int
|
|
check_valid_tool(int tool_id)
|
|
{
|
|
if (tool_id < 0 || tool_id >= PY_MONITORING_SYS_PROFILE_ID) {
|
|
PyErr_Format(PyExc_ValueError, "invalid tool %d (must be between 0 and 5)", tool_id);
|
|
return -1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/*[clinic input]
|
|
monitoring.use_tool_id
|
|
|
|
tool_id: int
|
|
name: object
|
|
/
|
|
|
|
[clinic start generated code]*/
|
|
|
|
static PyObject *
|
|
monitoring_use_tool_id_impl(PyObject *module, int tool_id, PyObject *name)
|
|
/*[clinic end generated code: output=30d76dc92b7cd653 input=ebc453761c621be1]*/
|
|
{
|
|
if (check_valid_tool(tool_id)) {
|
|
return NULL;
|
|
}
|
|
if (!PyUnicode_Check(name)) {
|
|
PyErr_SetString(PyExc_ValueError, "tool name must be a str");
|
|
return NULL;
|
|
}
|
|
PyInterpreterState *interp = _PyInterpreterState_GET();
|
|
if (interp->monitoring_tool_names[tool_id] != NULL) {
|
|
PyErr_Format(PyExc_ValueError, "tool %d is already in use", tool_id);
|
|
return NULL;
|
|
}
|
|
interp->monitoring_tool_names[tool_id] = Py_NewRef(name);
|
|
Py_RETURN_NONE;
|
|
}
|
|
|
|
/*[clinic input]
|
|
monitoring.clear_tool_id
|
|
|
|
tool_id: int
|
|
/
|
|
|
|
[clinic start generated code]*/
|
|
|
|
static PyObject *
|
|
monitoring_clear_tool_id_impl(PyObject *module, int tool_id)
|
|
/*[clinic end generated code: output=04defc23470b1be7 input=af643d6648a66163]*/
|
|
{
|
|
if (check_valid_tool(tool_id)) {
|
|
return NULL;
|
|
}
|
|
|
|
PyInterpreterState *interp = _PyInterpreterState_GET();
|
|
|
|
if (interp->monitoring_tool_names[tool_id] != NULL) {
|
|
if (_PyMonitoring_ClearToolId(tool_id) < 0) {
|
|
return NULL;
|
|
}
|
|
}
|
|
|
|
Py_RETURN_NONE;
|
|
}
|
|
|
|
/*[clinic input]
|
|
monitoring.free_tool_id
|
|
|
|
tool_id: int
|
|
/
|
|
|
|
[clinic start generated code]*/
|
|
|
|
static PyObject *
|
|
monitoring_free_tool_id_impl(PyObject *module, int tool_id)
|
|
/*[clinic end generated code: output=86c2d2a1219a8591 input=a23fb6be3a8618e9]*/
|
|
{
|
|
if (check_valid_tool(tool_id)) {
|
|
return NULL;
|
|
}
|
|
PyInterpreterState *interp = _PyInterpreterState_GET();
|
|
|
|
if (interp->monitoring_tool_names[tool_id] != NULL) {
|
|
if (_PyMonitoring_ClearToolId(tool_id) < 0) {
|
|
return NULL;
|
|
}
|
|
}
|
|
|
|
Py_CLEAR(interp->monitoring_tool_names[tool_id]);
|
|
Py_RETURN_NONE;
|
|
}
|
|
|
|
/*[clinic input]
|
|
monitoring.get_tool
|
|
|
|
tool_id: int
|
|
/
|
|
|
|
[clinic start generated code]*/
|
|
|
|
static PyObject *
|
|
monitoring_get_tool_impl(PyObject *module, int tool_id)
|
|
/*[clinic end generated code: output=1c05a98b404a9a16 input=eeee9bebd0bcae9d]*/
|
|
|
|
/*[clinic end generated code]*/
|
|
{
|
|
if (check_valid_tool(tool_id)) {
|
|
return NULL;
|
|
}
|
|
PyInterpreterState *interp = _PyInterpreterState_GET();
|
|
PyObject *name = interp->monitoring_tool_names[tool_id];
|
|
if (name == NULL) {
|
|
Py_RETURN_NONE;
|
|
}
|
|
return Py_NewRef(name);
|
|
}
|
|
|
|
/*[clinic input]
|
|
monitoring.register_callback
|
|
|
|
|
|
tool_id: int
|
|
event: int
|
|
func: object
|
|
/
|
|
|
|
[clinic start generated code]*/
|
|
|
|
static PyObject *
|
|
monitoring_register_callback_impl(PyObject *module, int tool_id, int event,
|
|
PyObject *func)
|
|
/*[clinic end generated code: output=e64daa363004030c input=df6d70ea4cf81007]*/
|
|
{
|
|
if (check_valid_tool(tool_id)) {
|
|
return NULL;
|
|
}
|
|
if (_Py_popcount32(event) != 1) {
|
|
PyErr_SetString(PyExc_ValueError, "The callback can only be set for one event at a time");
|
|
return NULL;
|
|
}
|
|
int event_id = _Py_bit_length(event)-1;
|
|
if (event_id < 0 || event_id >= _PY_MONITORING_EVENTS) {
|
|
PyErr_Format(PyExc_ValueError, "invalid event %d", event);
|
|
return NULL;
|
|
}
|
|
if (PySys_Audit("sys.monitoring.register_callback", "O", func) < 0) {
|
|
return NULL;
|
|
}
|
|
if (func == Py_None) {
|
|
func = NULL;
|
|
}
|
|
func = _PyMonitoring_RegisterCallback(tool_id, event_id, func);
|
|
if (func == NULL) {
|
|
Py_RETURN_NONE;
|
|
}
|
|
return func;
|
|
}
|
|
|
|
/*[clinic input]
|
|
monitoring.get_events -> int
|
|
|
|
tool_id: int
|
|
/
|
|
|
|
[clinic start generated code]*/
|
|
|
|
static int
|
|
monitoring_get_events_impl(PyObject *module, int tool_id)
|
|
/*[clinic end generated code: output=4450cc13f826c8c0 input=a64b238f76c4b2f7]*/
|
|
{
|
|
if (check_valid_tool(tool_id)) {
|
|
return -1;
|
|
}
|
|
_Py_GlobalMonitors *m = &_PyInterpreterState_GET()->monitors;
|
|
_PyMonitoringEventSet event_set = get_events(m, tool_id);
|
|
return event_set;
|
|
}
|
|
|
|
/*[clinic input]
|
|
monitoring.set_events
|
|
|
|
tool_id: int
|
|
event_set: int
|
|
/
|
|
|
|
[clinic start generated code]*/
|
|
|
|
static PyObject *
|
|
monitoring_set_events_impl(PyObject *module, int tool_id, int event_set)
|
|
/*[clinic end generated code: output=1916c1e49cfb5bdb input=a77ba729a242142b]*/
|
|
{
|
|
if (check_valid_tool(tool_id)) {
|
|
return NULL;
|
|
}
|
|
if (event_set < 0 || event_set >= (1 << _PY_MONITORING_EVENTS)) {
|
|
PyErr_Format(PyExc_ValueError, "invalid event set 0x%x", event_set);
|
|
return NULL;
|
|
}
|
|
if ((event_set & C_RETURN_EVENTS) && (event_set & C_CALL_EVENTS) != C_CALL_EVENTS) {
|
|
PyErr_Format(PyExc_ValueError, "cannot set C_RETURN or C_RAISE events independently");
|
|
return NULL;
|
|
}
|
|
event_set &= ~C_RETURN_EVENTS;
|
|
if (_PyMonitoring_SetEvents(tool_id, event_set)) {
|
|
return NULL;
|
|
}
|
|
Py_RETURN_NONE;
|
|
}
|
|
|
|
/*[clinic input]
|
|
monitoring.get_local_events -> int
|
|
|
|
tool_id: int
|
|
code: object
|
|
/
|
|
|
|
[clinic start generated code]*/
|
|
|
|
static int
|
|
monitoring_get_local_events_impl(PyObject *module, int tool_id,
|
|
PyObject *code)
|
|
/*[clinic end generated code: output=d3e92c1c9c1de8f9 input=bb0f927530386a94]*/
|
|
{
|
|
if (!PyCode_Check(code)) {
|
|
PyErr_Format(
|
|
PyExc_TypeError,
|
|
"code must be a code object"
|
|
);
|
|
return -1;
|
|
}
|
|
if (check_valid_tool(tool_id)) {
|
|
return -1;
|
|
}
|
|
_PyMonitoringEventSet event_set = 0;
|
|
_PyCoMonitoringData *data = ((PyCodeObject *)code)->_co_monitoring;
|
|
if (data != NULL) {
|
|
for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) {
|
|
if ((data->local_monitors.tools[e] >> tool_id) & 1) {
|
|
event_set |= (1 << e);
|
|
}
|
|
}
|
|
}
|
|
return event_set;
|
|
}
|
|
|
|
/*[clinic input]
|
|
monitoring.set_local_events
|
|
|
|
tool_id: int
|
|
code: object
|
|
event_set: int
|
|
/
|
|
|
|
[clinic start generated code]*/
|
|
|
|
static PyObject *
|
|
monitoring_set_local_events_impl(PyObject *module, int tool_id,
|
|
PyObject *code, int event_set)
|
|
/*[clinic end generated code: output=68cc755a65dfea99 input=5655ecd78d937a29]*/
|
|
{
|
|
if (!PyCode_Check(code)) {
|
|
PyErr_Format(
|
|
PyExc_TypeError,
|
|
"code must be a code object"
|
|
);
|
|
return NULL;
|
|
}
|
|
if (check_valid_tool(tool_id)) {
|
|
return NULL;
|
|
}
|
|
if ((event_set & C_RETURN_EVENTS) && (event_set & C_CALL_EVENTS) != C_CALL_EVENTS) {
|
|
PyErr_Format(PyExc_ValueError, "cannot set C_RETURN or C_RAISE events independently");
|
|
return NULL;
|
|
}
|
|
event_set &= ~C_RETURN_EVENTS;
|
|
if (event_set < 0 || event_set >= (1 << _PY_MONITORING_LOCAL_EVENTS)) {
|
|
PyErr_Format(PyExc_ValueError, "invalid local event set 0x%x", event_set);
|
|
return NULL;
|
|
}
|
|
|
|
if (_PyMonitoring_SetLocalEvents((PyCodeObject*)code, tool_id, event_set)) {
|
|
return NULL;
|
|
}
|
|
Py_RETURN_NONE;
|
|
}
|
|
|
|
/*[clinic input]
|
|
monitoring.restart_events
|
|
|
|
[clinic start generated code]*/
|
|
|
|
static PyObject *
|
|
monitoring_restart_events_impl(PyObject *module)
|
|
/*[clinic end generated code: output=e025dd5ba33314c4 input=add8a855063c8008]*/
|
|
{
|
|
/* We want to ensure that:
|
|
* last restart version > instrumented version for all code objects
|
|
* last restart version < current version
|
|
*/
|
|
PyThreadState *tstate = _PyThreadState_GET();
|
|
PyInterpreterState *interp = tstate->interp;
|
|
|
|
_PyEval_StopTheWorld(interp);
|
|
uint32_t restart_version = global_version(interp) + MONITORING_VERSION_INCREMENT;
|
|
uint32_t new_version = restart_version + MONITORING_VERSION_INCREMENT;
|
|
if (new_version <= MONITORING_VERSION_INCREMENT) {
|
|
_PyEval_StartTheWorld(interp);
|
|
PyErr_Format(PyExc_OverflowError, "events set too many times");
|
|
return NULL;
|
|
}
|
|
interp->last_restart_version = restart_version;
|
|
set_global_version(tstate, new_version);
|
|
int res = instrument_all_executing_code_objects(interp);
|
|
_PyEval_StartTheWorld(interp);
|
|
|
|
if (res) {
|
|
return NULL;
|
|
}
|
|
Py_RETURN_NONE;
|
|
}
|
|
|
|
static int
|
|
add_power2_constant(PyObject *obj, const char *name, int i)
|
|
{
|
|
PyObject *val = PyLong_FromLong(1<<i);
|
|
if (val == NULL) {
|
|
return -1;
|
|
}
|
|
int err = PyObject_SetAttrString(obj, name, val);
|
|
Py_DECREF(val);
|
|
return err;
|
|
}
|
|
|
|
/*[clinic input]
|
|
monitoring._all_events
|
|
[clinic start generated code]*/
|
|
|
|
static PyObject *
|
|
monitoring__all_events_impl(PyObject *module)
|
|
/*[clinic end generated code: output=6b7581e2dbb690f6 input=62ee9672c17b7f0e]*/
|
|
{
|
|
PyInterpreterState *interp = _PyInterpreterState_GET();
|
|
PyObject *res = PyDict_New();
|
|
if (res == NULL) {
|
|
return NULL;
|
|
}
|
|
for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) {
|
|
uint8_t tools = interp->monitors.tools[e];
|
|
if (tools == 0) {
|
|
continue;
|
|
}
|
|
PyObject *tools_obj = PyLong_FromLong(tools);
|
|
assert(tools_obj != NULL);
|
|
int err = PyDict_SetItemString(res, event_names[e], tools_obj);
|
|
Py_DECREF(tools_obj);
|
|
if (err < 0) {
|
|
Py_DECREF(res);
|
|
return NULL;
|
|
}
|
|
}
|
|
return res;
|
|
}
|
|
|
|
static PyMethodDef methods[] = {
|
|
MONITORING_USE_TOOL_ID_METHODDEF
|
|
MONITORING_CLEAR_TOOL_ID_METHODDEF
|
|
MONITORING_FREE_TOOL_ID_METHODDEF
|
|
MONITORING_GET_TOOL_METHODDEF
|
|
MONITORING_REGISTER_CALLBACK_METHODDEF
|
|
MONITORING_GET_EVENTS_METHODDEF
|
|
MONITORING_SET_EVENTS_METHODDEF
|
|
MONITORING_GET_LOCAL_EVENTS_METHODDEF
|
|
MONITORING_SET_LOCAL_EVENTS_METHODDEF
|
|
MONITORING_RESTART_EVENTS_METHODDEF
|
|
MONITORING__ALL_EVENTS_METHODDEF
|
|
{NULL, NULL} // sentinel
|
|
};
|
|
|
|
static struct PyModuleDef monitoring_module = {
|
|
PyModuleDef_HEAD_INIT,
|
|
.m_name = "sys.monitoring",
|
|
.m_size = -1, /* multiple "initialization" just copies the module dict. */
|
|
.m_methods = methods,
|
|
};
|
|
|
|
PyObject *_Py_CreateMonitoringObject(void)
|
|
{
|
|
PyObject *mod = _PyModule_CreateInitialized(&monitoring_module, PYTHON_API_VERSION);
|
|
if (mod == NULL) {
|
|
return NULL;
|
|
}
|
|
if (PyObject_SetAttrString(mod, "DISABLE", &_PyInstrumentation_DISABLE)) {
|
|
goto error;
|
|
}
|
|
if (PyObject_SetAttrString(mod, "MISSING", &_PyInstrumentation_MISSING)) {
|
|
goto error;
|
|
}
|
|
PyObject *events = _PyNamespace_New(NULL);
|
|
if (events == NULL) {
|
|
goto error;
|
|
}
|
|
int err = PyObject_SetAttrString(mod, "events", events);
|
|
Py_DECREF(events);
|
|
if (err) {
|
|
goto error;
|
|
}
|
|
for (int i = 0; i < _PY_MONITORING_EVENTS; i++) {
|
|
if (add_power2_constant(events, event_names[i], i)) {
|
|
goto error;
|
|
}
|
|
}
|
|
err = PyObject_SetAttrString(events, "NO_EVENTS", _PyLong_GetZero());
|
|
if (err) goto error;
|
|
PyObject *val = PyLong_FromLong(PY_MONITORING_DEBUGGER_ID);
|
|
err = PyObject_SetAttrString(mod, "DEBUGGER_ID", val);
|
|
Py_DECREF(val);
|
|
if (err) goto error;
|
|
val = PyLong_FromLong(PY_MONITORING_COVERAGE_ID);
|
|
err = PyObject_SetAttrString(mod, "COVERAGE_ID", val);
|
|
Py_DECREF(val);
|
|
if (err) goto error;
|
|
val = PyLong_FromLong(PY_MONITORING_PROFILER_ID);
|
|
err = PyObject_SetAttrString(mod, "PROFILER_ID", val);
|
|
Py_DECREF(val);
|
|
if (err) goto error;
|
|
val = PyLong_FromLong(PY_MONITORING_OPTIMIZER_ID);
|
|
err = PyObject_SetAttrString(mod, "OPTIMIZER_ID", val);
|
|
Py_DECREF(val);
|
|
if (err) goto error;
|
|
return mod;
|
|
error:
|
|
Py_DECREF(mod);
|
|
return NULL;
|
|
}
|
|
|
|
|
|
static int
|
|
capi_call_instrumentation(PyMonitoringState *state, PyObject *codelike, int32_t offset,
|
|
PyObject **args, Py_ssize_t nargs, int event)
|
|
{
|
|
PyThreadState *tstate = _PyThreadState_GET();
|
|
PyInterpreterState *interp = tstate->interp;
|
|
|
|
uint8_t tools = state->active;
|
|
assert(args[1] == NULL);
|
|
args[1] = codelike;
|
|
if (offset < 0) {
|
|
PyErr_SetString(PyExc_ValueError, "offset must be non-negative");
|
|
return -1;
|
|
}
|
|
if (event != PY_MONITORING_EVENT_LINE) {
|
|
PyObject *offset_obj = PyLong_FromLong(offset);
|
|
if (offset_obj == NULL) {
|
|
return -1;
|
|
}
|
|
assert(args[2] == NULL);
|
|
args[2] = offset_obj;
|
|
}
|
|
size_t nargsf = (size_t) nargs | PY_VECTORCALL_ARGUMENTS_OFFSET;
|
|
PyObject **callargs = &args[1];
|
|
int err = 0;
|
|
|
|
while (tools) {
|
|
int tool = most_significant_bit(tools);
|
|
assert(tool >= 0 && tool < 8);
|
|
assert(tools & (1 << tool));
|
|
tools ^= (1 << tool);
|
|
int res = call_one_instrument(interp, tstate, callargs, nargsf, tool, event);
|
|
if (res == 0) {
|
|
/* Nothing to do */
|
|
}
|
|
else if (res < 0) {
|
|
/* error */
|
|
err = -1;
|
|
break;
|
|
}
|
|
else {
|
|
/* DISABLE */
|
|
if (!PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) {
|
|
PyErr_Format(PyExc_ValueError,
|
|
"Cannot disable %s events. Callback removed.",
|
|
event_names[event]);
|
|
/* Clear tool to prevent infinite loop */
|
|
Py_CLEAR(interp->monitoring_callables[tool][event]);
|
|
err = -1;
|
|
break;
|
|
}
|
|
else {
|
|
state->active &= ~(1 << tool);
|
|
}
|
|
}
|
|
}
|
|
return err;
|
|
}
|
|
|
|
int
|
|
PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version,
|
|
const uint8_t *event_types, Py_ssize_t length)
|
|
{
|
|
PyInterpreterState *interp = _PyInterpreterState_GET();
|
|
if (global_version(interp) == *version) {
|
|
return 0;
|
|
}
|
|
|
|
_Py_GlobalMonitors *m = &interp->monitors;
|
|
for (Py_ssize_t i = 0; i < length; i++) {
|
|
int event = event_types[i];
|
|
state_array[i].active = m->tools[event];
|
|
}
|
|
*version = global_version(interp);
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
PyMonitoring_ExitScope(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
_PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
|
|
{
|
|
assert(state->active);
|
|
PyObject *args[3] = { NULL, NULL, NULL };
|
|
return capi_call_instrumentation(state, codelike, offset, args, 2,
|
|
PY_MONITORING_EVENT_PY_START);
|
|
}
|
|
|
|
int
|
|
_PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
|
|
{
|
|
assert(state->active);
|
|
PyObject *args[3] = { NULL, NULL, NULL };
|
|
return capi_call_instrumentation(state, codelike, offset, args, 2,
|
|
PY_MONITORING_EVENT_PY_RESUME);
|
|
}
|
|
|
|
|
|
|
|
int
|
|
_PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
|
|
PyObject* retval)
|
|
{
|
|
assert(state->active);
|
|
PyObject *args[4] = { NULL, NULL, NULL, retval };
|
|
return capi_call_instrumentation(state, codelike, offset, args, 3,
|
|
PY_MONITORING_EVENT_PY_RETURN);
|
|
}
|
|
|
|
int
|
|
_PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
|
|
PyObject* retval)
|
|
{
|
|
assert(state->active);
|
|
PyObject *args[4] = { NULL, NULL, NULL, retval };
|
|
return capi_call_instrumentation(state, codelike, offset, args, 3,
|
|
PY_MONITORING_EVENT_PY_YIELD);
|
|
}
|
|
|
|
int
|
|
_PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
|
|
PyObject* callable, PyObject *arg0)
|
|
{
|
|
assert(state->active);
|
|
PyObject *args[5] = { NULL, NULL, NULL, callable, arg0 };
|
|
return capi_call_instrumentation(state, codelike, offset, args, 4,
|
|
PY_MONITORING_EVENT_CALL);
|
|
}
|
|
|
|
int
|
|
_PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
|
|
int lineno)
|
|
{
|
|
assert(state->active);
|
|
PyObject *lno = PyLong_FromLong(lineno);
|
|
if (lno == NULL) {
|
|
return -1;
|
|
}
|
|
PyObject *args[3] = { NULL, NULL, lno };
|
|
int res= capi_call_instrumentation(state, codelike, offset, args, 2,
|
|
PY_MONITORING_EVENT_LINE);
|
|
Py_DECREF(lno);
|
|
return res;
|
|
}
|
|
|
|
int
|
|
_PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
|
|
PyObject *target_offset)
|
|
{
|
|
assert(state->active);
|
|
PyObject *args[4] = { NULL, NULL, NULL, target_offset };
|
|
return capi_call_instrumentation(state, codelike, offset, args, 3,
|
|
PY_MONITORING_EVENT_JUMP);
|
|
}
|
|
|
|
int
|
|
_PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
|
|
PyObject *target_offset)
|
|
{
|
|
assert(state->active);
|
|
PyObject *args[4] = { NULL, NULL, NULL, target_offset };
|
|
return capi_call_instrumentation(state, codelike, offset, args, 3,
|
|
PY_MONITORING_EVENT_BRANCH);
|
|
}
|
|
|
|
int
|
|
_PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
|
|
PyObject *retval)
|
|
{
|
|
assert(state->active);
|
|
PyObject *args[4] = { NULL, NULL, NULL, retval };
|
|
return capi_call_instrumentation(state, codelike, offset, args, 3,
|
|
PY_MONITORING_EVENT_C_RETURN);
|
|
}
|
|
|
|
static inline int
|
|
exception_event_setup(PyObject **exc, int event) {
|
|
*exc = PyErr_GetRaisedException();
|
|
if (*exc == NULL) {
|
|
PyErr_Format(PyExc_ValueError,
|
|
"Firing event %d with no exception set",
|
|
event);
|
|
return -1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
|
|
static inline int
|
|
exception_event_teardown(int err, PyObject *exc) {
|
|
if (err == 0) {
|
|
PyErr_SetRaisedException(exc);
|
|
}
|
|
else {
|
|
assert(PyErr_Occurred());
|
|
Py_XDECREF(exc);
|
|
}
|
|
return err;
|
|
}
|
|
|
|
int
|
|
_PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
|
|
{
|
|
int event = PY_MONITORING_EVENT_PY_THROW;
|
|
assert(state->active);
|
|
PyObject *exc;
|
|
if (exception_event_setup(&exc, event) < 0) {
|
|
return -1;
|
|
}
|
|
PyObject *args[4] = { NULL, NULL, NULL, exc };
|
|
int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
|
|
return exception_event_teardown(err, exc);
|
|
}
|
|
|
|
int
|
|
_PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
|
|
{
|
|
int event = PY_MONITORING_EVENT_RAISE;
|
|
assert(state->active);
|
|
PyObject *exc;
|
|
if (exception_event_setup(&exc, event) < 0) {
|
|
return -1;
|
|
}
|
|
PyObject *args[4] = { NULL, NULL, NULL, exc };
|
|
int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
|
|
return exception_event_teardown(err, exc);
|
|
}
|
|
|
|
int
|
|
_PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
|
|
{
|
|
int event = PY_MONITORING_EVENT_C_RAISE;
|
|
assert(state->active);
|
|
PyObject *exc;
|
|
if (exception_event_setup(&exc, event) < 0) {
|
|
return -1;
|
|
}
|
|
PyObject *args[4] = { NULL, NULL, NULL, exc };
|
|
int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
|
|
return exception_event_teardown(err, exc);
|
|
}
|
|
|
|
int
|
|
_PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
|
|
{
|
|
int event = PY_MONITORING_EVENT_RERAISE;
|
|
assert(state->active);
|
|
PyObject *exc;
|
|
if (exception_event_setup(&exc, event) < 0) {
|
|
return -1;
|
|
}
|
|
PyObject *args[4] = { NULL, NULL, NULL, exc };
|
|
int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
|
|
return exception_event_teardown(err, exc);
|
|
}
|
|
|
|
int
|
|
_PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
|
|
{
|
|
int event = PY_MONITORING_EVENT_EXCEPTION_HANDLED;
|
|
assert(state->active);
|
|
PyObject *exc;
|
|
if (exception_event_setup(&exc, event) < 0) {
|
|
return -1;
|
|
}
|
|
PyObject *args[4] = { NULL, NULL, NULL, exc };
|
|
int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
|
|
return exception_event_teardown(err, exc);
|
|
}
|
|
|
|
int
|
|
_PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
|
|
{
|
|
int event = PY_MONITORING_EVENT_PY_UNWIND;
|
|
assert(state->active);
|
|
PyObject *exc;
|
|
if (exception_event_setup(&exc, event) < 0) {
|
|
return -1;
|
|
}
|
|
PyObject *args[4] = { NULL, NULL, NULL, exc };
|
|
int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
|
|
return exception_event_teardown(err, exc);
|
|
}
|
|
|
|
int
|
|
_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value)
|
|
{
|
|
int event = PY_MONITORING_EVENT_STOP_ITERATION;
|
|
assert(state->active);
|
|
assert(!PyErr_Occurred());
|
|
PyErr_SetObject(PyExc_StopIteration, value);
|
|
PyObject *exc;
|
|
if (exception_event_setup(&exc, event) < 0) {
|
|
return -1;
|
|
}
|
|
PyObject *args[4] = { NULL, NULL, NULL, exc };
|
|
int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
|
|
Py_DECREF(exc);
|
|
return exception_event_teardown(err, NULL);
|
|
}
|