GeezerVison is showing github.com — JIT: assertion failure in pycore_backoff.h · Issue #141786 · python/cpython
JIT: assertion failure in pycore_backoff.h #141786DescriptionCrash report
What happened?
It's possible to trigger an assertion failure in a patched (diff follows) JIT build by running the code below. It's a finicky issue to reproduce, so the MRE isn't as compact as I'd like and requires a special environment, but changing minor details stops reproducing the abort.
To prepare the environment, run:
# Create a venv, then:
git clone https://github.com/devdanzin/lafleur.git
git clone https://github.com/devdanzin/fusil.git
pip install -e lafleur
pip install -e fusil Also, the way the script is invoked interferes with reproduction: calling it with pythoninside the active venv or with path/to/venv/bin/python works, while path/to/built/python doesn't work.
Given the sensitivity of the MRE to small changes, please let me know whether you can reproduce this issue.
The diff applied was (it might be possible to reduce this diff and keep reproducing):
diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h
index 71066f1bd9f..e4a31f76ff2 100644
--- a/Include/internal/pycore_backoff.h
+++ b/Include/internal/pycore_backoff.h
@@ -112,8 +112,8 @@ trigger_backoff_counter(void)
// For example, 4095 does not work for the nqueens benchmark on pyperformance
// as we always end up tracing the loop iteration's
// exhaustion iteration. Which aborts our current tracer.
-#define JUMP_BACKWARD_INITIAL_VALUE 4000
-#define JUMP_BACKWARD_INITIAL_BACKOFF 12
+#define JUMP_BACKWARD_INITIAL_VALUE 63
+#define JUMP_BACKWARD_INITIAL_BACKOFF 6
static inline _Py_BackoffCounter
initial_jump_backoff_counter(void)
{
@@ -125,8 +125,8 @@ initial_jump_backoff_counter(void)
* Must be larger than ADAPTIVE_COOLDOWN_VALUE,
* otherwise when a side exit warms up we may construct
* a new trace before the Tier 1 code has properly re-specialized. */
-#define SIDE_EXIT_INITIAL_VALUE 4000
-#define SIDE_EXIT_INITIAL_BACKOFF 12
+#define SIDE_EXIT_INITIAL_VALUE 63
+#define SIDE_EXIT_INITIAL_BACKOFF 6
static inline _Py_BackoffCounter
initial_temperature_backoff_counter(void)
diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h
index e7177552cf6..0d76a5a3df0 100644
--- a/Include/internal/pycore_optimizer.h
+++ b/Include/internal/pycore_optimizer.h
@@ -86,7 +86,7 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp);
// Used as the threshold to trigger executor invalidation when
// executor_creation_counter is greater than this value.
// This value is arbitrary and was not optimized.
-#define JIT_CLEANUP_THRESHOLD 1000
+#define JIT_CLEANUP_THRESHOLD 10000
int _Py_uop_analyze_and_optimize(
PyFunctionObject *func,
@@ -118,7 +118,7 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst)
}
// Holds locals, stack, locals, stack ... co_consts (in that order)
-#define MAX_ABSTRACT_INTERP_SIZE 4096
+#define MAX_ABSTRACT_INTERP_SIZE 8192
#define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5)
@@ -129,7 +129,7 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst)
// progress (and inserting a new ENTER_EXECUTOR instruction). In practice, this
// is the "maximum amount of polymorphism" that an isolated trace tree can
// handle before rejoining the rest of the program.
-#define MAX_CHAIN_DEPTH 4
+#define MAX_CHAIN_DEPTH 16
/* Symbols */
/* See explanation in optimizer_symbols.c */
diff --git a/Python/optimizer.c b/Python/optimizer.c
index 9db894f0bf0..14cbf670dec 100644
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -509,7 +509,7 @@ guard_ip_uop[MAX_UOP_ID + 1] = {
#define CONFIDENCE_RANGE 1000
-#define CONFIDENCE_CUTOFF 333
+#define CONFIDENCE_CUTOFF 100
#ifdef Py_DEBUG
#define DPRINTF(level, ...) \The admitedly large, but much reduced, MRE is:
import builtins
import gc
import math # Not used
def g():
gc.set_threshold(5)
original_len = builtins.len
try:
builtins.len = lambda x: 'evil_string'
_ = len([])
except Exception:
pass
finally:
builtins.len = original_len
THRESHOLD = 300
def f1():
class Base1:
def __init__(self):
pass
class Sub1(Base1):
def __init__(self):
super().__init__()
def f(self):
super().f()
if False:
Sub1.__bases__ = (object,)
instance_1 = Sub1()
for _ in range(THRESHOLD):
try:
_ = instance_1.f()
except AttributeError:
pass
Sub1.__bases__ = (object,)
g()
class Dummy1:
def __init__(self):
pass
def f(self):
pass
class Sub2(Base1):
def __init__(self):
super().__init__()
def f(self):
super().f()
if False:
Sub2.__bases__ = (object,)
instance_2 = Sub2()
for _ in range(THRESHOLD):
try:
_ = instance_2.f()
except AttributeError:
pass
Sub2.__bases__ = (object,)
class Base2:
def __init__(self):
pass
def f(self):
pass
class Dummy2:
def __init__(self):
pass
def f(self):
pass
class Sub4(Base1):
def __init__(self):
super().__init__()
def f(self):
super().f()
if False:
Sub4.__bases__ = (object,)
instance_4 = Sub4()
for _ in range(THRESHOLD):
try:
_ = instance_4.f()
except AttributeError:
pass
Sub4.__bases__ = (object,)
class Sub3(Base2):
def __init__(self):
pass
def f(self):
super().f()
if self.non_existing_attribute:
Sub3.__bases__ = (object,)
instance_3 = Sub3()
for _ in range(THRESHOLD):
try:
_ = instance_3.f()
except AttributeError:
pass
Sub3.__bases__ = (object,)
for _ in range(50):
try:
_ = instance_3.f()
except AttributeError:
pass
for i_f1 in range(300):
print(i_f1)
f1()The backtrace (with printing of iteration number) is:
1
[...]
64
65
python: ./Include/internal/pycore_backoff.h:65: _Py_BackoffCounter restart_backoff_counter(_Py_BackoffCounter): Assertion `!is_unreachable_backoff_counter(counter)' failed.
Program received signal SIGABRT, Aborted.
#0 __pthread_kill_implementation (threadid=<optimized out>, signo=6, no_tid=0) at ./nptl/pthread_kill.c:44
#1 __pthread_kill_internal (threadid=<optimized out>, signo=6) at ./nptl/pthread_kill.c:89
#2 __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6) at ./nptl/pthread_kill.c:100
#3 0x00007ffff7c45e2e in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26
#4 0x00007ffff7c28888 in __GI_abort () at ./stdlib/abort.c:77
#5 0x00007ffff7c287f0 in __assert_fail_base (fmt=<optimized out>, assertion=<optimized out>, file=<optimized out>, line=<optimized out>, function=<optimized out>) at ./assert/assert.c:118
#6 0x00007ffff7c3c19f in __assert_fail (assertion=<optimized out>, file=<optimized out>, line=<optimized out>, function=<optimized out>) at ./assert/assert.c:127
#7 0x0000555555ec14c5 in restart_backoff_counter (counter=...) at ./Include/internal/pycore_backoff.h:65
#8 0x0000555555ec14c5 in stop_tracing_and_jit (tstate=0x555556aebf00 <_PyRuntime+358560>, frame=frame@entry=0x7e8ff6de5298)
#9 0x0000555555e6e4f3 in _PyEval_EvalFrameDefault (tstate=<optimized out>, frame=<optimized out>, throwflag=<optimized out>) at Python/generated_cases.c.h:11712
#10 0x0000555555e4a67b in _PyEval_EvalFrame (tstate=0x555556aebf00 <_PyRuntime+358560>, frame=0x7e8ff6de5220, throwflag=0) at ./Include/internal/pycore_ceval.h:121
#11 _PyEval_Vector (tstate=<optimized out>, func=<optimized out>, locals=<optimized out>, args=<optimized out>, argcount=<optimized out>, kwnames=0x0) at Python/ceval.c:2159
#12 0x0000555555e4a095 in PyEval_EvalCode (co=<optimized out>, globals=<optimized out>, locals=0x7c7ff6e871c0) at Python/ceval.c:995
#13 0x00005555561bbc6f in run_eval_code_obj (tstate=tstate@entry=0x555556aebf00 <_PyRuntime+358560>, co=co@entry=0x7d2ff6e36c10, globals=globals@entry=0x7c7ff6e871c0,
locals=locals@entry=0x7c7ff6e871c0) at Python/pythonrun.c:1372
#14 0x00005555561bae3c in run_mod (mod=<optimized out>, filename=<optimized out>, globals=<optimized out>, locals=<optimized out>, flags=<optimized out>, arena=<optimized out>,
interactive_src=<optimized out>, generate_new_source=<optimized out>) at Python/pythonrun.c:1475
Output from running with PYTHON_LLTRACE=4:
89_abort_lltrace.txt
Output from running with PYTHON_OPT_DEBUG=4:
89_abort_opt_debug.txt
Sorry for the convoluted repro steps, I've been trying to make it work with a simpler setup for many days, without success.
Found using lafleur.
CPython versions tested on:
CPython main branch
Operating systems tested on:
Linux
Output from running 'python -VV' on the command line:
Python 3.15.0a2+ (heads/main-dirty:41b9ad5b38e, Nov 20 2025, 08:38:13) [Clang 21.1.2 (2ubuntu6)]
Linked PRs
ActivityFidget-Spinner commented on Nov 20, 2025 What I think is happening is the following:
- During trace recording of a side trace, the original parent trace's executor is cleared.
- When the trace record concludes, it tries to JIT the trace, though the parent executor is already cleared.
- The assertion fails.
Can you please try the following diff and see it if still fails? Thank you
diff --git a/Python/optimizer.c b/Python/optimizer.c
index 9db894f0bf0..9707f46c02f 100644
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -1671,7 +1671,7 @@ executor_clear(PyObject *op)
_PyExecutorObject *cold = _PyExecutor_GetColdExecutor();
Py_INCREF(executor);
for (uint32_t i = 0; i < executor->exit_count; i++) {
- executor->exits[i].temperature = initial_unreachable_backoff_counter();
+ // executor->exits[i].temperature = initial_unreachable_backoff_counter();
_PyExecutorObject *e = executor->exits[i].executor;
executor->exits[i].executor = cold;
Py_DECREF(e);
devdanzin commented on Nov 20, 2025 Applying the diff fixes the crash, thank you! devdanzin commented on Nov 24, 2025 Here's another long MRE that results in a different abort, but is fixed by the same patch above:
import random
rand = random.Random(38720)
import sys
def b():
for c in range(23):
rand.random()
rand.random()
if rand.random() > 0.1:
rand.random()
rand.random()
rand.random()
for c in range(4):
rand.random()
if rand.random():
0
if rand.random() > 0.1:
rand.random()
rand.random()
if rand.random():
rand.random()
if rand.random():
rand.random()
rand.random()
if rand.random():
rand.random()
rand.random()
if rand.random():
if rand.random() > 0.1:
rand.random()
if rand.random() < rand.random():
...
if rand.random() < rand.random():
...
if rand.random():
...
if rand.random() > 0.1:
rand.random()
rand.random()
for c in range(12):
rand.random()
def random(B):
if rand.random() < 0.1:
random(B)
def Random(B):
if rand.random():
random(B)
def b(B):
if rand.random():
Random(B)
def c(B):
if rand.random() < 0.1:
b(B)
def a(B):
if rand.random() > 0.1:
c(B)
def A(B):
if rand.random() > 0.1:
a(B)
def C(B):
if rand.random() > 0.1:
A(B)
def B(B):
if rand.random() < 0.1:
C(B)
for D in range(200):
try:
B(D)
except:
...
class A:
def __del__(C):
b = sys._getframe(1)
exec("D=0", b.f_globals, b.f_locals)
B = A()
if rand.random() < 0.1:
if rand.random() < 0.1:
if rand.random() < 0.1:
for D in range(D):
if rand.random() < 0.1:
if rand.random() > 0.1:
if rand.random():
if rand.random() > rand.random():
C
if rand.random() < 0.1:
if rand.random() < 0.1:
if rand.random() < 0.1:
del B
if rand.random() < 0.1:
if rand.random() < 0.1:
if rand.random():
0
if rand.random() > 0.1:
if rand.random() > 0.1:
rand.random()
rand.random()
if rand.random() > 0.1:
rand.random()
rand.random()
if rand.random() > 0.1:
if rand.random() > 0.1:
rand.random()
rand.random()
rand.random()
rand.random()
if rand.random() > 0.1:
if rand.random() > 0.1:
rand.random()
if rand.random() > rand.random():
0
if rand.random():
C
if rand.random() > 0.1:
rand.random()
rand.random()
for c in range(3):
if rand.random():
0
for c in range(2):
rand.random()
for D in range(5064):
b()Backtrace:
python: ./Include/internal/pycore_backoff.h:81: _Py_BackoffCounter restart_backoff_counter(_Py_BackoffCounter): Assertion `backoff <= MAX_BACKOFF' failed.
Program received signal SIGABRT, Aborted.
#0 __pthread_kill_implementation (threadid=<optimized out>, signo=6, no_tid=0) at ./nptl/pthread_kill.c:44
#1 __pthread_kill_internal (threadid=<optimized out>, signo=6) at ./nptl/pthread_kill.c:89
#2 __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6) at ./nptl/pthread_kill.c:100
#3 0x00007ffff7c45e2e in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26
#4 0x00007ffff7c28888 in __GI_abort () at ./stdlib/abort.c:77
#5 0x00007ffff7c287f0 in __assert_fail_base (fmt=<optimized out>, assertion=<optimized out>, file=<optimized out>, line=<optimized out>, function=<optimized out>) at ./assert/assert.c:118
#6 0x00007ffff7c3c19f in __assert_fail (assertion=<optimized out>, file=<optimized out>, line=<optimized out>, function=<optimized out>) at ./assert/assert.c:127
#7 0x000055555586e596 in restart_backoff_counter (counter=...) at ./Include/internal/pycore_backoff.h:81
#8 0x000055555586e596 in stop_tracing_and_jit (tstate=tstate@entry=0x555555d988a8 <_PyRuntime+358616>, frame=frame@entry=0x7ffff7fa7090)
#9 0x0000555555865724 in _PyEval_EvalFrameDefault (tstate=tstate@entry=0x555555d988a8 <_PyRuntime+358616>, frame=<optimized out>, frame@entry=0x7ffff7fa7020, throwflag=throwflag@entry=0)
at Python/generated_cases.c.h:11712
#10 0x000055555582d56b in _PyEval_EvalFrame (tstate=0x555555d988a8 <_PyRuntime+358616>, frame=0x7ffff7fa7020, throwflag=0) at ./Include/internal/pycore_ceval.h:121
#11 _PyEval_Vector (tstate=tstate@entry=0x555555d988a8 <_PyRuntime+358616>, func=func@entry=0x7ffff6bf6a50, locals=locals@entry=0x7ffff6c05190, args=args@entry=0x0,
argcount=argcount@entry=0, kwnames=kwnames@entry=0x0) at Python/ceval.c:2159
This MRE was reduced in about 8 hours, first manually then running Shrinkray. caje731 commented on Dec 6, 2025 markshannon commented on Dec 8, 2025 @Fidget-Spinner when you say "parent executor is already cleared" do you mean that it has been invalidated?
It shouldn't have been cleared while there is a strong reference to it.
It tracing from a side exit, the tracer should be keeping a strong reference to the originating trace.
I would have thought the simplest fix is to check for the parent's validity before linking the new trace. Fidget-Spinner commented on Dec 8, 2025
@Fidget-Spinner when you say "parent executor is already cleared" do you mean that it has been invalidated?
Yes.
I would have thought the simplest fix is to check for the parent's validity before linking the new trace.
Yes the fix is indeed checking the validity. However, note that the parent executor could also be the cold executor, in which case we must just skip the check. Fidget-Spinner commented on Mar 23, 2026 devdanzin commented on Mar 23, 2026 No, I can't get it to reproduce. It reproduces if I checkout an old enough CPython revision, so the environment looks right to reproduce it, but it doesn't.
MetadataMetadataDevelopmentNo branches or pull requestsIssue actions
|