GeezerVison is showing github.com — Align faulthandler watchdog timeouts with CPython by youknowdot · Pull Request #9775 · RustPython/RustPython
Align faulthandler watchdog timeouts with CPython
#9775
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement . We’ll occasionally send you account related emails.
Already on GitHub?
Sign in
to your account
Uh oh!
There was an error while loading. Please reload this page.
Show file tree
Hide file tree
Changes from all commits
Commits
File filter
Filter by extension
.py
(1)
.rs
(2)
All 2 file types selected
Viewed files
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Diff view
Hide whitespace
Diff view
Hide whitespace
There are no files selected for viewing
Original file line number
Diff line number
Diff line change
Expand Up
@@ -6,7 +6,8 @@ mod decl {
use crate::vm::{
AsObject, PyObjectRef, PyResult, VirtualMachine,
builtins::PyInt,
function::{ArgIntoBool, ArgIntoFloat, OptionalArg},
function::{ArgIntoBool, OptionalArg},
stdlib::time::pytime,
};
use alloc::sync::Arc;
use core::sync::atomic::{AtomicBool, AtomicI32, Ordering};
Expand Down
Expand Up
@@ -673,7 +674,8 @@ mod decl {
return;
}
let timeout = Duration::from_micros(guard.timeout_us);
cvar.wait_for(&mut guard, timeout);
// A notification or spurious wake does not mean the deadline passed.
cvar.wait_while_for(&mut guard, |state| !state.cancel, timeout);
// Check if cancelled after wait
if guard.cancel {
Expand Down
Expand Up
@@ -797,8 +799,9 @@ mod decl {
#[derive(FromArgs)]
#[allow(unused)]
struct DumpTracebackLaterArgs {
#[pyarg(positional, error_msg = "timeout must be a number (int or float)")]
timeout: ArgIntoFloat,
// Clinic converts repeat/exit before interpreting the timeout object.
#[pyarg(any)]
timeout: PyObjectRef,
#[pyarg(any, default = ArgIntoBool::FALSE)]
repeat: ArgIntoBool,
#[pyarg(any, default, py_default = "sys.stderr")]
Expand All
@@ -811,20 +814,21 @@ mod decl {
#[pyfunction]
fn dump_traceback_later(args: DumpTracebackLaterArgs, vm: &VirtualMachine) -> PyResult<()> {
let timeout: f64 = args.timeout.into_float() ;
if timeout <= 0. 0 {
let timeout = pytime::from_seconds_object(& args.timeout, pytime::Round::Up, vm)? ;
let timeout_us = pytime::Round::Up.divide(timeout, 1_000);
if timeout_us <= 0 {
return Err(vm.new_value_error("timeout must be greater than 0"));
}
let (fd, _file) = get_fd_from_file_opt(args.file, vm)?;
// Convert timeout to microseconds
let timeout_us = (timeout * 1_000_000.0) as u64;
if timeout_us == 0 {
return Err(vm.new_value_error("timeout must be greater than 0"));
pytime::check_thread_timeout(timeout, vm)?;
// CPython also bounds the seconds passed to format_timeout by LONG_MAX.
#[allow(clippy::unnecessary_cast)] // c_long is i32 or i64.
let max_seconds = core::ffi::c_long::MAX as i64;
if timeout_us / 1_000_000 > max_seconds {
return Err(vm.new_overflow_error("timeout value is too large"));
}
let timeout_us = timeout_us as u64;
let (fd, _file) = get_fd_from_file_opt(args.file, vm)?;
let header = format_timeout(timeout_us);
// Snapshot thread frame slots so watchdog can dump tracebacks
Expand Down
Expand Up
@@ -929,7 +933,7 @@ mod decl {
#[derive(FromArgs)]
#[allow(unused)]
struct RegisterArgs {
#[pyarg(positional )]
#[pyarg(any )]
signum: i32,
#[pyarg(any, default, py_default = "sys.stderr")]
file: OptionalArg<PyObjectRef>,
Expand Down
Expand Up
@@ -1095,4 +1099,57 @@ mod decl {
suppress_crash_report();
host_faulthandler::raise_exception(args.code, args.flags);
}
#[cfg(test)]
#[cfg(any(unix, windows))]
mod tests {
use super::*;
use std::sync::mpsc;
use std::time::Instant;
#[test]
fn watchdog_notification_does_not_expire_timeout() {
let state = Arc::new((
Mutex::new(WatchdogState {
cancel: false,
fd: 2,
timeout_us: 60_000_000,
max_threads: 100,
repeat: false,
exit: false,
header: String::new(),
#[cfg(feature = "threading")]
thread_frame_slots: Vec::new(),
}),
Condvar::new(),
));
let worker_state = Arc::clone(&state);
let (done_tx, done_rx) = mpsc::channel();
let worker = thread::spawn(move || {
watchdog_thread(worker_state);
done_tx.send(()).unwrap();
});
// Wait for a real waiter so the notification cannot be lost before
// the watchdog starts. No Python frames are exposed to this worker.
let deadline = Instant::now() + Duration::from_secs(5);
let notified = loop {
if state.1.notify_one() {
break true;
}
if Instant::now() >= deadline {
break false;
}
thread::sleep(Duration::from_millis(1));
};
let completed = done_rx.recv_timeout(Duration::from_millis(100));
// Always stop and join before asserting, including failure paths.
state.0.lock().cancel = true;
state.1.notify_all();
worker.join().unwrap();
assert!(notified, "watchdog did not start waiting");
assert!(matches!(completed, Err(mpsc::RecvTimeoutError::Timeout)));
}
}
}
Original file line number
Diff line number
Diff line change
Expand Up
@@ -147,7 +147,7 @@ fn from_object(object: &PyObject, scale: i64, round: Round, vm: &VirtualMachine)
/// Check the platform wait limit after the caller has handled negative values
/// and any API-specific indefinite-wait sentinel.
pub(crate) fn check_thread_timeout(nanoseconds: i64, vm: &VirtualMachine) -> PyResult<()> {
pub fn check_thread_timeout(nanoseconds: i64, vm: &VirtualMachine) -> PyResult<()> {
if Round::Up.divide(nanoseconds, 1_000) > THREAD_TIMEOUT_MAX_MICROSECONDS {
Err(vm.new_overflow_error("timeout value is too large"))
} else {
Expand Down
Original file line number
Diff line number
Diff line change
@@ -1,13 +1,210 @@
import faulthandler
import signal
import struct
import sys
import time
from testutils import assert_raises
# Faulthandler watchdogs use the same signed-nanosecond conversion as sleep.
def rejected_timeout(expected, message, timeout, **kwargs):
try:
with assert_raises(expected) as caught:
faulthandler.dump_traceback_later(timeout, **kwargs)
assert str(caught.exception) == message
finally:
faulthandler.cancel_dump_traceback_later()
# Keyword binding must reach timeout validation before inspecting the file.
try:
with assert_raises(ValueError) as caught:
faulthandler.dump_traceback_later(timeout=0, file=-1)
assert str(caught.exception) == "timeout must be greater than 0"
assert_raises(TypeError, faulthandler.dump_traceback_later)
assert_raises(TypeError, faulthandler.dump_traceback_later, 60, timeout=60)
assert_raises(TypeError, faulthandler.dump_traceback_later, 60, False, -1, False, 1)
finally:
faulthandler.cancel_dump_traceback_later()
# All probes below reject the file before arming a watchdog, including tiny
# positive durations that could expire before a subsequent cancellation.
file_error = "file is not a valid file descriptor"
for timeout in (1e-20, 5e-324, 1e-9):
rejected_timeout(ValueError, file_error, timeout, file=-1)
for timeout in (0, -0.0, -1, -1e-20, -5e-324):
rejected_timeout(ValueError, "timeout must be greater than 0", timeout, file=-1)
rejected_timeout(ValueError, "Invalid value NaN (not a number)", float("nan"), file=-1)
for timeout in (float("inf"), -float("inf"), 10**100, -(10**100), 9223372037):
rejected_timeout(
OverflowError, "timestamp out of range for C PyTime_t", timeout, file=-1
)
for timeout in (None, "60", object()):
rejected_timeout(
TypeError,
"must be real number, not " + type(timeout).__name__,
timeout,
file=-1,
)
calls = []
class TruthValue:
def __init__(self, name, error=None):
self.name = name
self.error = error
def __bool__(self):
calls.append(self.name)
if self.error is not None:
raise self.error
return False
class IndexedTimeout:
def __index__(self):
calls.append("index")
return 60
def __float__(self):
raise AssertionError("timeout must prefer __index__")
class FloatTimeout:
def __float__(self):
calls.append("float")
return 60.0
class InvalidFile:
def fileno(self):
calls.append("fileno")
return -1
def flush(self):
raise AssertionError("invalid descriptor must not be flushed")
for timeout, conversion in ((IndexedTimeout(), "index"), (FloatTimeout(), "float")):
calls.clear()
rejected_timeout(
RuntimeError,
"file.fileno() is not a valid file descriptor",
timeout,
repeat=TruthValue("repeat"),
exit=TruthValue("exit"),
file=InvalidFile(),
)
assert calls == ["repeat", "exit", conversion, "fileno"]
for failing_argument in ("repeat", "exit"):
calls.clear()
failure = RuntimeError(failing_argument + " failed")
rejected_timeout(
RuntimeError,
str(failure),
None,
repeat=TruthValue("repeat", failure if failing_argument == "repeat" else None),
exit=TruthValue("exit", failure if failing_argument == "exit" else None),
file=InvalidFile(),
)
assert calls == (["repeat"] if failing_argument == "repeat" else ["repeat", "exit"])
for timeout, expected, message in (
(0, ValueError, "timeout must be greater than 0"),
(float("nan"), ValueError, "Invalid value NaN (not a number)"),
(float("inf"), OverflowError, "timestamp out of range for C PyTime_t"),
):
calls.clear()
rejected_timeout(
expected,
message,
timeout,
repeat=TruthValue("repeat"),
exit=TruthValue("exit"),
file=InvalidFile(),
)
assert calls == ["repeat", "exit"]
class FailingIndex:
def __index__(self):
raise RuntimeError("index failed")
def __float__(self):
raise AssertionError("failed index must not fall back to float")
class FailingFloat:
def __float__(self):
raise RuntimeError("float failed")
class InvalidIndex:
def __index__(self):
return 60.0
def __float__(self):
raise AssertionError("invalid index must not fall back to float")
rejected_timeout(RuntimeError, "index failed", FailingIndex(), file=-1)
rejected_timeout(RuntimeError, "float failed", FailingFloat(), file=-1)
try:
assert_raises(TypeError, faulthandler.dump_traceback_later, InvalidIndex(), file=-1)
finally:
faulthandler.cancel_dump_traceback_later()
class FloatSubclass(float):
def __float__(self):
raise AssertionError("float subclass override must be bypassed")
class IndexedFloatSubclass(FloatSubclass):
def __index__(self):
return -1
rejected_timeout(ValueError, file_error, FloatSubclass(60), file=-1)
rejected_timeout(
ValueError, "timeout must be greater than 0", IndexedFloatSubclass(60), file=-1
)
# The platform limit and the signed C long seconds limit precede file lookup.
# This is below the signed-nanosecond limit on every tested platform.
if sys.platform == "win32":
rejected_timeout(OverflowError, "timeout value is too large", 4_294_968, file=-1)
elif struct.calcsize("l") == 4:
rejected_timeout(OverflowError, "timeout value is too large", 2**31, file=-1)
else:
rejected_timeout(ValueError, file_error, 9223372036, file=-1)
# Exercise the successful keyword path only with a long timeout, immediately
# cancelled. Wasm targets do not provide the native watchdog thread.
if sys.platform not in ("wasi", "emscripten"):
try:
assert faulthandler.dump_traceback_later(timeout=60) is None
finally:
faulthandler.cancel_dump_traceback_later()
# An invalid file proves register's signum keyword binds without installing
# any signal handler. This API is absent on Windows.
if hasattr(faulthandler, "register") and hasattr(signal, "SIGUSR1"):
with assert_raises(ValueError) as caught:
faulthandler.register(signum=signal.SIGUSR1, file=-1)
assert str(caught.exception) == "file is not a valid file descriptor"
# The watchdog's thread cap is keyword-only and uses Py_ssize_t conversion.
# Its output must distinguish an exact cap from a truncated thread dump.
if sys.platform not in ("wasi", "emscripten"):
import faulthandler
for cap in (None, 1.5):
assert_raises(TypeError, faulthandler.dump_traceback_later, 60, max_threads=cap)
for cap in (sys.maxsize + 1, -sys.maxsize - 2):
Expand Down
Loading
Oops, something went wrong.
Uh oh!
There was an error while loading. Please reload this page.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.