GV

GeezerVison

The World Wide Web, restored to 800 pixels of glory.

Home|About|Gaming cabinet|Archives|How it works|Starter sites

On this site

  • Sign in
  • ActionsCI/CD and automation
  • CodespacesCloud development environments
  • All Features
  • Documentation
  • Training
  • Certifications
  • Build with Copilot
  • Customize Copilot
  • Integrations
  • Trending repos
  • GitHub Sponsors
  • Developer articles
  • Enterprises

This view

  • Original page
  • New address

GeezerVison is showing github.com — [3.13] gh-75876: Run bigmem tests in a subprocess (GH-155302) (#156014) · python/cpython@20c0747

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
  • Lib/test
    • _isolated_sample.py
    • memory_watchdog.py
    • support
      • __init__.py
      • isolation.py
    • test_support.py
  • Misc/NEWS.d/next/Tests
    • 2026-08-18-10-16-44.gh-issue-75876.PEZplM.rst
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎Lib/test/_isolated_sample.py‎

Copy file name to clipboardExpand all lines: Lib/test/_isolated_sample.py
+10Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import sys
1111
import time
1212
import unittest
13+
from test import support
1314
from test.support import isolation
1415

1516
# DurationSample sleeps this long in the subprocess; a parent-reported duration
@@ -178,3 +179,12 @@ class TimeoutSample(unittest.TestCase):
178179
@isolation.runInSubprocess(timeout=TIMEOUT)
179180
def test_hang(self):
180181
time.sleep(TIMEOUT_HANG)
182+
183+
184+
class BigmemSample(unittest.TestCase):
185+
186+
@support.bigmemtest(size=1024, memuse=1)
187+
def test_where_it_runs(self, size):
188+
# A real run is isolated by bigmemtest() itself, a dummy run is not.
189+
self.assertEqual(isolation.runningInSubprocess,
190+
bool(support.real_max_memuse))
Collapse file

‎Lib/test/memory_watchdog.py‎

Copy file name to clipboardExpand all lines: Lib/test/memory_watchdog.py
-21Lines changed: 0 additions & 21 deletions
This file was deleted.
Collapse file

‎Lib/test/support/__init__.py‎

Copy file name to clipboardExpand all lines: Lib/test/support/__init__.py
+19-44Lines changed: 19 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1149,38 +1149,6 @@ def set_memlimit(limit: str) -> None:
11491149
max_memuse = memlimit
11501150

11511151

1152-
class _MemoryWatchdog:
1153-
"""An object which periodically watches the process' memory consumption
1154-
and prints it out.
1155-
"""
1156-
1157-
def __init__(self):
1158-
self.procfile = '/proc/{pid}/statm'.format(pid=os.getpid())
1159-
self.started = False
1160-
1161-
def start(self):
1162-
import warnings
1163-
try:
1164-
f = open(self.procfile, 'r')
1165-
except OSError as e:
1166-
logging.getLogger(__name__).warning('/proc not available for stats: %s', e, exc_info=e)
1167-
sys.stderr.flush()
1168-
return
1169-
1170-
import subprocess
1171-
with f:
1172-
watchdog_script = findfile("memory_watchdog.py")
1173-
self.mem_watchdog = subprocess.Popen([sys.executable, watchdog_script],
1174-
stdin=f,
1175-
stderr=subprocess.DEVNULL)
1176-
self.started = True
1177-
1178-
def stop(self):
1179-
if self.started:
1180-
self.mem_watchdog.terminate()
1181-
self.mem_watchdog.wait()
1182-
1183-
11841152
def bigmemtest(size, memuse, dry_run=True):
11851153
"""Decorator for bigmem tests.
11861154
@@ -1193,8 +1161,14 @@ def bigmemtest(size, memuse, dry_run=True):
11931161
extra argument. If 'dry_run' is true, the value passed to the test method
11941162
may be less than the requested value. If 'dry_run' is false, it means the
11951163
test doesn't support dummy runs when -M is not specified.
1164+
1165+
A test that actually allocates the requested memory (that is, one run with
1166+
-M) runs in a subprocess, so that the memory it uses and the address space
1167+
it fragments are released when it ends. A dummy run stays in the process.
11961168
"""
11971169
def decorator(f):
1170+
from test.support import isolation
1171+
11981172
@functools.wraps(f)
11991173
def wrapper(self):
12001174
size = wrapper.size
@@ -1210,20 +1184,21 @@ def wrapper(self):
12101184
"not enough memory: %.1fG minimum needed"
12111185
% (size * memuse / (1024 ** 3)))
12121186

1213-
if real_max_memuse and verbose:
1187+
if (real_max_memuse and verbose
1188+
and not isolation.runningInSubprocess):
12141189
print()
1215-
print(" ... expected peak memory use: {peak:.1f}G"
1216-
.format(peak=size * memuse / (1024 ** 3)))
1217-
watchdog = _MemoryWatchdog()
1218-
watchdog.start()
1219-
else:
1220-
watchdog = None
1190+
peak = (size * memuse) / (1024 ** 3)
1191+
print(f" ... expected peak memory use: {peak:.1f} GiB")
1192+
1193+
if (real_max_memuse and has_subprocess_support
1194+
and not isolation.runningInSubprocess):
1195+
cls = type(self)
1196+
qualname = f'{cls.__qualname__}.{f.__name__}'
1197+
proc = isolation._start_test(cls.__module__, qualname)
1198+
isolation._replay_test(self, *proc.wait())
1199+
return
12211200

1222-
try:
1223-
return f(self, maxsize)
1224-
finally:
1225-
if watchdog:
1226-
watchdog.stop()
1201+
return f(self, maxsize)
12271202

12281203
wrapper.size = size
12291204
wrapper.memuse = memuse
Collapse file

‎Lib/test/support/isolation.py‎

Copy file name to clipboardExpand all lines: Lib/test/support/isolation.py
+91-36Lines changed: 91 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -108,12 +108,75 @@ def _child_environ(env):
108108
return environ
109109

110110

111-
def _run_in_subprocess(module, qualname, options, env, timeout):
112-
"""Run module.qualname (a test method or class) in a fresh subprocess.
111+
class _SubprocessTest:
112+
"""A test running in a subprocess, started by _start_test().
113113
114-
Return ``(payload, output, returncode)``, where *payload* is the decoded
115-
``{'outcomes': ..., 'durations': ...}`` mapping from the subprocess, or
116-
``None`` if it did not run to completion (crash, import error, ...).
114+
The parent can watch the subprocess (its pid) while the test runs, and
115+
must wait() for it.
116+
"""
117+
118+
def __init__(self, proc, result_path):
119+
self._proc = proc
120+
self._result_path = result_path
121+
122+
@property
123+
def pid(self):
124+
return self._proc.pid
125+
126+
def wait(self, timeout=None, tick=None, interval=1.0):
127+
"""Wait for the test to finish, calling *tick* every *interval* seconds.
128+
129+
Return ``(payload, output, returncode)``, where *payload* is the
130+
decoded ``{'outcomes': ..., 'durations': ...}`` mapping from the
131+
subprocess, or ``None`` if it did not run to completion (crash,
132+
import error, ...).
133+
"""
134+
import marshal
135+
import subprocess
136+
import time
137+
deadline = None if timeout is None else time.monotonic() + timeout
138+
try:
139+
while True:
140+
step = None if deadline is None else max(
141+
0.0, deadline - time.monotonic())
142+
# Wake up for the next tick, unless the timeout comes first.
143+
ticking = tick is not None and (step is None or step > interval)
144+
try:
145+
# communicate(), not wait(): a test writing more than a
146+
# pipe buffer would block. Retrying keeps what it read.
147+
stdout, stderr = self._proc.communicate(
148+
timeout=interval if ticking else step)
149+
break
150+
except subprocess.TimeoutExpired:
151+
if ticking:
152+
tick()
153+
continue
154+
# Report the hang rather than leaving the runner stuck.
155+
self._proc.kill()
156+
stdout, stderr = self._proc.communicate()
157+
raise _SubprocessTestError(
158+
f'test did not complete in a subprocess '
159+
f'within {timeout} seconds'
160+
) from _remote(_decode(stdout) + _decode(stderr))
161+
try:
162+
with open(self._result_path, 'rb') as f:
163+
payload = marshal.load(f)
164+
except (OSError, EOFError, ValueError):
165+
payload = None
166+
output = _decode(stdout) + _decode(stderr)
167+
return payload, output, self._proc.returncode
168+
finally:
169+
try:
170+
os.unlink(self._result_path)
171+
except OSError:
172+
pass
173+
174+
175+
def _start_test(module, qualname, options=(), env=None):
176+
"""Start module.qualname (a test method or class) in a fresh subprocess.
177+
178+
Return a _SubprocessTest. Its wait() is what removes the temporary file
179+
the subprocess writes its result to.
117180
"""
118181
import marshal
119182
import subprocess
@@ -129,26 +192,16 @@ def _run_in_subprocess(module, qualname, options, env, timeout):
129192
cmd = [sys.executable, *options, '-m', 'test.support.subprocess_runner',
130193
module, qualname, result_path,
131194
marshal.dumps(_child_config()).hex()]
132-
try:
133-
proc = subprocess.run(cmd, capture_output=True,
134-
env=_child_environ(env), timeout=timeout)
135-
except subprocess.TimeoutExpired as exc:
136-
# Report the hang rather than leaving the test runner stuck.
137-
output = _decode(exc.stdout) + _decode(exc.stderr)
138-
raise _SubprocessTestError(
139-
f'test did not complete in a subprocess '
140-
f'within {timeout} seconds') from _remote(output)
141-
try:
142-
with open(result_path, 'rb') as f:
143-
payload = marshal.load(f)
144-
except (OSError, EOFError, ValueError):
145-
payload = None
146-
finally:
195+
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
196+
stderr=subprocess.PIPE, env=_child_environ(env))
197+
except BaseException:
147198
try:
148199
os.unlink(result_path)
149200
except OSError:
150201
pass
151-
return payload, _decode(proc.stdout) + _decode(proc.stderr), proc.returncode
202+
raise
203+
return _SubprocessTest(proc, result_path)
204+
152205

153206

154207
def _replay_outcome(test, outcome):
@@ -200,6 +253,19 @@ def _check_returncode(returncode, output, what):
200253
raise exc from _remote(output)
201254

202255

256+
def _replay_test(test, payload, output, returncode):
257+
"""Reproduce in *test* the result that _SubprocessTest.wait() returned."""
258+
if payload is None:
259+
exc = _SubprocessTestError(
260+
f'test did not complete in a subprocess (exit code {returncode})')
261+
raise exc from _remote(output)
262+
# The parent measures the test method's own duration (the real cost of the
263+
# isolated run, subprocess startup included), so nothing to forward here.
264+
# Replay the outcomes first: a failure of the test itself is more useful.
265+
_replay_outcomes(test, payload['outcomes'])
266+
_check_returncode(returncode, output, 'test')
267+
268+
203269
def _isolate_method(func, options, env, timeout):
204270
@functools.wraps(func)
205271
def wrapper(self, /, *args, **kwargs):
@@ -209,18 +275,8 @@ def wrapper(self, /, *args, **kwargs):
209275
_check_subprocess_support()
210276
cls = type(self)
211277
qualname = f'{cls.__qualname__}.{func.__name__}'
212-
payload, output, returncode = _run_in_subprocess(cls.__module__,
213-
qualname, options,
214-
env, timeout)
215-
if payload is None:
216-
exc = _SubprocessTestError(
217-
f'test did not complete in a subprocess (exit code {returncode})')
218-
raise exc from _remote(output)
219-
# The parent measures this method's own duration (the real cost of the
220-
# isolated run, subprocess startup included), so nothing to forward here.
221-
# Replay the outcomes first: a failure of the test itself is more useful.
222-
_replay_outcomes(self, payload['outcomes'])
223-
_check_returncode(returncode, output, 'test')
278+
proc = _start_test(cls.__module__, qualname, options, env)
279+
_replay_test(self, *proc.wait(timeout))
224280
return wrapper
225281

226282

@@ -244,9 +300,8 @@ def setUpClass(cls):
244300
_check_subprocess_support()
245301
# Run the whole class in a single subprocess and stash the outcomes
246302
# for the test methods to replay.
247-
payload, output, returncode = _run_in_subprocess(cls.__module__,
248-
cls.__qualname__,
249-
options, env, timeout)
303+
proc = _start_test(cls.__module__, cls.__qualname__, options, env)
304+
payload, output, returncode = proc.wait(timeout)
250305
if payload is None:
251306
exc = _SubprocessTestError(
252307
f'class did not complete in a subprocess (exit code {returncode})')
Collapse file

‎Lib/test/test_support.py‎

Copy file name to clipboardExpand all lines: Lib/test/test_support.py
+14-3Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -965,17 +965,28 @@ def test_timeout_reported_as_error(self):
965965
self.assertEqual(len(result.errors), 1)
966966
self.assertIn(f'within {TIMEOUT} seconds', result.errors[0][1])
967967

968+
@support.requires_subprocess()
969+
def test_bigmemtest_isolates_a_real_run(self):
970+
# A dummy run (no -M) stays in this process, a real run does not.
971+
for memlimit in (0, support._1G):
972+
with self.subTest(real_max_memuse=memlimit):
973+
with support.swap_attr(support, 'real_max_memuse', memlimit):
974+
result = self._run('BigmemSample')
975+
self.assertEqual(result.testsRun, 1)
976+
self.assertEqual(self._names(result.failures), [])
977+
self.assertEqual(self._names(result.errors), [])
978+
968979
def test_skipped_without_subprocess_support(self):
969980
# On a platform without subprocess support the test is skipped in the
970981
# parent, before any subprocess is spawned.
971982
calls = []
972-
orig = isolation._run_in_subprocess
983+
orig = isolation._start_test
973984
with support.swap_attr(support, 'has_subprocess_support', False):
974-
isolation._run_in_subprocess = lambda *a, **k: calls.append(a)
985+
isolation._start_test = lambda *a, **k: calls.append(a)
975986
try:
976987
result = self._run('MethodSample.test_pass')
977988
finally:
978-
isolation._run_in_subprocess = orig
989+
isolation._start_test = orig
979990
self.assertEqual(result.testsRun, 1)
980991
self.assertEqual(len(result.skipped), 1)
981992
self.assertEqual(calls, [])
Collapse file

‎Misc/NEWS.d/next/Tests/2026-08-18-10-16-44.gh-issue-75876.PEZplM.rst‎

Copy file name to clipboard
+5Lines changed: 5 additions & 0 deletions
  • Display the source diff
  • Display the rich diff
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
A test decorated with :func:`~test.support.bigmemtest` now runs in a
2+
subprocess if it really allocates the memory it asks for (that is, if the
3+
``-M`` option is used), so that the memory it uses and the address space it
4+
fragments are released when it ends. A dummy run stays in the process. The
5+
separate memory watchdog process is no longer used.

0 commit comments

Comments
0 (0)
You are visitor 01402520 — GeezerVison does not store the pages you look up.
Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required.