GeezerVison is showing github.com — gh-150114: Log the memory usage in regrtest · vstinner/cpython@6fe3f5a
File treeExpand file treeCollapse file tree Open diff view settings Expand file treeCollapse file tree Open diff view settings Collapse file Copy file name to clipboardExpand all lines: Lib/test/libregrtest/logger.py +12Lines changed: 12 additions & 0 deletions | Original file line number | Diff line number | Diff line change |
|---|
| 1 | 1 | import os | 2 | 2 | import time | | 3 | +from typing import Callable | 3 | 4 |
| 4 | 5 | from test.support import MS_WINDOWS | 5 | 6 | from .results import TestResults | @@ -19,10 +20,21 @@ def __init__(self, results: TestResults, quiet: bool, pgo: bool): | 19 | 20 | self._results: TestResults = results | 20 | 21 | self._quiet: bool = quiet | 21 | 22 | self._pgo: bool = pgo | | 23 | + self.get_mem_usage: Callable[[], int | None] | None = None | 22 | 24 |
| 23 | 25 | def log(self, line: str = '') -> None: | 24 | 26 | empty = not line | 25 | 27 |
| | 28 | + # add the peak memory usage: "mem: 1 GiB " | | 29 | + if self.get_mem_usage is not None: | | 30 | + mem = self.get_mem_usage() | | 31 | + if mem: | | 32 | + mib = mem / (1024*1024) | | 33 | + if mib >= 1024: | | 34 | + line = f"mem: {mib / 1024:.1f} GiB {line}" | | 35 | + else: | | 36 | + line = f"mem: {mib:.1f} MiB {line}" | | 37 | + | 26 | 38 | # add the system load prefix: "load avg: 1.80 " | 27 | 39 | load_avg = self.get_load_avg() | 28 | 40 | if load_avg is not None: | |
Collapse file Copy file name to clipboardExpand all lines: Lib/test/libregrtest/run_workers.py +22-1Lines changed: 22 additions & 1 deletion | Original file line number | Diff line number | Diff line change |
|---|
| 22 | 22 | from .single import PROGRESS_MIN_TIME | 23 | 23 | from .utils import ( | 24 | 24 | StrPath, TestName, | 25 | | - format_duration, print_warning, count, plural) | | 25 | + format_duration, print_warning, count, plural, get_process_memory_usage) | 26 | 26 | from .worker import create_worker_process, USE_PROCESS_GROUP | 27 | 27 |
| 28 | 28 | if MS_WINDOWS: | @@ -452,6 +452,13 @@ def wait_stopped(self, start_time: float) -> None: | 452 | 452 | print_warning(f"Failed to join {self} in {format_duration(dt)}") | 453 | 453 | break | 454 | 454 |
| | 455 | + def get_mem_usage(self): | | 456 | + # Get "VmPeak" from /proc/<pid>/status | | 457 | + popen = self._popen | | 458 | + if popen is None: | | 459 | + return | | 460 | + return get_process_memory_usage(popen.pid) | | 461 | + | 455 | 462 |
| 456 | 463 | def get_running(workers: list[WorkerThread]) -> str | None: | 457 | 464 | running: list[str] = [] | @@ -473,6 +480,7 @@ def __init__(self, num_workers: int, runtests: RunTests, | 473 | 480 | logger: Logger, results: TestResults) -> None: | 474 | 481 | self.num_workers = num_workers | 475 | 482 | self.runtests = runtests | | 483 | + self.logger = logger | 476 | 484 | self.log = logger.log | 477 | 485 | self.display_progress = logger.display_progress | 478 | 486 | self.results: TestResults = results | @@ -598,9 +606,21 @@ def _process_result(self, item: QueueOutput) -> TestResult: | 598 | 606 |
| 599 | 607 | return result | 600 | 608 |
| | 609 | + def get_mem_usage(self): | | 610 | + usage = 0 | | 611 | + main_mem = get_process_memory_usage(os.getpid()) | | 612 | + if main_mem: | | 613 | + usage += main_mem | | 614 | + for worker in self.workers: | | 615 | + worker_mem = worker.get_mem_usage() | | 616 | + if worker_mem: | | 617 | + usage += worker_mem | | 618 | + return usage | | 619 | + | 601 | 620 | def run(self) -> None: | 602 | 621 | fail_fast = self.runtests.fail_fast | 603 | 622 | fail_env_changed = self.runtests.fail_env_changed | | 623 | + self.logger.get_mem_usage = self.get_mem_usage | 604 | 624 |
| 605 | 625 | self.start_workers() | 606 | 626 |
| @@ -625,3 +645,4 @@ def run(self) -> None: | 625 | 645 | # worker when we exit this function | 626 | 646 | self.pending.stop() | 627 | 647 | self.stop_workers() | | 648 | + self.logger.get_mem_usage = None |
Collapse file Copy file name to clipboardExpand all lines: Lib/test/libregrtest/utils.py +20Lines changed: 20 additions & 0 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -752,3 +752,23 @@ def display_title(title): | 752 | 752 | print(title) | 753 | 753 | print("#" * len(title)) | 754 | 754 | print(flush=True) | | 755 | + | | 756 | + | | 757 | +def get_process_memory_usage(pid: int) -> int | None: | | 758 | + try: | | 759 | + fp = open(f"/proc/{pid}/smaps", "rb") | | 760 | + except OSError: | | 761 | + return None | | 762 | + | | 763 | + try: | | 764 | + total = 0 | | 765 | + with fp: | | 766 | + for line in fp: | | 767 | + # Include both Private_Clean and Private_Dirty sections. | | 768 | + line = line.rstrip() | | 769 | + if line.startswith(b"Private_") and line.endswith(b'kB'): | | 770 | + parts = line.split() | | 771 | + total += int(parts[1]) * 1024 | | 772 | + return total | | 773 | + except ProcessLookupError: | | 774 | + return None |
Collapse file Copy file name to clipboardExpand all lines: Lib/test/test_regrtest.py +5-1Lines changed: 5 additions & 1 deletion | Original file line number | Diff line number | Diff line change |
|---|
| 41 | 41 |
| 42 | 42 | ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') | 43 | 43 | ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) | 44 | | -LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?' | | 44 | +LOG_PREFIX = ( | | 45 | + r'[0-9]+:[0-9]+:[0-9]+ ' | | 46 | + r'(?:load avg: [0-9]+\.[0-9]{2} )?' | | 47 | + r'(?:mem: [0-9]+\.[0-9] (?:MiB|GiB) )?' | | 48 | +) | 45 | 49 | RESULT_REGEX = ( | 46 | 50 | 'passed', | 47 | 51 | 'failed', | |
Collapse file Copy file name to clipboard +2Lines changed: 2 additions & 0 deletions - Display the source diff
- Display the rich diff
| Original file line number | Diff line number | Diff line change |
|---|
| | 1 | +On Linux, regrtest now logs the total memory usage of all Python processes. | | 2 | +Read the private memory in ``/proc/pid/smaps``. Patch by Victor Stinner. |
|