GV

GeezerVison

The World Wide Web, restored to 800 pixels of glory.

Home|About|Gaming cabinet|Archives|How it works|Starter sites

On this site

  • All gists
  • Back to GitHub
  • Sign in
  • Sign up
  • Code
  • Revisions 5
  • Terms
  • Privacy
  • Security
  • Status
  • Community
  • Docs
  • Contact

This view

  • Original page
  • New address

GeezerVison is showing gist.github.com — jsonl_batch_stats.py

Instantly share code, notes, and snippets.

@maurycy

maurycy/gist:c03173b016ae4564a8600ea1cc3c7255

Last active May 3, 2026 22:57
Show Gist options
  • Download ZIP
  • Star 0 (0) You must be signed in to star a gist
  • Fork 0 (0) You must be signed in to fork a gist
  • Clone this repository at <script src="https://gist.github.com/maurycy/c03173b016ae4564a8600ea1cc3c7255.js"></script>
  • Save maurycy/c03173b016ae4564a8600ea1cc3c7255 to your computer and use it in GitHub Desktop.
Clone this repository at <script src="https://gist.github.com/maurycy/c03173b016ae4564a8600ea1cc3c7255.js"></script>
Save maurycy/c03173b016ae4564a8600ea1cc3c7255 to your computer and use it in GitHub Desktop.
Download ZIP
jsonl_batch_stats.py
Raw
gistfile1.txt
"""Compute per-batch JSONL str_def + frame_def sizes for any Tachyon recording.
Usage:
python.exe jsonl_batch_stats.py <prof.bin> [batch_window_sec=5.0] [run_id=hex]
Replays the binary and reports, for each time-bucketed batch, the bytes of
str_def + frame_def needed to make that batch decodable in isolation
(self-contained: every entry referenced by any sample in the batch is
re-emitted, regardless of whether it was first seen earlier). This models a
streaming format where each batch must stand alone, e.g. for a ring-buffer
transport where older batches may be evicted before the consumer reads them.
Format used:
{"type":"str_def","v":1,"run_id":"<hex>","defs":[{"str_id":N,"value":"..."}]}
{"type":"frame_def","v":1,"run_id":"<hex>","defs":[{"frame_id":N,
"path_str_id":P,"func_str_id":F,"line":L,"end_line":L,
"col":C,"end_col":C}]}
"""
import json
import sys
import uuid
from profiling.sampling.binary_reader import BinaryReader
from profiling.sampling.collector import extract_lineno
from profiling.sampling.stack_collector import StackTraceCollector
class BatchTracker(StackTraceCollector):
"""Track per-batch reference sets for strings and frames."""
def __init__(self, sample_interval_usec, batch_us):
super().__init__(sample_interval_usec)
self.batch_us = batch_us
self.strings_seen = {} # str_value -> str_id (stable across run)
self.frames_seen = {} # frame_key -> frame_id (stable across run)
self.strings_per_batch = {} # batch_idx -> set of string values
self.frames_per_batch = {} # batch_idx -> set of frame keys
self.first_ts = None
self.last_ts = None
self._sample_idx = 0
self._interval = sample_interval_usec
def collect(self, stack_frames, timestamps_us=None):
if timestamps_us:
ts = timestamps_us[0]
weight = len(timestamps_us)
else:
ts = self._sample_idx * self._interval
weight = 1
if self.first_ts is None:
self.first_ts = ts
self.last_ts = ts + (weight - 1) * self._interval
batch_idx = (ts - self.first_ts) // self.batch_us
strs = self.strings_per_batch.setdefault(batch_idx, set())
frms = self.frames_per_batch.setdefault(batch_idx, set())
for frames, thread_id in self._iter_stacks(
stack_frames, skip_idle=self.skip_idle
):
for frame in frames:
filename, location, funcname, _ = frame
if isinstance(location, (list, tuple)) and len(location) >= 4:
line, end_line, col, end_col = (
location[0],
location[1],
location[2],
location[3],
)
else:
line = extract_lineno(location)
end_line, col, end_col = line, 0, 0
if filename not in self.strings_seen:
self.strings_seen[filename] = len(self.strings_seen) + 1
if funcname not in self.strings_seen:
self.strings_seen[funcname] = len(self.strings_seen) + 1
key = (filename, line, end_line, col, end_col, funcname)
if key not in self.frames_seen:
self.frames_seen[key] = len(self.frames_seen) + 1
strs.add(filename)
strs.add(funcname)
frms.add(key)
self._sample_idx += weight
def export(self, filename):
pass
def percentiles(values, ps=(0.5, 0.9, 0.95, 0.99)):
if not values:
return {p: 0 for p in ps}
s = sorted(values)
n = len(s)
return {p: s[min(int(p * n), n - 1)] for p in ps}
def fmt_bytes(n):
for unit in ("B", "KB", "MB", "GB"):
if n < 1024:
return f"{n:.2f} {unit}"
n /= 1024
return f"{n:.2f} TB"
def encode_str_def(run_id, defs):
rec = {"type": "str_def", "v": 1, "run_id": run_id, "defs": defs}
return len((json.dumps(rec, ensure_ascii=False) + "\n").encode("utf-8"))
def encode_frame_def(run_id, defs):
rec = {"type": "frame_def", "v": 1, "run_id": run_id, "defs": defs}
return len((json.dumps(rec, ensure_ascii=False) + "\n").encode("utf-8"))
def main():
path = sys.argv[1]
batch_sec = float(sys.argv[2]) if len(sys.argv) > 2 else 5.0
run_id = sys.argv[3] if len(sys.argv) > 3 else uuid.uuid4().hex
batch_us = int(batch_sec * 1_000_000)
with BinaryReader(path) as r:
info = r.get_info()
coll = BatchTracker(info["sample_interval_us"], batch_us)
try:
r.replay_samples(coll)
except (ValueError, RuntimeError) as e:
print(f"# partial replay: {e}", file=sys.stderr)
if coll.first_ts is None:
print("No samples replayed.", file=sys.stderr)
sys.exit(1)
duration_us = max(0, coll.last_ts - coll.first_ts)
num_batches = max(1, (duration_us // batch_us) + 1)
str_ids = coll.strings_seen
frame_ids = coll.frames_seen
str_per_batch = [
sorted(coll.strings_per_batch.get(b, set()), key=lambda s: str_ids[s])
for b in range(num_batches)
]
frame_per_batch = [
sorted(
coll.frames_per_batch.get(b, set()), key=lambda fr: frame_ids[fr]
)
for b in range(num_batches)
]
str_counts, str_bytes = [], []
frame_counts, frame_bytes = [], []
for b in range(num_batches):
ss = str_per_batch[b]
fs = frame_per_batch[b]
str_counts.append(len(ss))
frame_counts.append(len(fs))
if ss:
defs = [{"str_id": str_ids[s], "value": s} for s in ss]
str_bytes.append(encode_str_def(run_id, defs))
else:
str_bytes.append(0)
if fs:
defs = []
for fr in fs:
fname, ln, eln, c, ec, fn = fr
defs.append(
{
"frame_id": frame_ids[fr],
"path_str_id": str_ids[fname],
"func_str_id": str_ids[fn],
"line": ln,
"end_line": eln,
"col": c,
"end_col": ec,
}
)
frame_bytes.append(encode_frame_def(run_id, defs))
else:
frame_bytes.append(0)
combined_counts = [s + f for s, f in zip(str_counts, frame_counts)]
combined_bytes = [s + f for s, f in zip(str_bytes, frame_bytes)]
print(f"# JSONL batch stats: `{path}`")
print()
print("## Configuration")
print()
print("| Field | Value |")
print("|---|---:|")
print(f"| Wall-clock duration | {duration_us / 1_000_000:.2f} s |")
print(f"| Batch window | {batch_sec} s |")
print(f"| Number of batches | {num_batches} |")
print(f"| Unique strings (whole run) | {len(coll.strings_seen):,} |")
print(f"| Unique frames (whole run) | {len(coll.frames_seen):,} |")
print(f"| run_id length | {len(run_id)} chars |")
print()
def report(name, counts, byte_vals):
cp = percentiles(counts)
bp = percentiles(byte_vals)
non_zero = [v for v in byte_vals if v > 0]
print(f"### `{name}` per batch")
print()
print(f"_Non-empty batches: {len(non_zero)} / {len(byte_vals)}_")
print()
print("| Stat | Entries | Bytes | |")
print("|---|---:|---:|---|")
rows = [
("p50", cp[0.5], bp[0.5]),
("p90", cp[0.9], bp[0.9]),
("p95", cp[0.95], bp[0.95]),
("p99", cp[0.99], bp[0.99]),
(
"max",
max(counts) if counts else 0,
max(byte_vals) if byte_vals else 0,
),
(
"mean",
(sum(counts) // len(counts)) if counts else 0,
(sum(byte_vals) // len(byte_vals)) if byte_vals else 0,
),
("sum", sum(counts), sum(byte_vals)),
]
for stat, c, b in rows:
print(f"| **{stat}** | {c:,} | {b:,} | {fmt_bytes(b)} |")
print()
print("## Self-contained dictionaries per batch")
print()
report("str_def", str_counts, str_bytes)
report("frame_def", frame_counts, frame_bytes)
report("str_def + frame_def combined", combined_counts, combined_bytes)
SAMPLE_BYTES_FULL = 200
SAMPLE_BYTES_ABBREV = 150
SAMPLING_RATES_HZ = (1_000, 10_000, 100_000, 1_000_000)
p99 = percentiles(combined_bytes)[0.99]
print(
"## Dict overhead vs. sample stream (constant per-sample assumptions)"
)
print()
print(
f"_Per-sample message size: full = {SAMPLE_BYTES_FULL} B, "
f"abbreviated = {SAMPLE_BYTES_ABBREV} B (ballpark, ~20-frame stack)._"
)
print()
print(
"_Overhead = dict_bytes / (dict_bytes + sample_bytes_in_batch). "
"Computed for p99 dict bytes per batch and a single batch window._"
)
print()
print(
"| Rate | Sample data / batch (full) | Sample data / batch (abbrev) "
"| Dict % (v. full raw) | Dict % (v. abbreviated raw) |"
)
print("|---|---:|---:|---:|---:|")
for rate in SAMPLING_RATES_HZ:
samples_per_batch = rate * batch_sec
sample_bytes_full = samples_per_batch * SAMPLE_BYTES_FULL
sample_bytes_abbrev = samples_per_batch * SAMPLE_BYTES_ABBREV
pct_full = p99 / (p99 + sample_bytes_full) * 100
pct_abbrev = p99 / (p99 + sample_bytes_abbrev) * 100
rate_label = (
f"{rate // 1000} kHz"
if rate < 1_000_000
else f"{rate // 1_000_000} MHz"
)
print(
f"| {rate_label} | {fmt_bytes(sample_bytes_full)} "
f"| {fmt_bytes(sample_bytes_abbrev)} "
f"| {pct_full:.2f}% | {pct_abbrev:.2f}% |"
)
print()
if __name__ == "__main__":
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You are visitor 01402622 — GeezerVison does not store the pages you look up.
Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required.