|
"""Compute per-batch JSONL str_def + frame_def sizes for any Tachyon recording. |
|
|
|
Usage: |
|
python.exe jsonl_batch_stats.py <prof.bin> [batch_window_sec=5.0] [run_id=hex] |
|
|
|
Replays the binary and reports, for each time-bucketed batch, the bytes of |
|
str_def + frame_def needed to make that batch decodable in isolation |
|
(self-contained: every entry referenced by any sample in the batch is |
|
re-emitted, regardless of whether it was first seen earlier). This models a |
|
streaming format where each batch must stand alone, e.g. for a ring-buffer |
|
transport where older batches may be evicted before the consumer reads them. |
|
|
|
Format used: |
|
{"type":"str_def","v":1,"run_id":"<hex>","defs":[{"str_id":N,"value":"..."}]} |
|
{"type":"frame_def","v":1,"run_id":"<hex>","defs":[{"frame_id":N, |
|
"path_str_id":P,"func_str_id":F,"line":L,"end_line":L, |
|
"col":C,"end_col":C}]} |
|
""" |
|
|
|
import json |
|
import sys |
|
import uuid |
|
from profiling.sampling.binary_reader import BinaryReader |
|
from profiling.sampling.collector import extract_lineno |
|
from profiling.sampling.stack_collector import StackTraceCollector |
|
|
|
|
|
class BatchTracker(StackTraceCollector): |
|
"""Track per-batch reference sets for strings and frames.""" |
|
|
|
def __init__(self, sample_interval_usec, batch_us): |
|
super().__init__(sample_interval_usec) |
|
self.batch_us = batch_us |
|
self.strings_seen = {} # str_value -> str_id (stable across run) |
|
self.frames_seen = {} # frame_key -> frame_id (stable across run) |
|
self.strings_per_batch = {} # batch_idx -> set of string values |
|
self.frames_per_batch = {} # batch_idx -> set of frame keys |
|
self.first_ts = None |
|
self.last_ts = None |
|
self._sample_idx = 0 |
|
self._interval = sample_interval_usec |
|
|
|
def collect(self, stack_frames, timestamps_us=None): |
|
if timestamps_us: |
|
ts = timestamps_us[0] |
|
weight = len(timestamps_us) |
|
else: |
|
ts = self._sample_idx * self._interval |
|
weight = 1 |
|
if self.first_ts is None: |
|
self.first_ts = ts |
|
self.last_ts = ts + (weight - 1) * self._interval |
|
|
|
batch_idx = (ts - self.first_ts) // self.batch_us |
|
strs = self.strings_per_batch.setdefault(batch_idx, set()) |
|
frms = self.frames_per_batch.setdefault(batch_idx, set()) |
|
|
|
for frames, thread_id in self._iter_stacks( |
|
stack_frames, skip_idle=self.skip_idle |
|
): |
|
for frame in frames: |
|
filename, location, funcname, _ = frame |
|
if isinstance(location, (list, tuple)) and len(location) >= 4: |
|
line, end_line, col, end_col = ( |
|
location[0], |
|
location[1], |
|
location[2], |
|
location[3], |
|
) |
|
else: |
|
line = extract_lineno(location) |
|
end_line, col, end_col = line, 0, 0 |
|
if filename not in self.strings_seen: |
|
self.strings_seen[filename] = len(self.strings_seen) + 1 |
|
if funcname not in self.strings_seen: |
|
self.strings_seen[funcname] = len(self.strings_seen) + 1 |
|
key = (filename, line, end_line, col, end_col, funcname) |
|
if key not in self.frames_seen: |
|
self.frames_seen[key] = len(self.frames_seen) + 1 |
|
strs.add(filename) |
|
strs.add(funcname) |
|
frms.add(key) |
|
self._sample_idx += weight |
|
|
|
def export(self, filename): |
|
pass |
|
|
|
|
|
def percentiles(values, ps=(0.5, 0.9, 0.95, 0.99)): |
|
if not values: |
|
return {p: 0 for p in ps} |
|
s = sorted(values) |
|
n = len(s) |
|
return {p: s[min(int(p * n), n - 1)] for p in ps} |
|
|
|
|
|
def fmt_bytes(n): |
|
for unit in ("B", "KB", "MB", "GB"): |
|
if n < 1024: |
|
return f"{n:.2f} {unit}" |
|
n /= 1024 |
|
return f"{n:.2f} TB" |
|
|
|
|
|
def encode_str_def(run_id, defs): |
|
rec = {"type": "str_def", "v": 1, "run_id": run_id, "defs": defs} |
|
return len((json.dumps(rec, ensure_ascii=False) + "\n").encode("utf-8")) |
|
|
|
|
|
def encode_frame_def(run_id, defs): |
|
rec = {"type": "frame_def", "v": 1, "run_id": run_id, "defs": defs} |
|
return len((json.dumps(rec, ensure_ascii=False) + "\n").encode("utf-8")) |
|
|
|
|
|
def main(): |
|
path = sys.argv[1] |
|
batch_sec = float(sys.argv[2]) if len(sys.argv) > 2 else 5.0 |
|
run_id = sys.argv[3] if len(sys.argv) > 3 else uuid.uuid4().hex |
|
batch_us = int(batch_sec * 1_000_000) |
|
|
|
with BinaryReader(path) as r: |
|
info = r.get_info() |
|
coll = BatchTracker(info["sample_interval_us"], batch_us) |
|
try: |
|
r.replay_samples(coll) |
|
except (ValueError, RuntimeError) as e: |
|
print(f"# partial replay: {e}", file=sys.stderr) |
|
|
|
if coll.first_ts is None: |
|
print("No samples replayed.", file=sys.stderr) |
|
sys.exit(1) |
|
|
|
duration_us = max(0, coll.last_ts - coll.first_ts) |
|
num_batches = max(1, (duration_us // batch_us) + 1) |
|
|
|
str_ids = coll.strings_seen |
|
frame_ids = coll.frames_seen |
|
|
|
str_per_batch = [ |
|
sorted(coll.strings_per_batch.get(b, set()), key=lambda s: str_ids[s]) |
|
for b in range(num_batches) |
|
] |
|
frame_per_batch = [ |
|
sorted( |
|
coll.frames_per_batch.get(b, set()), key=lambda fr: frame_ids[fr] |
|
) |
|
for b in range(num_batches) |
|
] |
|
|
|
str_counts, str_bytes = [], [] |
|
frame_counts, frame_bytes = [], [] |
|
for b in range(num_batches): |
|
ss = str_per_batch[b] |
|
fs = frame_per_batch[b] |
|
str_counts.append(len(ss)) |
|
frame_counts.append(len(fs)) |
|
|
|
if ss: |
|
defs = [{"str_id": str_ids[s], "value": s} for s in ss] |
|
str_bytes.append(encode_str_def(run_id, defs)) |
|
else: |
|
str_bytes.append(0) |
|
|
|
if fs: |
|
defs = [] |
|
for fr in fs: |
|
fname, ln, eln, c, ec, fn = fr |
|
defs.append( |
|
{ |
|
"frame_id": frame_ids[fr], |
|
"path_str_id": str_ids[fname], |
|
"func_str_id": str_ids[fn], |
|
"line": ln, |
|
"end_line": eln, |
|
"col": c, |
|
"end_col": ec, |
|
} |
|
) |
|
frame_bytes.append(encode_frame_def(run_id, defs)) |
|
else: |
|
frame_bytes.append(0) |
|
|
|
combined_counts = [s + f for s, f in zip(str_counts, frame_counts)] |
|
combined_bytes = [s + f for s, f in zip(str_bytes, frame_bytes)] |
|
|
|
print(f"# JSONL batch stats: `{path}`") |
|
print() |
|
print("## Configuration") |
|
print() |
|
print("| Field | Value |") |
|
print("|---|---:|") |
|
print(f"| Wall-clock duration | {duration_us / 1_000_000:.2f} s |") |
|
print(f"| Batch window | {batch_sec} s |") |
|
print(f"| Number of batches | {num_batches} |") |
|
print(f"| Unique strings (whole run) | {len(coll.strings_seen):,} |") |
|
print(f"| Unique frames (whole run) | {len(coll.frames_seen):,} |") |
|
print(f"| run_id length | {len(run_id)} chars |") |
|
print() |
|
|
|
def report(name, counts, byte_vals): |
|
cp = percentiles(counts) |
|
bp = percentiles(byte_vals) |
|
non_zero = [v for v in byte_vals if v > 0] |
|
print(f"### `{name}` per batch") |
|
print() |
|
print(f"_Non-empty batches: {len(non_zero)} / {len(byte_vals)}_") |
|
print() |
|
print("| Stat | Entries | Bytes | |") |
|
print("|---|---:|---:|---|") |
|
rows = [ |
|
("p50", cp[0.5], bp[0.5]), |
|
("p90", cp[0.9], bp[0.9]), |
|
("p95", cp[0.95], bp[0.95]), |
|
("p99", cp[0.99], bp[0.99]), |
|
( |
|
"max", |
|
max(counts) if counts else 0, |
|
max(byte_vals) if byte_vals else 0, |
|
), |
|
( |
|
"mean", |
|
(sum(counts) // len(counts)) if counts else 0, |
|
(sum(byte_vals) // len(byte_vals)) if byte_vals else 0, |
|
), |
|
("sum", sum(counts), sum(byte_vals)), |
|
] |
|
for stat, c, b in rows: |
|
print(f"| **{stat}** | {c:,} | {b:,} | {fmt_bytes(b)} |") |
|
print() |
|
|
|
print("## Self-contained dictionaries per batch") |
|
print() |
|
report("str_def", str_counts, str_bytes) |
|
report("frame_def", frame_counts, frame_bytes) |
|
report("str_def + frame_def combined", combined_counts, combined_bytes) |
|
|
|
SAMPLE_BYTES_FULL = 200 |
|
SAMPLE_BYTES_ABBREV = 150 |
|
SAMPLING_RATES_HZ = (1_000, 10_000, 100_000, 1_000_000) |
|
p99 = percentiles(combined_bytes)[0.99] |
|
|
|
print( |
|
"## Dict overhead vs. sample stream (constant per-sample assumptions)" |
|
) |
|
print() |
|
print( |
|
f"_Per-sample message size: full = {SAMPLE_BYTES_FULL} B, " |
|
f"abbreviated = {SAMPLE_BYTES_ABBREV} B (ballpark, ~20-frame stack)._" |
|
) |
|
print() |
|
print( |
|
"_Overhead = dict_bytes / (dict_bytes + sample_bytes_in_batch). " |
|
"Computed for p99 dict bytes per batch and a single batch window._" |
|
) |
|
print() |
|
print( |
|
"| Rate | Sample data / batch (full) | Sample data / batch (abbrev) " |
|
"| Dict % (v. full raw) | Dict % (v. abbreviated raw) |" |
|
) |
|
print("|---|---:|---:|---:|---:|") |
|
for rate in SAMPLING_RATES_HZ: |
|
samples_per_batch = rate * batch_sec |
|
sample_bytes_full = samples_per_batch * SAMPLE_BYTES_FULL |
|
sample_bytes_abbrev = samples_per_batch * SAMPLE_BYTES_ABBREV |
|
pct_full = p99 / (p99 + sample_bytes_full) * 100 |
|
pct_abbrev = p99 / (p99 + sample_bytes_abbrev) * 100 |
|
rate_label = ( |
|
f"{rate // 1000} kHz" |
|
if rate < 1_000_000 |
|
else f"{rate // 1_000_000} MHz" |
|
) |
|
print( |
|
f"| {rate_label} | {fmt_bytes(sample_bytes_full)} " |
|
f"| {fmt_bytes(sample_bytes_abbrev)} " |
|
f"| {pct_full:.2f}% | {pct_abbrev:.2f}% |" |
|
) |
|
print() |
|
|
|
|
|
if __name__ == "__main__": |
|
main() |