GeezerVison is showing github.com — feat: stream CLI log transcripts and run status to Socket backend (#201) · SocketDev/socket-python-cli@82a4756
File treeExpand file treeCollapse file tree Open diff view settings Expand file treeCollapse file tree Open diff view settings Collapse file Copy file name to clipboardExpand all lines: CHANGELOG.md +8Lines changed: 8 additions & 0 deletions - Display the source diff
- Display the rich diff
| Original file line number | Diff line number | Diff line change |
|---|
| 1 | 1 | # Changelog | 2 | 2 |
| | 3 | +## 2.4.9 | | 4 | + | | 5 | +### Added: opt-in streaming log channel via `--upload-logs` | | 6 | + | | 7 | +- New `--upload-logs` flag (default off). When set, each CLI invocation registers a run, reports a per-run status (`in_progress` / `success` / `failure` / `cancelled`), and uploads a transcript of its own log output to the Socket backend for that run, visible in the Socket admin views. The transcript is captured regardless of the local `--enable-debug` state; the existing terminal verbosity is unchanged. | | 8 | +- New `--no-upload-logs` flag (mutually exclusive with `--upload-logs`) explicitly opts the run out of uploading logs, even when an org-level override would otherwise enable it. Use this when you need a guaranteed no-upload guarantee (e.g. legal/consent reasons). | | 9 | +- The Socket backend can also force-enable streaming for specific orgs in the absence of an explicit opt-out. The feature is best-effort — registration or upload failures silently degrade and never block the scan. | | 10 | + | 3 | 11 | ## 2.4.8 | 4 | 12 |
| 5 | 13 | ### Fixed: retry transient full-scan upload failures | |
Collapse file Copy file name to clipboardExpand all lines: pyproject.toml +1-1Lines changed: 1 addition & 1 deletion | Original file line number | Diff line number | Diff line change |
|---|
@@ -6,7 +6,7 @@ build-backend = "hatchling.build" | 6 | 6 |
| 7 | 7 | [project] | 8 | 8 | name = "socketsecurity" | 9 | | -version = "2.4.8" | | 9 | +version = "2.4.9" | 10 | 10 | requires-python = ">= 3.11" | 11 | 11 | license = {"file" = "LICENSE"} | 12 | 12 | dependencies = [ | |
Collapse file Copy file name to clipboard +1-1Lines changed: 1 addition & 1 deletion | Original file line number | Diff line number | Diff line change |
|---|
| 1 | 1 | __author__ = 'socket.dev' | 2 | | -__version__ = '2.4.8' | | 2 | +__version__ = '2.4.9' | 3 | 3 | USER_AGENT = f'SocketPythonCLI/{__version__}' |
Collapse file Copy file name to clipboardExpand all lines: socketsecurity/config.py +24Lines changed: 24 additions & 0 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -139,6 +139,9 @@ class CliConfig: | 139 | 139 | ignore_commit_files: bool = False | 140 | 140 | disable_blocking: bool = False | 141 | 141 | disable_ignore: bool = False | | 142 | + # Tri-state log-upload preference: True = --upload-logs, False = --no-upload-logs, | | 143 | + # None = neither (server-side override decides). | | 144 | + upload_logs: Optional[bool] = None | 142 | 145 | strict_blocking: bool = False | 143 | 146 | integration_type: IntegrationType = "api" | 144 | 147 | integration_org_slug: Optional[str] = None | @@ -282,6 +285,7 @@ def from_args(cls, args_list: Optional[List[str]] = None) -> 'CliConfig': | 282 | 285 | 'ignore_commit_files': args.ignore_commit_files, | 283 | 286 | 'disable_blocking': args.disable_blocking, | 284 | 287 | 'disable_ignore': args.disable_ignore, | | 288 | + 'upload_logs': args.upload_logs, | 285 | 289 | 'strict_blocking': args.strict_blocking, | 286 | 290 | 'integration_type': args.integration, | 287 | 291 | 'pending_head': args.pending_head, | @@ -866,6 +870,26 @@ def create_argument_parser() -> argparse.ArgumentParser: | 866 | 870 | action="store_true", | 867 | 871 | help=argparse.SUPPRESS | 868 | 872 | ) | | 873 | + log_upload_group = advanced_group.add_mutually_exclusive_group() | | 874 | + log_upload_group.add_argument( | | 875 | + "--upload-logs", | | 876 | + dest="upload_logs", | | 877 | + action="store_const", | | 878 | + const=True, | | 879 | + help="Upload the CLI's log output to the Socket backend for this run. " | | 880 | + "When set, the CLI registers the run with share_logs=true and streams " | | 881 | + "its log records in 5s batches. Default off. Mutually exclusive with " | | 882 | + "--no-upload-logs." | | 883 | + ) | | 884 | + log_upload_group.add_argument( | | 885 | + "--no-upload-logs", | | 886 | + dest="upload_logs", | | 887 | + action="store_const", | | 888 | + const=False, | | 889 | + help="Explicitly opt out of uploading CLI logs to the Socket backend, even " | | 890 | + "when an org-level override would otherwise enable it. Mutually " | | 891 | + "exclusive with --upload-logs." | | 892 | + ) | 869 | 893 | advanced_group.add_argument( | 870 | 894 | "--strict-blocking", | 871 | 895 | dest="strict_blocking", | |
Collapse file Copy file name to clipboard +79Lines changed: 79 additions & 0 deletions | Original file line number | Diff line number | Diff line change |
|---|
| | 1 | +"""Lifecycle helpers for a CLI run on the Socket backend. | | 2 | +
| | 3 | +A "run" represents a single CLI invocation. `register_cli_run` opens it and | | 4 | +returns a server-issued `run_id` when streaming is enabled; `finalize_cli_run` | | 5 | +closes it on exit. The run_id keys the rows that `BatchedLogUploader` POSTs to | | 6 | +`/python-cli-runs/<run_id>/logs` during the run so the dashboard can show | | 7 | +what the user saw in their terminal. | | 8 | +
| | 9 | +Streaming is opt-in via the `share_logs` field on register. The server may | | 10 | +also force-enable streaming for an org regardless of the client's request, | | 11 | +so the CLI always calls register and gates on the response's | | 12 | +`log_streaming_enabled` flag rather than the client's intent. | | 13 | +
| | 14 | +Both calls are best-effort: failures fall back to no-streaming and never | | 15 | +prevent the scan from running. | | 16 | +""" | | 17 | + | | 18 | +import json | | 19 | +import logging | | 20 | +from typing import Optional | | 21 | + | | 22 | +from .cli_client import CliClient | | 23 | + | | 24 | +log = logging.getLogger("socketcli") | | 25 | + | | 26 | + | | 27 | +def register_cli_run( | | 28 | + client: CliClient, | | 29 | + client_version: str, | | 30 | + upload_logs: Optional[bool], | | 31 | +) -> Optional[str]: | | 32 | + """Register a CLI run with the backend. | | 33 | +
| | 34 | + `upload_logs` is the user's tri-state preference (True / False / None); | | 35 | + it's projected to the wire-format `share_logs` and `decline_logs` | | 36 | + booleans here, at the API boundary. | | 37 | +
| | 38 | + Best-effort: any failure (network, malformed response, unexpected JSON | | 39 | + shape, etc.) falls back to no-streaming and must never prevent the | | 40 | + scan from running. The single broad except is intentional. | | 41 | + """ | | 42 | + try: | | 43 | + resp = client.request( | | 44 | + path="python-cli-runs", | | 45 | + method="POST", | | 46 | + payload=json.dumps({ | | 47 | + "client_version": client_version, | | 48 | + "share_logs": upload_logs is True, | | 49 | + "decline_logs": upload_logs is False, | | 50 | + }), | | 51 | + ) | | 52 | + body = resp.json() | | 53 | + if not body.get("log_streaming_enabled"): | | 54 | + log.debug("cli-run register: log streaming not enabled by server") | | 55 | + return None | | 56 | + run_id = body.get("run_id") | | 57 | + if not isinstance(run_id, str) or not run_id: | | 58 | + log.debug(f"cli-run register: enabled but missing run_id in response: {body!r}") | | 59 | + return None | | 60 | + return run_id | | 61 | + except Exception as e: | | 62 | + log.debug(f"cli-run register failed (streaming disabled): {e}") | | 63 | + return None | | 64 | + | | 65 | + | | 66 | +def finalize_cli_run( | | 67 | + client: CliClient, | | 68 | + run_id: str, | | 69 | + status: str = "success", | | 70 | + report_run_id: Optional[str] = None, | | 71 | +) -> None: | | 72 | + try: | | 73 | + client.request( | | 74 | + path=f"python-cli-runs/{run_id}/finalize", | | 75 | + method="POST", | | 76 | + payload=json.dumps({"status": status, "report_run_id": report_run_id}), | | 77 | + ) | | 78 | + except Exception as e: | | 79 | + log.debug(f"cli-run finalize failed (swallowed): {e}") |
Collapse file Copy file name to clipboard +112Lines changed: 112 additions & 0 deletions | Original file line number | Diff line number | Diff line change |
|---|
| | 1 | +"""Buffer the CLI's local log records and POST them in batches to | | 2 | +/python-cli-runs/<run_id>/logs so the dashboard's view of a CLI run | | 3 | +mirrors what the user sees in their terminal. | | 4 | +
| | 5 | +Behavior: | | 6 | +- daemon thread, 5s flush | | 7 | +- swallow all network errors (debug log only) | | 8 | +- skip empty buffers | | 9 | +- drain on shutdown | | 10 | +- at-most-once semantics (failed batches dropped, not retried) | | 11 | +
| | 12 | +A thread-local recursion guard prevents the uploader's own request-error | | 13 | +log lines (emitted by `cli_client.py`'s `socketdev` logger) from being | | 14 | +re-enqueued during a flush. | | 15 | +""" | | 16 | + | | 17 | +import json | | 18 | +import logging | | 19 | +import threading | | 20 | +from datetime import datetime, timezone | | 21 | +from typing import Optional | | 22 | + | | 23 | +from .cli_client import CliClient | | 24 | + | | 25 | +log = logging.getLogger(__name__) | | 26 | + | | 27 | +_FLUSH_GUARD = threading.local() | | 28 | + | | 29 | + | | 30 | +def _now_str() -> str: | | 31 | + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] | | 32 | + | | 33 | + | | 34 | +class BatchedLogUploader: | | 35 | + def __init__( | | 36 | + self, | | 37 | + client: CliClient, | | 38 | + run_id: str, | | 39 | + flush_interval: float = 5.0, | | 40 | + ): | | 41 | + self._client = client | | 42 | + self._run_id = run_id | | 43 | + self._flush_interval = flush_interval | | 44 | + self._buf: list = [] | | 45 | + self._lock = threading.Lock() | | 46 | + self._stop = threading.Event() | | 47 | + self._thread: Optional[threading.Thread] = None | | 48 | + | | 49 | + def add(self, entry: dict) -> None: | | 50 | + with self._lock: | | 51 | + self._buf.append(entry) | | 52 | + | | 53 | + def start(self) -> None: | | 54 | + if self._thread is not None: | | 55 | + return | | 56 | + self._thread = threading.Thread( | | 57 | + target=self._run, | | 58 | + name=f"socket-log-uploader-{self._run_id[:8]}", | | 59 | + daemon=True, | | 60 | + ) | | 61 | + self._thread.start() | | 62 | + | | 63 | + def stop(self, timeout: float = 2.0) -> None: | | 64 | + if self._thread is not None: | | 65 | + self._stop.set() | | 66 | + self._thread.join(timeout=timeout) | | 67 | + self._thread = None | | 68 | + self._flush() | | 69 | + | | 70 | + def _run(self) -> None: | | 71 | + while not self._stop.is_set(): | | 72 | + self._flush() | | 73 | + self._stop.wait(self._flush_interval) | | 74 | + | | 75 | + def _flush(self) -> None: | | 76 | + with self._lock: | | 77 | + if not self._buf: | | 78 | + return | | 79 | + batch = self._buf | | 80 | + self._buf = [] | | 81 | + | | 82 | + _FLUSH_GUARD.active = True | | 83 | + try: | | 84 | + self._client.request( | | 85 | + path=f"python-cli-runs/{self._run_id}/logs", | | 86 | + method="POST", | | 87 | + payload=json.dumps({"logs": batch}), | | 88 | + ) | | 89 | + except Exception as e: | | 90 | + log.debug(f"log upload failed (swallowed, {len(batch)} entries dropped): {e}") | | 91 | + finally: | | 92 | + _FLUSH_GUARD.active = False | | 93 | + | | 94 | + | | 95 | +class UploadingLogHandler(logging.Handler): | | 96 | + def __init__(self, uploader: BatchedLogUploader, context: str = "socket-python-cli"): | | 97 | + super().__init__() | | 98 | + self._uploader = uploader | | 99 | + self._context = context | | 100 | + | | 101 | + def emit(self, record: logging.LogRecord) -> None: | | 102 | + if getattr(_FLUSH_GUARD, "active", False): | | 103 | + return | | 104 | + try: | | 105 | + self._uploader.add({ | | 106 | + "timestamp": _now_str(), | | 107 | + "level": logging.getLevelName(record.levelno), | | 108 | + "message": self.format(record), | | 109 | + "context": self._context, | | 110 | + }) | | 111 | + except Exception: | | 112 | + self.handleError(record) |
Collapse file Copy file name to clipboard +112Lines changed: 112 additions & 0 deletions | Original file line number | Diff line number | Diff line change |
|---|
| | 1 | +"""Server log streaming pipeline for one CLI run. | | 2 | +
| | 3 | +`StreamingLogs` is a context manager. On enter it registers a run with the | | 4 | +backend, attaches handlers that route the CLI's own log output through both | | 5 | +the local terminal and a batched uploader, and forces the loggers into DEBUG | | 6 | +so the upload captures everything regardless of local terminal verbosity. | | 7 | +On exit it tears the handlers back down and finalizes the run; the status | | 8 | +sent to finalize is inferred from the exception that closed the `with` | | 9 | +block (success / failure / cancelled). | | 10 | +
| | 11 | +If registration fails the manager becomes a no-op — nothing is wired up and | | 12 | +__exit__ does nothing. | | 13 | +""" | | 14 | + | | 15 | +import logging | | 16 | +from typing import Optional | | 17 | + | | 18 | +from .cli_client import CliClient | | 19 | +from .cli_run import finalize_cli_run, register_cli_run | | 20 | +from .log_uploader import BatchedLogUploader, UploadingLogHandler | | 21 | + | | 22 | + | | 23 | +class StreamingLogs: | | 24 | + def __init__( | | 25 | + self, | | 26 | + *, | | 27 | + client: CliClient, | | 28 | + cli_logger: logging.Logger, | | 29 | + sdk_logger: logging.Logger, | | 30 | + client_version: str, | | 31 | + upload_logs: Optional[bool], | | 32 | + enable_debug: bool, | | 33 | + ): | | 34 | + self._client = client | | 35 | + self._loggers = (cli_logger, sdk_logger) | | 36 | + self._client_version = client_version | | 37 | + self._upload_logs = upload_logs | | 38 | + self._enable_debug = enable_debug | | 39 | + | | 40 | + self._run_id: Optional[str] = None | | 41 | + self._report_run_id: Optional[str] = None | | 42 | + self._uploader: Optional[BatchedLogUploader] = None | | 43 | + self._upload_handler: Optional[UploadingLogHandler] = None | | 44 | + self._terminal_handler: Optional[logging.StreamHandler] = None | | 45 | + self._saved_levels: tuple = () | | 46 | + self._saved_propagate: tuple = () | | 47 | + | | 48 | + def set_report_run_id(self, report_run_id: Optional[str]) -> None: | | 49 | + self._report_run_id = report_run_id | | 50 | + | | 51 | + def __enter__(self) -> "StreamingLogs": | | 52 | + self._run_id = register_cli_run( | | 53 | + self._client, | | 54 | + client_version=self._client_version, | | 55 | + upload_logs=self._upload_logs, | | 56 | + ) | | 57 | + cli_logger = self._loggers[0] | | 58 | + if not self._run_id: | | 59 | + cli_logger.debug("server log streaming not active for this run") | | 60 | + return self | | 61 | + | | 62 | + self._uploader = BatchedLogUploader(self._client, self._run_id) | | 63 | + self._uploader.start() | | 64 | + self._upload_handler = UploadingLogHandler(self._uploader, context="socket-python-cli") | | 65 | + self._upload_handler.setFormatter(logging.Formatter("%(message)s")) | | 66 | + | | 67 | + self._terminal_handler = logging.StreamHandler() | | 68 | + self._terminal_handler.setLevel(logging.DEBUG if self._enable_debug else logging.INFO) | | 69 | + self._terminal_handler.setFormatter(logging.Formatter("%(asctime)s: %(message)s")) | | 70 | + | | 71 | + self._saved_levels = tuple(lg.level for lg in self._loggers) | | 72 | + self._saved_propagate = tuple(lg.propagate for lg in self._loggers) | | 73 | + for lg in self._loggers: | | 74 | + lg.setLevel(logging.DEBUG) | | 75 | + lg.propagate = False | | 76 | + lg.addHandler(self._terminal_handler) | | 77 | + lg.addHandler(self._upload_handler) | | 78 | + | | 79 | + cli_logger.debug(f"server log streaming enabled (run_id={self._run_id})") | | 80 | + return self | | 81 | + | | 82 | + def __exit__(self, exc_type, exc_val, exc_tb) -> bool: | | 83 | + if self._run_id is None: | | 84 | + return False | | 85 | + | | 86 | + status = self._status_for_exit(exc_type, exc_val) | | 87 | + for lg in self._loggers: | | 88 | + lg.removeHandler(self._upload_handler) | | 89 | + self._uploader.stop() | | 90 | + finalize_cli_run( | | 91 | + self._client, | | 92 | + self._run_id, | | 93 | + status=status, | | 94 | + report_run_id=self._report_run_id, | | 95 | + ) | | 96 | + for lg in self._loggers: | | 97 | + lg.removeHandler(self._terminal_handler) | | 98 | + for lg, level, propagate in zip(self._loggers, self._saved_levels, self._saved_propagate): | | 99 | + lg.setLevel(level) | | 100 | + lg.propagate = propagate | | 101 | + return False | | 102 | + | | 103 | + @staticmethod | | 104 | + def _status_for_exit(exc_type, exc_val) -> str: | | 105 | + if exc_type is None: | | 106 | + return "success" | | 107 | + if issubclass(exc_type, KeyboardInterrupt): | | 108 | + return "cancelled" | | 109 | + # SystemExit with code 0 / None is a clean exit; non-zero codes signal failure. | | 110 | + if issubclass(exc_type, SystemExit) and not getattr(exc_val, "code", None): | | 111 | + return "success" | | 112 | + return "failure" |
|