Skip to content

Commit 18dc8ce

Browse files
nathanschramclaude
andcommitted
fix: v0.34.1 — stall diagnostics, liveness watchdog, stream threading (#97, #98)
- /proc process diagnostics: CPU, RSS, TCP, FDs, children (proc_diag.py) - JsonlStreamState event tracking: timestamps, event type, ring buffer, stderr - Progressive stall warnings with /proc snapshots + Telegram notifications - Liveness watchdog: alive-but-silent detection after 10min, optional auto-kill - session.summary structured log on every completion - [watchdog] config section for liveness_timeout, stall_auto_kill, stall_repeat - Fix _ResumeLineProxy hiding current_stream from ProgressEdits (#98) - Fix Claude run_impl missing self.current_stream + stderr_capture threading Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 0ee3c55 commit 18dc8ce

12 files changed

Lines changed: 883 additions & 49 deletions

File tree

CHANGELOG.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,22 @@
11
# changelog
22

3+
## v0.34.1 (2026-03-07)
4+
5+
### fixes
6+
7+
- session stall diagnostics: add `/proc` process diagnostics (CPU, RSS, TCP, FDs, children), progressive stall warnings, liveness watchdog, event timeline tracking, and session completion summary [#97](https://github.com/littlebearapps/untether/issues/97)
8+
- new `utils/proc_diag.py` module: `collect_proc_diag()`, `format_diag()`, `is_cpu_active()`
9+
- `JsonlStreamState` tracks `last_stdout_at`, `event_count`, `last_event_type`, `recent_events` ring buffer, `stderr_capture`
10+
- PID auto-injected into `StartedEvent.meta` via base class (all engines)
11+
- progressive `_stall_monitor`: repeating warnings every 3 min with fresh `/proc` snapshots and Telegram notifications
12+
- liveness watchdog: detects alive-but-silent subprocesses after 10 min with diagnostics; optional auto-kill (off by default, triple safety gate)
13+
- `session.summary` structured log on every session completion
14+
- `[watchdog]` config section: `liveness_timeout`, `stall_auto_kill`, `stall_repeat_seconds`
15+
- stream threading broken: `_ResumeLineProxy` hides `current_stream` from `ProgressEdits`, causing `event_count=0` and `last_event_type=None` for all engines [#98](https://github.com/littlebearapps/untether/issues/98)
16+
- add `current_stream` property to `_ResumeLineProxy` and `_PreludeRunner`
17+
- set `self.current_stream = stream` in Claude's overridden `run_impl`
18+
- use `stream.stderr_capture` instead of separate `stderr_lines` in Claude's `run_impl`
19+
320
## v0.34.0 (2026-03-07)
421

522
### fixes

pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
name = "untether"
33
authors = [{name = "Little Bear Apps", email = "hello@littlebearapps.com"}]
44
maintainers = [{name = "Little Bear Apps", email = "hello@littlebearapps.com"}]
5-
version = "0.34.0"
5+
version = "0.34.1"
66
keywords = ["telegram", "claude-code", "codex", "opencode", "pi", "gemini-cli", "amp", "ai-agents", "coding-assistant", "remote-control", "cli-bridge"]
77
description = "Run AI coding agents from your phone. Bridges Claude Code, Codex, OpenCode, Pi, Gemini CLI, and Amp to Telegram with interactive permissions, voice input, cost tracking, and live progress."
88
readme = {file = "README.md", content-type = "text/markdown"}

src/untether/runner.py

Lines changed: 106 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,15 @@
22

33
from __future__ import annotations
44

5+
import contextlib
56
import json
67
import re
78
import signal
89
import subprocess
10+
import time
11+
from collections import deque
912
from collections.abc import AsyncIterator, Callable
10-
from dataclasses import dataclass
13+
from dataclasses import dataclass, field, replace
1114
from typing import Any, Protocol, cast
1215
from weakref import WeakValueDictionary
1316

@@ -192,9 +195,21 @@ class JsonlStreamState:
192195
did_emit_completed: bool = False
193196
ignored_after_completed: bool = False
194197
jsonl_seq: int = 0
198+
# Activity tracking for stall diagnostics
199+
last_stdout_at: float = 0.0
200+
last_event_type: str | None = None
201+
last_event_tool: str | None = None
202+
event_count: int = 0
203+
recent_events: deque[tuple[float, str]] = field(
204+
default_factory=lambda: deque(maxlen=10)
205+
)
206+
stderr_capture: list[str] = field(default_factory=list)
195207

196208

197209
class JsonlSubprocessRunner(BaseRunner):
210+
# Exposed for diagnostics — set during run_impl, cleared on exit
211+
current_stream: JsonlStreamState | None = None
212+
198213
def get_logger(self) -> Any:
199214
return getattr(self, "logger", get_logger(__name__))
200215

@@ -600,6 +615,10 @@ def _handle_jsonl_line(
600615
line = raw_line.strip()
601616
if not line:
602617
return []
618+
# Track raw I/O activity
619+
now = time.monotonic()
620+
stream.last_stdout_at = now
621+
stream.event_count += 1
603622
stream.jsonl_seq += 1
604623
seq = stream.jsonl_seq
605624
events = self._decode_jsonl_events(
@@ -612,9 +631,37 @@ def _handle_jsonl_line(
612631
logger=logger,
613632
pid=pid,
614633
)
634+
# Peek at raw JSON for event timeline (engine-agnostic)
635+
try:
636+
raw_dict = json.loads(line)
637+
except (json.JSONDecodeError, ValueError):
638+
raw_dict = None
639+
if isinstance(raw_dict, dict):
640+
etype = str(raw_dict.get("type", "unknown"))
641+
etool = None
642+
# Cover common engine conventions for tool name
643+
for key in ("tool_name", "tool", "name"):
644+
val = raw_dict.get(key)
645+
if isinstance(val, str) and val:
646+
etool = val
647+
break
648+
# Also check nested item.type for Codex-style events
649+
item = raw_dict.get("item")
650+
if etool is None and isinstance(item, dict):
651+
itype = item.get("type")
652+
if isinstance(itype, str) and itype:
653+
etool = itype
654+
stream.last_event_type = etype
655+
stream.last_event_tool = etool
656+
label = f"tool:{etool}" if etool else etype
657+
stream.recent_events.append((now, label))
615658
output: list[UntetherEvent] = []
616659
for evt in events:
617660
if isinstance(evt, StartedEvent):
661+
# Inject subprocess PID into meta for diagnostics
662+
meta = dict(evt.meta) if evt.meta else {}
663+
meta["pid"] = pid
664+
evt = replace(evt, meta=meta)
618665
stream.found_session, emit = self._process_started_event(
619666
evt,
620667
expected_session=stream.expected_session,
@@ -663,6 +710,10 @@ async def _iter_jsonl_events(
663710

664711
_WATCHDOG_POLL_SECONDS: float = 0.5
665712

713+
_LIVENESS_TIMEOUT_SECONDS: float = 600.0
714+
715+
_stall_auto_kill: bool = False
716+
666717
async def _subprocess_watchdog(
667718
self,
668719
proc: Any,
@@ -678,15 +729,66 @@ async def _subprocess_watchdog(
678729
process death (``proc.wait()`` blocks until pipes drain, so we use
679730
``os.kill(pid, 0)``), then after a grace period kills the process group
680731
to terminate orphan children and unblock the readers.
732+
733+
Also detects liveness stalls: process alive but no stdout for
734+
``_LIVENESS_TIMEOUT_SECONDS``.
681735
"""
682736
import os as _os
683737

738+
from .utils.proc_diag import collect_proc_diag, is_cpu_active
739+
740+
liveness_warned = False
741+
prev_diag = None
742+
684743
# Poll until the process is dead or the reader finishes.
685744
while not reader_done.is_set():
686745
try:
687746
_os.kill(pid, 0)
688747
except (ProcessLookupError, PermissionError):
689748
break # process exited
749+
750+
# Liveness stall detection
751+
if (
752+
not liveness_warned
753+
and stream.last_stdout_at > 0
754+
and not stream.did_emit_completed
755+
):
756+
idle = time.monotonic() - stream.last_stdout_at
757+
if idle >= self._LIVENESS_TIMEOUT_SECONDS:
758+
liveness_warned = True
759+
diag = collect_proc_diag(pid)
760+
cpu_active = is_cpu_active(prev_diag, diag)
761+
recent = list(stream.recent_events)[-5:]
762+
logger.warning(
763+
"subprocess.liveness_stall",
764+
pid=pid,
765+
idle_seconds=round(idle, 1),
766+
event_count=stream.event_count,
767+
last_event_type=stream.last_event_type,
768+
tcp_established=diag.tcp_established if diag else None,
769+
rss_kb=diag.rss_kb if diag else None,
770+
cpu_active=cpu_active,
771+
recent_events=[(round(t, 1), lbl) for t, lbl in recent],
772+
)
773+
# Auto-kill: config enabled + zero TCP + CPU NOT active
774+
if (
775+
self._stall_auto_kill
776+
and diag is not None
777+
and diag.tcp_established == 0
778+
and diag.alive
779+
and cpu_active is not True
780+
):
781+
logger.warning(
782+
"subprocess.liveness_kill",
783+
pid=pid,
784+
reason="zero_tcp_zero_cpu",
785+
)
786+
with contextlib.suppress(
787+
ProcessLookupError, PermissionError, OSError
788+
):
789+
_os.killpg(pid, signal.SIGKILL)
790+
prev_diag = diag
791+
690792
await anyio.sleep(self._WATCHDOG_POLL_SECONDS)
691793
if stream.did_emit_completed or reader_done.is_set():
692794
return
@@ -755,7 +857,7 @@ async def run_impl(
755857
await self._send_payload(proc, payload, logger=logger, resume=resume)
756858

757859
stream = JsonlStreamState(expected_session=resume)
758-
stderr_lines: list[str] = []
860+
self.current_stream = stream
759861
reader_done = anyio.Event()
760862

761863
async with anyio.create_task_group() as tg:
@@ -764,7 +866,7 @@ async def run_impl(
764866
proc.stderr,
765867
logger,
766868
tag,
767-
stderr_lines,
869+
stream.stderr_capture,
768870
)
769871
tg.start_soon(
770872
self._subprocess_watchdog,
@@ -796,7 +898,7 @@ async def run_impl(
796898
resume=resume,
797899
found_session=found_session,
798900
state=state,
799-
stderr_lines=stderr_lines or None,
901+
stderr_lines=stream.stderr_capture or None,
800902
)
801903
for evt in events:
802904
if isinstance(evt, CompletedEvent):

0 commit comments

Comments
 (0)