22
33from __future__ import annotations
44
5+ import contextlib
56import json
67import re
78import signal
89import subprocess
10+ import time
11+ from collections import deque
912from collections .abc import AsyncIterator , Callable
10- from dataclasses import dataclass
13+ from dataclasses import dataclass , field , replace
1114from typing import Any , Protocol , cast
1215from weakref import WeakValueDictionary
1316
@@ -192,9 +195,21 @@ class JsonlStreamState:
192195 did_emit_completed : bool = False
193196 ignored_after_completed : bool = False
194197 jsonl_seq : int = 0
198+ # Activity tracking for stall diagnostics
199+ last_stdout_at : float = 0.0
200+ last_event_type : str | None = None
201+ last_event_tool : str | None = None
202+ event_count : int = 0
203+ recent_events : deque [tuple [float , str ]] = field (
204+ default_factory = lambda : deque (maxlen = 10 )
205+ )
206+ stderr_capture : list [str ] = field (default_factory = list )
195207
196208
197209class JsonlSubprocessRunner (BaseRunner ):
210+ # Exposed for diagnostics — set during run_impl, cleared on exit
211+ current_stream : JsonlStreamState | None = None
212+
198213 def get_logger (self ) -> Any :
199214 return getattr (self , "logger" , get_logger (__name__ ))
200215
@@ -600,6 +615,10 @@ def _handle_jsonl_line(
600615 line = raw_line .strip ()
601616 if not line :
602617 return []
618+ # Track raw I/O activity
619+ now = time .monotonic ()
620+ stream .last_stdout_at = now
621+ stream .event_count += 1
603622 stream .jsonl_seq += 1
604623 seq = stream .jsonl_seq
605624 events = self ._decode_jsonl_events (
@@ -612,9 +631,37 @@ def _handle_jsonl_line(
612631 logger = logger ,
613632 pid = pid ,
614633 )
634+ # Peek at raw JSON for event timeline (engine-agnostic)
635+ try :
636+ raw_dict = json .loads (line )
637+ except (json .JSONDecodeError , ValueError ):
638+ raw_dict = None
639+ if isinstance (raw_dict , dict ):
640+ etype = str (raw_dict .get ("type" , "unknown" ))
641+ etool = None
642+ # Cover common engine conventions for tool name
643+ for key in ("tool_name" , "tool" , "name" ):
644+ val = raw_dict .get (key )
645+ if isinstance (val , str ) and val :
646+ etool = val
647+ break
648+ # Also check nested item.type for Codex-style events
649+ item = raw_dict .get ("item" )
650+ if etool is None and isinstance (item , dict ):
651+ itype = item .get ("type" )
652+ if isinstance (itype , str ) and itype :
653+ etool = itype
654+ stream .last_event_type = etype
655+ stream .last_event_tool = etool
656+ label = f"tool:{ etool } " if etool else etype
657+ stream .recent_events .append ((now , label ))
615658 output : list [UntetherEvent ] = []
616659 for evt in events :
617660 if isinstance (evt , StartedEvent ):
661+ # Inject subprocess PID into meta for diagnostics
662+ meta = dict (evt .meta ) if evt .meta else {}
663+ meta ["pid" ] = pid
664+ evt = replace (evt , meta = meta )
618665 stream .found_session , emit = self ._process_started_event (
619666 evt ,
620667 expected_session = stream .expected_session ,
@@ -663,6 +710,10 @@ async def _iter_jsonl_events(
663710
664711 _WATCHDOG_POLL_SECONDS : float = 0.5
665712
713+ _LIVENESS_TIMEOUT_SECONDS : float = 600.0
714+
715+ _stall_auto_kill : bool = False
716+
666717 async def _subprocess_watchdog (
667718 self ,
668719 proc : Any ,
@@ -678,15 +729,66 @@ async def _subprocess_watchdog(
678729 process death (``proc.wait()`` blocks until pipes drain, so we use
679730 ``os.kill(pid, 0)``), then after a grace period kills the process group
680731 to terminate orphan children and unblock the readers.
732+
733+ Also detects liveness stalls: process alive but no stdout for
734+ ``_LIVENESS_TIMEOUT_SECONDS``.
681735 """
682736 import os as _os
683737
738+ from .utils .proc_diag import collect_proc_diag , is_cpu_active
739+
740+ liveness_warned = False
741+ prev_diag = None
742+
684743 # Poll until the process is dead or the reader finishes.
685744 while not reader_done .is_set ():
686745 try :
687746 _os .kill (pid , 0 )
688747 except (ProcessLookupError , PermissionError ):
689748 break # process exited
749+
750+ # Liveness stall detection
751+ if (
752+ not liveness_warned
753+ and stream .last_stdout_at > 0
754+ and not stream .did_emit_completed
755+ ):
756+ idle = time .monotonic () - stream .last_stdout_at
757+ if idle >= self ._LIVENESS_TIMEOUT_SECONDS :
758+ liveness_warned = True
759+ diag = collect_proc_diag (pid )
760+ cpu_active = is_cpu_active (prev_diag , diag )
761+ recent = list (stream .recent_events )[- 5 :]
762+ logger .warning (
763+ "subprocess.liveness_stall" ,
764+ pid = pid ,
765+ idle_seconds = round (idle , 1 ),
766+ event_count = stream .event_count ,
767+ last_event_type = stream .last_event_type ,
768+ tcp_established = diag .tcp_established if diag else None ,
769+ rss_kb = diag .rss_kb if diag else None ,
770+ cpu_active = cpu_active ,
771+ recent_events = [(round (t , 1 ), lbl ) for t , lbl in recent ],
772+ )
773+ # Auto-kill: config enabled + zero TCP + CPU NOT active
774+ if (
775+ self ._stall_auto_kill
776+ and diag is not None
777+ and diag .tcp_established == 0
778+ and diag .alive
779+ and cpu_active is not True
780+ ):
781+ logger .warning (
782+ "subprocess.liveness_kill" ,
783+ pid = pid ,
784+ reason = "zero_tcp_zero_cpu" ,
785+ )
786+ with contextlib .suppress (
787+ ProcessLookupError , PermissionError , OSError
788+ ):
789+ _os .killpg (pid , signal .SIGKILL )
790+ prev_diag = diag
791+
690792 await anyio .sleep (self ._WATCHDOG_POLL_SECONDS )
691793 if stream .did_emit_completed or reader_done .is_set ():
692794 return
@@ -755,7 +857,7 @@ async def run_impl(
755857 await self ._send_payload (proc , payload , logger = logger , resume = resume )
756858
757859 stream = JsonlStreamState (expected_session = resume )
758- stderr_lines : list [ str ] = []
860+ self . current_stream = stream
759861 reader_done = anyio .Event ()
760862
761863 async with anyio .create_task_group () as tg :
@@ -764,7 +866,7 @@ async def run_impl(
764866 proc .stderr ,
765867 logger ,
766868 tag ,
767- stderr_lines ,
869+ stream . stderr_capture ,
768870 )
769871 tg .start_soon (
770872 self ._subprocess_watchdog ,
@@ -796,7 +898,7 @@ async def run_impl(
796898 resume = resume ,
797899 found_session = found_session ,
798900 state = state ,
799- stderr_lines = stderr_lines or None ,
901+ stderr_lines = stream . stderr_capture or None ,
800902 )
801903 for evt in events :
802904 if isinstance (evt , CompletedEvent ):
0 commit comments