// process.rs --- T M4.4 process supervisor. //! Process supervisor: spawn, monitor, signal, restart, reap child //! processes. Pipe and PTY stdio. Per spec §3 (system overview) and //! §5 (concurrency). //! //! # Topology //! //! The supervisor itself is **main-thread** state, like //! [`crate::async_runtime::AsyncRuntime`]. Per-process I/O reader //! threads stream stdout/stderr (or PTY master output) over a //! crossbeam channel; [`ProcessSupervisor::tick`] drains the channel //! into per-process event queues and polls each running child for //! exit, applying the configured [`RestartPolicy`] when a process //! terminates. //! //! # Lifecycle //! //! ```text //! Starting --(spawn ok)--> Running //! Running --(SIGTERM/kill)--> Exiting //! Running --(child exited)--> Terminated //! Exiting --(child exited)--> Terminated //! Terminated --(restart policy + backoff)--> Starting //! ``` //! //! Every transition emits a [`ProcessEvent`] visible through //! [`ProcessSupervisor::take_events`]. The main thread (and the Lua //! surface in [`crate::lua_bindings`]) is the sole consumer. //! //! # Cleanup //! //! [`ProcessSupervisor::shutdown`] sends SIGTERM to every running //! child, waits up to a grace period, then SIGKILLs anything still //! alive. The supervisor's `Drop` impl calls `shutdown` so that an //! editor exit (panic or normal) cannot leave zombies. The reader //! threads join when their pipe end closes (which the kernel does //! once the child is reaped) so they don't need explicit teardown. //! //! # `unsafe_code` boundary //! //! pmacs's crate-level `unsafe_code = "forbid"` lint stands. Signal //! sending uses [`nix`] (safe wrapper around `kill(2)`); PTY //! support uses [`portable-pty`], which contains internal `unsafe` //! but exposes a fully safe surface. PTY line-discipline setup //! (raw/canonical mode per spec §sec:repl-supervisor) bridges to //! `tcsetattr(3)` via a `/bin/sh` trampoline instead of a local //! `unsafe` block — see [`build_pty_command`] for the full //! rationale. use std::collections::HashMap; use std::io::{Read, Write}; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; use std::time::{Duration, Instant}; use crossbeam::channel::{self, Receiver, Sender}; use nix::sys::signal::Signal; use nix::unistd::Pid; use crate::ansi::{AnsiEvent, AnsiParser, AnsiParserProfile}; // --------------------------------------------------------------------------- // Identity and configuration // --------------------------------------------------------------------------- /// Stable identifier for a managed process. Allocated in monotonic /// order from a process-wide counter. A restart re-uses the same /// id (the *managed* process is the same; the OS pid changes per /// generation). #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] pub struct ProcessId(u64); impl ProcessId { /// Mint a fresh id. #[must_use] pub fn next() -> Self { static COUNTER: AtomicU64 = AtomicU64::new(1); Self(COUNTER.fetch_add(1, Ordering::Relaxed)) } /// Raw counter value. Useful for debug formatting and Lua /// boundary marshalling. #[must_use] pub fn raw(self) -> u64 { self.0 } } impl std::fmt::Display for ProcessId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "ProcessId({})", self.0) } } /// Line-discipline configuration for a PTY-mode child. /// /// Per spec §sec:repl-supervisor: REPLs run in raw mode by default /// so the child controls echo (the kernel does not echo input back /// to the master). Canonical mode is preserved as a fallback for /// non-shell line-oriented filters that `read()` from stdin and /// expect kernel line buffering. /// /// The mode is applied to the master's termios after `openpty` and /// before the child is spawned, so the child inherits the mode /// from its very first read. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum TerminalMode { /// `cfmakeraw`-equivalent: no kernel echo, no canonical /// processing, no signal generation from input characters. /// Shells, language REPLs, and anything that calls `tcsetattr` /// itself want this default. Raw, /// Kernel default: line-buffered input, kernel echo on, signal /// characters interpreted (`Ctrl-C` raises `SIGINT`, etc.). /// Useful for line-oriented filters. Canonical, } /// I/O mode for a managed process. /// /// `Pipes` allocates separate stdout / stderr readers and a stdin /// writer, all unbuffered byte streams. `Pty` allocates a single /// pty pair: stdin is the master writer, output is the master /// reader (stdout + stderr are merged at the kernel level), and the /// child sees a controlling tty. PTY mode is what enables /// terminal-aware children (REPLs that probe `isatty`, programs that /// emit ANSI escape sequences when interactive, etc.) per the M4.4 /// acceptance criterion. The `mode` field selects the line /// discipline per spec §sec:repl-supervisor. #[derive(Clone, Copy, Debug)] pub enum ProcessMode { /// Three plain pipes (stdin, stdout, stderr). Pipes, /// PTY pair sized to `(rows, cols)`. The child's stdin/stdout/ /// stderr are all the pty slave; the supervisor holds the /// master. Pty { /// Rows in the pty's window size (`TIOCSWINSZ`). rows: u16, /// Cols in the pty's window size. cols: u16, /// Line discipline applied before the child is spawned. mode: TerminalMode, }, } impl ProcessMode { /// Convenience: a 24x80 PTY in raw mode (the conventional REPL /// default per spec §sec:repl-supervisor). #[must_use] pub const fn default_pty() -> Self { Self::Pty { rows: 24, cols: 80, mode: TerminalMode::Raw, } } } /// Stdin disposition for a pipe-mode child. /// /// Compile-mode (Q#CM3) runs noninteractive commands that may probe /// or read stdin (`cat`, tools that block on a tty check); `Null` /// gives them immediate EOF from `/dev/null` with no writer thread /// and no close-after-spawn race. PTY children have no separable /// stdin, so `Null` is rejected at spawn under PTY mode. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum StdinMode { /// Piped writer thread (the default; see [`StdinWriter`]). Piped, /// `/dev/null`: immediate EOF; `write_stdin` errors with the /// stdin-not-piped message. Null, } /// What to do when a managed process terminates. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum RestartPolicy { /// Never restart. The process stays in /// [`ProcessState::Terminated`] forever. Never, /// Restart only on a non-clean exit (signal or crash). OnCrash, /// Restart on any termination (clean or otherwise). Always, } /// Description of a managed process. #[derive(Clone, Debug)] pub struct ProcessSpec { /// Human-readable label. Surfaced in events and the /// `pmacs.process.list` output. Distinct from the program name /// so multiple processes can run the same binary with /// distinguishable labels. pub label: String, /// Program to execute. Looked up via the system PATH unless an /// absolute path is supplied. pub command: String, /// Argument vector (does *not* include `argv[0]`; the supervisor /// supplies that). pub args: Vec, /// Working directory. `None` inherits from the editor process. pub cwd: Option, /// Environment variables to set / override for the child. Each /// entry replaces the inherited value; absent entries are /// inherited untouched. pub env: Vec<(String, String)>, /// I/O mode (pipes vs pty). pub mode: ProcessMode, /// What to do on termination. pub restart: RestartPolicy, /// Parse PTY output on a worker and emit structured ANSI events /// instead of raw stdout bytes. Opt-in so LSP and other byte-stream /// consumers keep their existing stdout/stderr contract. pub ansi_events: bool, /// Compatibility profile for structured ANSI parsing. Ignored unless /// `ansi_events` is true; ordinary process/Lua callers remain line-oriented. pub ansi_profile: AnsiParserProfile, /// Stdin disposition (pipe-mode only; rejected under PTY). pub stdin: StdinMode, /// Compile-mode group lifecycle (Q#CM3; pipe-mode only, rejected /// under PTY — PTY children already lead their own session). /// When set: the child is spawned as the leader of a fresh /// process group (`process_group(0)`), fatal signals are /// group-directed (negative pid, mirroring the PTY branch of /// [`signal_target`]), the group receives SIGTERM and enters the /// liveness-probed reap ledger on the leader's terminal event, /// and the generation's readers are poll-based and cancellable /// so teardown is bounded even when an escaped descendant holds /// the output pipe. pub group: bool, } impl ProcessSpec { /// Construct a spec with the bare-minimum fields. Convenience /// for tests and one-off scripts. #[must_use] pub fn new(label: impl Into, command: impl Into) -> Self { Self { label: label.into(), command: command.into(), args: Vec::new(), cwd: None, env: Vec::new(), mode: ProcessMode::Pipes, restart: RestartPolicy::Never, ansi_events: false, ansi_profile: AnsiParserProfile::LineOriented, stdin: StdinMode::Piped, group: false, } } } // --------------------------------------------------------------------------- // Lifecycle state // --------------------------------------------------------------------------- /// State of one managed process generation. /// /// A *generation* is one spawn-to-terminate cycle. Restarts mint a /// new generation under the same [`ProcessId`]. #[derive(Clone, Debug)] pub enum ProcessState { /// The supervisor is between `spawn` and the kernel returning a /// pid (very brief). Reflects "we asked for a fork, haven't /// observed the child yet." Starting, /// Child is alive. `pid` is the OS process id; `started` is the /// instant the supervisor observed the spawn. Running { /// OS process id. pid: u32, /// When the supervisor observed the spawn. started: Instant, }, /// SIGTERM (or equivalent) has been sent; the supervisor is /// waiting for the child to actually exit. Distinct from /// `Running` so a `kill`-then-respawn caller doesn't /// double-signal. Exiting { /// Last known OS pid (before signal). pid: u32, /// When the supervisor sent the terminating signal. signaled_at: Instant, }, /// Final state for this generation. Restart policy may transition /// back to `Starting` later. Terminated(Termination), } /// Reason a generation ended. #[derive(Clone, Debug)] pub enum Termination { /// Clean exit with a status code. Exited { /// Exit code as reported by the OS. code: i32, /// When the spawn happened. started: Instant, /// When the exit was observed. ended: Instant, }, /// Killed by signal. `signal` is the symbolic name (e.g. /// `"SIGTERM"`). Signaled { /// Symbolic signal name as reported by the OS. signal: String, /// When the spawn happened. started: Instant, /// When the signal was observed. ended: Instant, }, /// The supervisor itself failed to spawn or interact with the /// child --- the child never reached `Running`, or the kernel /// returned an error during a poll. `error` carries the /// `Display`-formatted error. Crashed { /// Failure description. error: String, /// When the supervisor noticed the failure. ended: Instant, }, } // --------------------------------------------------------------------------- // Events // --------------------------------------------------------------------------- /// One event in a process's lifecycle. #[derive(Clone, Debug)] pub struct ProcessEvent { /// Process the event belongs to. pub id: ProcessId, /// What happened. pub kind: ProcessEventKind, /// When the event was generated. Monotonic; useful for ordering /// and replay. pub at: Instant, } /// Discriminator over what kind of thing happened. #[derive(Clone, Debug)] pub enum ProcessEventKind { /// `spawn` returned successfully and the kernel reported a pid. Started { /// OS process id of the new generation. pid: u32, }, /// A chunk of bytes from the child's stdout. Stdout(Vec), /// A chunk of bytes from the child's stderr. PTY-mode processes /// never emit `Stderr` (the pty merges output streams); they /// emit only `Stdout`. Stderr(Vec), /// Structured ANSI events decoded from a PTY byte stream on the /// parser worker. Only emitted when [`ProcessSpec::ansi_events`] /// is true. Ansi(Vec), /// Child exited cleanly. Exited { /// Exit code. code: i32, }, /// Child died from a signal. Signaled { /// Symbolic signal name (`"SIGTERM"`, `"SIGKILL"`, etc.). signal: String, }, /// Spawn or supervision failed. Crashed { /// Failure description. error: String, }, /// The supervisor is about to spawn a fresh generation per the /// configured restart policy. `attempt` counts cumulative spawn /// attempts (1 = first spawn, 2 = first restart, ...). Restarting { /// Cumulative spawn attempt number. attempt: u32, }, } // --------------------------------------------------------------------------- // Streaming pipeline constants (T M6.2) // --------------------------------------------------------------------------- /// Size of one read from a child's stdout/stderr or PTY master. /// 8 KiB is a kernel-pipe sweet spot: large enough that small /// outputs land in one read, small enough that a saturating producer /// emits chunks at a steady rate (rather than one giant read after /// a long block). pub const BYTE_CHUNK_SIZE: usize = 8 * 1024; /// In-flight byte ceiling for a single generation's output, per spec /// §sec:repl-streaming. The bounded channel between the reader thread /// and the supervisor is sized so a saturating producer fills the /// channel, then stalls in `send`, then stalls in `read`, then the /// kernel pipe fills, then the child stalls in `write` --- which is /// the right outcome. Default 1 MiB matches the spec's PTY-read → /// parser ceiling. pub const PTY_READ_CEILING_BYTES: usize = 1 << 20; /// Capacity of the per-generation byte chunk channel. With 8 KiB /// chunks and a 1 MiB ceiling, this is 128 slots. const BYTE_CHUNK_CHANNEL_CAP: usize = PTY_READ_CEILING_BYTES / BYTE_CHUNK_SIZE; /// In-flight structured-event ceiling between ANSI parser worker and /// main-thread supervisor drain. The spec names this as 256 KiB; with /// 8 KiB read chunks this is 32 parser batches in flight. pub const ANSI_EVENT_CEILING_BYTES: usize = 256 * 1024; const ANSI_EVENT_CHANNEL_CAP: usize = ANSI_EVENT_CEILING_BYTES / BYTE_CHUNK_SIZE; /// How long a reader thread waits in a bounded `send` before polling /// its cancel flag. 50 ms is long enough that healthy steady-state /// flow doesn't burn cycles re-checking, short enough that a /// shutting-down supervisor sees readers exit promptly. const READER_SEND_POLL_INTERVAL: Duration = Duration::from_millis(50); /// Bounded grace window used when a child has exited but its reader / /// parser worker may still have already-read bytes in flight. This is /// not process termination grace; it is only the final output flush /// before the runtime handles are dropped. const EXIT_OUTPUT_DRAIN_TIMEOUT: Duration = Duration::from_secs(2); /// TERM→KILL escalation window for `group = true` process groups /// (Q#CM3). Armed into the reap ledger when the group receives /// SIGTERM — on explicit kill/supersede and on the leader's terminal /// event — /// and enforced both by the per-tick ledger probe and from inside /// the group-aware final drain loop. Deliberately short: this is /// child-tree cleanup, not polite application shutdown (the polite /// TERM already went out when the window starts). pub const GROUP_TERM_GRACE: Duration = Duration::from_millis(500); // --------------------------------------------------------------------------- // Supervisor // --------------------------------------------------------------------------- /// Owner of all managed processes. One per editor. pub struct ProcessSupervisor { processes: HashMap, events_tx: Sender, events_rx: Receiver, /// Buffered events per process, populated by `tick` from /// `events_rx` and drained by [`Self::take_events`]. pending: HashMap>, /// SIGTERM-then-SIGKILL grace window for shutdown. grace_period: Duration, /// Restart back-off (constant for v0.1; M4.5+ may add /// exponential). restart_backoff: Duration, /// True once `shutdown()` has run; subsequent `spawn` calls /// fail and `maybe_restart` is inert (a `restart = always` /// process must not respawn mid-teardown). shut_down: bool, /// Liveness-probed TERM→KILL reap ledger for `group = true` /// process groups (Q#CM3). Keyed by pgid; independent of the /// managed-process records so it survives `forget` and leader /// exit. Armed insert-if-absent (earliest deadline wins — a /// repeated TERM must not push the SIGKILL bound out). Probed /// every tick with `kill(-pgid, 0)`: ESRCH drops the entry; /// alive past the deadline SIGKILLs the group. `shutdown()` /// force-kills outstanding entries and probes them to ESRCH /// inside its bounded reap loop. reap_ledger: HashMap, /// TERM→KILL window used when arming the ledger. Constant /// [`GROUP_TERM_GRACE`] in production; overridable in tests. group_term_grace: Duration, /// Q#PD4 test seam: forces the next `kill(2)` attempt in /// [`Self::signal`] to fail with this errno, consumed once. /// Always `None` in production — there is no way to set it outside /// `cfg(test)`. It replaces the *kill result only*, so the leader /// observation still runs against the real child handle; a stubbed /// observation would bypass the code path under test. forced_kill_errno: Option, } /// One armed group in the reap ledger. struct GroupReap { /// When to SIGKILL the group if it still probes alive. deadline: Instant, /// SIGKILL already sent — keep probing to ESRCH but don't /// re-kill every tick. killed: bool, } struct ManagedProcess { spec: ProcessSpec, state: ProcessState, runtime: Option, attempt_count: u32, /// When the supervisor should attempt the next restart, or /// `None` if no restart is pending. next_restart_at: Option, } /// Handles tied to one running generation. Dropped (and joined) /// when the generation ends. struct RuntimeHandles { child: ChildHandle, stdin: Option, pid: u32, /// Reader-thread join handles, drained by `Drop` of /// [`RuntimeHandles`] so a generation's worker threads don't /// outlive the supervisor. readers: Vec>, /// Bounded output channel drained by the supervisor. Raw processes /// expose bytes directly; ANSI-enabled PTY processes expose parser /// batches from the worker stage. output_rx: RuntimeOutputRx, /// Cancel flag observed by reader threads when their bounded /// `send` blocks. Set on generation end / supervisor drop so a /// reader stuck in `send` (consumer fell behind) wakes promptly /// instead of leaking until the kernel ends the producer. cancel: Arc, /// Live reader-thread count for this generation, maintained by /// [`spawn_group_reader`] via a drop guard. Unit tests hold a /// clone across teardown as the deterministic proof that the /// joined threads ended and their owned read FDs dropped /// (join-return alone cannot distinguish "never started", and a /// process-global thread/FD count is racy under the parallel /// test runner). Always present — one Arc and two atomics per /// reader lifetime — because cfg-gating the field would spread /// cfg attributes through every construction site; only the /// probe accessor is test-gated, hence the not(test) allow. #[cfg_attr(not(test), allow(dead_code))] active_readers: Arc, } /// Byte budget for stdin data queued but not yet written, per /// generation. A child this far behind on reading its own stdin is /// effectively not consuming it; erroring beats unbounded queue /// growth, and callers already treat `write_stdin` errors as /// process failure. Generous so it never triggers for a merely-busy /// child (LSP full-document didChange on a large file is ~MB-scale). const STDIN_QUEUE_MAX_BYTES: usize = 64 * 1024 * 1024; /// Queued stdin writer: a dedicated thread owns the child's stdin /// handle and drains a channel of byte chunks. This decouples /// callers — the editor main thread, notably the LSP manager's /// full-document `didChange` notifications — from pipe /// backpressure: a child that stops reading (kernel pipe buffers /// are ~64 KiB) stalls this queue, not the editor frame loop. /// /// Closing: dropping the sender (`close_stdin` / generation end) /// lets the thread drain whatever is queued, then drop the handle — /// the child sees EOF *after* the queued bytes, preserving the /// flush-then-EOF shutdown contract MCP relies on. The thread is /// detached rather than joined: joining at drop could block forever /// on a wedged pipe, and generation teardown (SIGTERM/SIGKILL) /// breaks the pipe and ends the thread shortly after anyway. struct StdinWriter { tx: Sender>, /// Bytes accepted by [`Self::write`] but not yet written by the /// thread. Backpressure signal for the queue budget. queued_bytes: Arc, /// First write error observed by the writer thread. Writes are /// asynchronous, so the failure surfaces on the *next* `write` /// call instead of the one that hit it. error: Arc>>, } impl StdinWriter { fn spawn(mut sink: Box) -> Self { let (tx, rx) = channel::unbounded::>(); let queued_bytes = Arc::new(AtomicUsize::new(0)); let error = Arc::new(Mutex::new(None)); let thread_queued = Arc::clone(&queued_bytes); let thread_error = Arc::clone(&error); std::thread::Builder::new() .name("pmacs stdin writer".into()) .spawn(move || { while let Ok(bytes) = rx.recv() { let result = sink.write_all(&bytes).and_then(|()| sink.flush()); thread_queued.fetch_sub(bytes.len(), Ordering::Relaxed); if let Err(e) = result { *thread_error .lock() .expect("stdin writer error mutex poisoned") = Some(e.to_string()); return; } } // Channel closed: all queued chunks written. `sink` // drops here, closing the pipe — the child sees EOF. }) .expect("spawn stdin writer thread"); Self { tx, queued_bytes, error, } } fn write(&self, bytes: &[u8]) -> Result<(), String> { if let Some(e) = self .error .lock() .expect("stdin writer error mutex poisoned") .as_ref() { return Err(format!("write_stdin: {e}")); } let queued = self.queued_bytes.load(Ordering::Relaxed); if queued.saturating_add(bytes.len()) > STDIN_QUEUE_MAX_BYTES { return Err(format!( "write_stdin: child is not draining stdin ({queued} bytes already queued)" )); } self.queued_bytes.fetch_add(bytes.len(), Ordering::Relaxed); self.tx.send(bytes.to_vec()).map_err(|_| { // Thread exited after a write error; report the stored // cause when we have it. self.queued_bytes.fetch_sub(bytes.len(), Ordering::Relaxed); let stored = self .error .lock() .expect("stdin writer error mutex poisoned") .clone(); stored.map_or_else( || "write_stdin: writer thread stopped".to_owned(), |e| format!("write_stdin: {e}"), ) }) } } impl Drop for RuntimeHandles { fn drop(&mut self) { // Wake any reader thread blocked in a bounded `send` --- // dropping the master closes the kernel pipe and unblocks // `read`, but does nothing for a reader stuck on a full // channel because the consumer fell behind. Cancel flag // unwedges that case before we join. T M6.2. self.cancel.store(true, Ordering::Relaxed); for h in std::mem::take(&mut self.readers) { let _ = h.join(); } } } /// Reader-thread output: one chunk read from a stream of a given /// kind (stdout vs stderr; PTY-mode generations only ever emit /// `Stdout`). Lives on the per-generation bounded byte channel. type ByteChunk = (ReaderKind, Vec); type AnsiBatch = Vec; enum RuntimeOutputRx { Bytes(Receiver), Ansi(Receiver), } /// Discriminated wrapper over a pipe-mode `std::process::Child` and /// a pty-mode portable-pty pair. The variants share `try_wait` / /// pid retrieval through a thin enum match. enum ChildHandle { Pipes(std::process::Child), Pty { child: Arc>>, /// Master end held for as long as the generation lives; /// dropping the master closes all reader/writer handles /// derived from it. Held in `_master` even though we only /// access it via the cloned reader/writer to ensure the /// drop order is right. _master: Box, }, } impl ChildHandle { /// Non-blocking poll for child exit. fn try_wait(&mut self) -> Result, String> { match self { Self::Pipes(child) => match child.try_wait() { Ok(None) => Ok(None), Ok(Some(status)) => Ok(Some(TermStatus::from_std(status))), Err(e) => Err(format!("try_wait: {e}")), }, Self::Pty { child, .. } => { let mut guard = child.lock().expect("pty child mutex poisoned"); match guard.try_wait() { Ok(None) => Ok(None), Ok(Some(status)) => Ok(Some(TermStatus::from_pty(&status))), Err(e) => Err(format!("try_wait: {e}")), } } } } } /// Which branch of [`signal_target`] chose the target (Q#PD1). /// /// Recorded on failure because the branches differ in what a failing /// `kill` can possibly mean: only [`Self::LeaderPid`] aims at the /// spawned child itself. The other two aim at a *group*, which for a /// PTY is read from the terminal and can belong to something the /// supervisor never spawned. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum TargetSource { /// The tty's current foreground process group, read at signal /// time. Diverges from the leader exactly when job control has /// moved the terminal. ForegroundGroup, /// A `group = true` pipe child leading its own process group. SpawnGroup, /// The child's own pid. LeaderPid, } impl TargetSource { fn as_str(self) -> &'static str { match self { Self::ForegroundGroup => "tcgetpgrp", Self::SpawnGroup => "group", Self::LeaderPid => "leader-pid", } } /// Whether the target is a process group rather than one process. fn is_group(self) -> bool { matches!(self, Self::ForegroundGroup | Self::SpawnGroup) } } /// The entity a signal was actually aimed at, plus the branch that /// chose it. Carried so a failure can report the target as a fact /// separate from the leader's state (Q#PD1). #[derive(Debug, Clone, Copy)] struct SignalTarget { pid: Pid, source: TargetSource, } fn signal_target(proc: &ManagedProcess, pid: u32) -> Result { if let Some(runtime) = proc.runtime.as_ref() && let ChildHandle::Pty { _master: master, .. } = &runtime.child && let Some(pgrp) = master.process_group_leader() && pgrp > 0 { return Ok(SignalTarget { pid: Pid::from_raw(-pgrp), source: TargetSource::ForegroundGroup, }); } // `group = true` pipe children lead a fresh process group // (`process_group(0)` at spawn ⇒ pgid == pid), so fatal signals // reach the whole `sh -c` tree — mirroring the PTY branch above // (Q#CM3). if proc.spec.group { let pgid = i32::try_from(pid).map_err(|e| e.to_string())?; return Ok(SignalTarget { pid: Pid::from_raw(-pgid), source: TargetSource::SpawnGroup, }); } Ok(SignalTarget { pid: Pid::from_raw(i32::try_from(pid).map_err(|e| e.to_string())?), source: TargetSource::LeaderPid, }) } /// The spawned leader's state at the moment a `kill` failed (Q#PD1). /// /// Deliberately reported *beside* the target rather than folded into a /// verdict: for a PTY the two are different entities whenever job /// control has moved the terminal, and three successive designs for /// this code were unsound precisely because they collapsed them. enum LeaderObservation { Exited(TermStatus), Live, Unobservable(String), NoRuntime, } impl LeaderObservation { fn render(&self) -> String { match self { Self::Exited(TermStatus::Exited(code)) => format!("exited(code {code})"), Self::Exited(TermStatus::Signaled(sig)) => format!("exited(signal {sig})"), Self::Live => "live".to_owned(), Self::Unobservable(e) => format!("unobservable({e})"), Self::NoRuntime => "no-runtime".to_owned(), } } } /// Observe the spawned leader. Note this *reaps* an exited child and /// caches its status; that is why Q#PD3 claims "no disposition change" /// rather than "strictly additive", and why an event-count test pins /// that `poll_one` still emits exactly one exit event afterwards. fn observe_leader(proc: &mut ManagedProcess) -> LeaderObservation { let Some(runtime) = proc.runtime.as_mut() else { return LeaderObservation::NoRuntime; }; match runtime.child.try_wait() { Ok(Some(status)) => LeaderObservation::Exited(status), Ok(None) => LeaderObservation::Live, Err(e) => LeaderObservation::Unobservable(e), } } /// Render a failing `kill` as the five facts of Q#PD1. The disposition /// is unchanged (Q#PD2) — this only replaces a message that said /// nothing but the errno. fn signal_failure_report( target: SignalTarget, leader_pid: u32, errno: nix::errno::Errno, leader: &LeaderObservation, ) -> String { let expected = if target.source.is_group() { match i32::try_from(leader_pid) { Ok(p) => format!(", expected_group=-{p}"), Err(_) => String::new(), } } else { String::new() }; format!( "kill: {errno} (target={} via {}, leader_pid={leader_pid}{expected}, leader={})", target.pid.as_raw(), target.source.as_str(), leader.render(), ) } /// Termination status of one generation. Internal --- the supervisor /// translates this into a [`Termination`] with timing. enum TermStatus { Exited(i32), Signaled(String), } impl TermStatus { fn from_std(status: std::process::ExitStatus) -> Self { #[cfg(unix)] { use std::os::unix::process::ExitStatusExt; if let Some(sig) = status.signal() { // Resolve to a symbolic name (`SIGTERM`, `SIGKILL`, // ...) when the signal number is one nix knows // about; fall back to a numeric placeholder // otherwise. The Display of `nix::sys::signal::Signal` // produces the SIGFOO form. let label = match nix::sys::signal::Signal::try_from(sig) { Ok(s) => s.as_str().to_owned(), Err(_) => format!("SIG{sig}"), }; return Self::Signaled(label); } } Self::Exited(status.code().unwrap_or(-1)) } fn from_pty(status: &portable_pty::ExitStatus) -> Self { if let Some(sig) = status.signal() { // portable-pty stringifies via libc::strsignal, which // returns descriptions ("Interrupt") rather than the // symbolic SIGFOO name. We canonicalize to match the // pipe-mode path (`from_unix`) so M6.5's exit-marker // contract surfaces "SIGINT" identically across modes. Self::Signaled(canonicalize_pty_signal_name(sig)) } else { // portable-pty's exit code is `u32`; values above // i32::MAX are exotic and treated as -1. let code = i32::try_from(status.exit_code()).unwrap_or(-1); Self::Exited(code) } } } /// Map `libc::strsignal` description strings (as surfaced by /// `portable-pty`) to symbolic SIGFOO names. Darwin appends the signal /// number (for example, `"Terminated: 15"`), while glibc returns only /// the description. Unknown descriptions pass through unchanged — /// better to surface an unfamiliar string than to fabricate a wrong /// name. Covers every signal in /// [`super::lua_bindings::parse_signal`]'s accept-list plus the common /// fault signals that surface during process crashes. fn canonicalize_pty_signal_name(desc: &str) -> String { let base = desc .rsplit_once(": ") .filter(|(_, number)| number.parse::().is_ok()) .map_or(desc, |(description, _)| description); match base { "Interrupt" => "SIGINT", "Terminated" => "SIGTERM", "Killed" => "SIGKILL", "Hangup" => "SIGHUP", "Quit" => "SIGQUIT", "User defined signal 1" => "SIGUSR1", "User defined signal 2" => "SIGUSR2", "Aborted" => "SIGABRT", "Segmentation fault" => "SIGSEGV", "Floating point exception" => "SIGFPE", "Illegal instruction" => "SIGILL", "Broken pipe" => "SIGPIPE", "Alarm clock" => "SIGALRM", "Bus error" => "SIGBUS", _ => desc, } .to_owned() } impl Default for ProcessSupervisor { fn default() -> Self { Self::new() } } impl ProcessSupervisor { /// Construct an empty supervisor with sensible defaults /// (`grace_period` = 2s, `restart_backoff` = 250ms). #[must_use] pub fn new() -> Self { let (tx, rx) = channel::unbounded(); Self { processes: HashMap::new(), events_tx: tx, events_rx: rx, pending: HashMap::new(), grace_period: Duration::from_secs(2), restart_backoff: Duration::from_millis(250), shut_down: false, reap_ledger: HashMap::new(), group_term_grace: GROUP_TERM_GRACE, forced_kill_errno: None, } } /// Q#PD4 test seam: make the next `kill(2)` attempt in /// [`Self::signal`] report `errno` instead of calling the kernel. /// Consumed by that one attempt. Everything downstream — target /// selection, the leader observation against the real child, and /// the error construction — runs unmodified. #[cfg(test)] fn force_next_kill_errno(&mut self, errno: nix::errno::Errno) { self.forced_kill_errno = Some(errno); } /// Override the SIGTERM-to-SIGKILL grace window. Test helper. pub fn set_grace_period(&mut self, d: Duration) { self.grace_period = d; } /// Override the group TERM→KILL escalation window. Test helper. pub fn set_group_term_grace(&mut self, d: Duration) { self.group_term_grace = d; } /// Override the restart back-off. Test helper. pub fn set_restart_backoff(&mut self, d: Duration) { self.restart_backoff = d; } /// Spawn a new managed process. Returns a stable id; consult /// [`Self::state`] / [`Self::take_events`] to follow its /// lifecycle. /// /// Errors only on synchronous spawn failure (the kernel rejects /// the exec, the binary is unreadable, etc.). A child that /// crashes *after* spawn shows up as a [`Termination::Crashed`] /// in the event stream, not as a return error. pub fn spawn(&mut self, spec: ProcessSpec) -> Result { self.spawn_inner(spec, true) } /// Spawn an unpublished terminal-owned process. /// /// Unlike the public Lua/process path, synchronous failure does not emit an /// event for an ID no caller can own. `TerminalManager` rolls back its /// temporary identity buffer and returns the error directly. pub(crate) fn spawn_terminal(&mut self, spec: ProcessSpec) -> Result { self.spawn_inner(spec, false) } fn spawn_inner( &mut self, spec: ProcessSpec, publish_synchronous_failure: bool, ) -> Result { if self.shut_down { return Err("supervisor is shut down".to_owned()); } let id = ProcessId::next(); let mut managed = ManagedProcess { spec, state: ProcessState::Starting, runtime: None, attempt_count: 0, next_restart_at: None, }; self.start_generation(id, &mut managed, publish_synchronous_failure)?; self.processes.insert(id, managed); Ok(id) } /// Start a fresh generation for `managed`. Mutates `managed` in place; on /// failure its state is `Terminated(Crashed{...})`, and the event is emitted /// only when `publish_failure` is true. fn start_generation( &self, id: ProcessId, managed: &mut ManagedProcess, publish_failure: bool, ) -> Result<(), String> { managed.attempt_count += 1; managed.next_restart_at = None; match build_runtime(&managed.spec, id) { Ok(runtime) => { let pid = runtime.pid; let now = Instant::now(); managed.state = ProcessState::Running { pid, started: now }; managed.runtime = Some(runtime); let _ = self.events_tx.send(ProcessEvent { id, kind: ProcessEventKind::Started { pid }, at: now, }); Ok(()) } Err(e) => { let now = Instant::now(); managed.state = ProcessState::Terminated(Termination::Crashed { error: e.clone(), ended: now, }); managed.runtime = None; if publish_failure { let _ = self.events_tx.send(ProcessEvent { id, kind: ProcessEventKind::Crashed { error: e.clone() }, at: now, }); } Err(e) } } } /// Send `signal` to `id`. Errors if the id is unknown or the /// process is not currently running. Pipe-mode children are /// signaled by OS pid; PTY-mode children are signaled via the /// foreground process group when the kernel reports one, matching /// terminal C-c behavior for shells and REPLs. Nothing about the /// supervisor's state changes synchronously --- the lifecycle /// transition happens when the supervisor next observes the /// child's exit through `tick`. pub fn signal(&mut self, id: ProcessId, signal: Signal) -> Result<(), String> { let proc = self .processes .get_mut(&id) .ok_or_else(|| format!("unknown process: {id}"))?; let (ProcessState::Running { pid, .. } | ProcessState::Exiting { pid, .. }) = proc.state else { return Err(format!("process {id} is not running")); }; let target = signal_target(proc, pid)?; // Q#PD4: the seam injects the KILL attempt's result only — // never the observation below — so target selection, the real // `ChildHandle::try_wait` against the real child, and the error // construction all run for real. Consumed once. let kill_result = match self.forced_kill_errno.take() { Some(errno) => Err(errno), None => nix::sys::signal::kill(target.pid, Some(signal)), }; if let Err(errno) = kill_result { // Q#PD1/Q#PD2: the failure describes itself; the // disposition is unchanged — this still returns `Err`, // with no state transition and no ledger arming. let leader = observe_leader(proc); return Err(signal_failure_report(target, pid, errno, &leader)); } if matches!(signal, Signal::SIGTERM | Signal::SIGKILL | Signal::SIGHUP) { proc.state = ProcessState::Exiting { pid, signaled_at: Instant::now(), }; // Arm the group reap ledger on the first fatal signal // (Q#CM3). Insert-if-absent: a repeated `terminate` must // not push the SIGKILL bound out. if proc.spec.group && let Ok(pgid) = i32::try_from(pid) { let deadline = Instant::now() + self.group_term_grace; self.reap_ledger.entry(pgid).or_insert(GroupReap { deadline, killed: false, }); } } Ok(()) } /// Convenience wrapper around `signal(id, SIGTERM)`. On a v0.1 /// SIGTERM-tolerant child this is the polite shutdown path; the /// supervisor's `shutdown` enforces the SIGKILL fallback if the /// child doesn't exit within the grace window. pub fn terminate(&mut self, id: ProcessId) -> Result<(), String> { self.signal(id, Signal::SIGTERM) } /// Close `id`'s stdin pipe by dropping the writer. The writer /// thread drains any queued bytes first, then drops the handle, /// so the child observes EOF *after* everything already written /// — the canonical stdio-graceful-shutdown signal for protocols /// (notably MCP) that have no protocol-level shutdown message. /// Idempotent: a second call after the writer is gone is a /// no-op. Errors only if the process id is unknown. /// /// Note: this does NOT kill the process. Callers that want a /// guaranteed exit follow up with [`Self::terminate`] (SIGTERM) /// after a grace window, and the supervisor's /// [`Self::shutdown`] applies the SIGKILL fallback at editor /// drop time. pub fn close_stdin(&mut self, id: ProcessId) -> Result<(), String> { let proc = self .processes .get_mut(&id) .ok_or_else(|| format!("unknown process: {id}"))?; if let Some(runtime) = proc.runtime.as_mut() { // Dropping the writer closes the pipe at the kernel // level. `take()` is idempotent — second call sees None. let _ = runtime.stdin.take(); } Ok(()) } /// Write `bytes` to `id`'s stdin. Errors if the id is unknown, /// the process is not running, stdin is closed (the child /// closed stdin on its end, or stdin was never piped in the /// first place), or the per-generation queue budget is /// exhausted. The write itself is queued to a dedicated writer /// thread, so this never blocks on pipe backpressure — a write /// *failure* (broken pipe) therefore surfaces on a subsequent /// call rather than the one that queued the bytes; callers that /// need liveness should watch the supervisor's exit events. pub fn write_stdin(&mut self, id: ProcessId, bytes: &[u8]) -> Result<(), String> { let proc = self .processes .get_mut(&id) .ok_or_else(|| format!("unknown process: {id}"))?; let runtime = proc .runtime .as_mut() .ok_or_else(|| format!("process {id} has no live generation"))?; let stdin = runtime .stdin .as_ref() .ok_or_else(|| format!("process {id} stdin is not piped"))?; stdin.write(bytes) } /// Resize the PTY for `id`. Errors if the id is unknown, the /// process isn't running, or the process is in pipe mode. pub fn resize_pty(&mut self, id: ProcessId, rows: u16, cols: u16) -> Result<(), String> { let proc = self .processes .get_mut(&id) .ok_or_else(|| format!("unknown process: {id}"))?; let runtime = proc .runtime .as_mut() .ok_or_else(|| format!("process {id} has no live generation"))?; match &runtime.child { ChildHandle::Pty { _master: master, .. } => master .resize(portable_pty::PtySize { rows, cols, pixel_width: 0, pixel_height: 0, }) .map_err(|e| format!("resize_pty: {e}")), ChildHandle::Pipes(_) => Err(format!("process {id} is not a PTY process")), } } /// Drain pending events into per-process buffers, poll each /// running child for exit, and apply restart policy. Call once /// per editor frame. pub fn tick(&mut self) { // Drain pending lifecycle events from the supervisor-wide // unbounded channel (Started/Exited/Signaled/Crashed/Restarting // — small, infrequent, not subject to byte-stream backpressure). while let Ok(ev) = self.events_rx.try_recv() { self.pending.entry(ev.id).or_default().push(ev); } // Drain per-generation byte channels and coalesce into one // event per (process, kind) per tick. T M6.2 / spec // §sec:repl-streaming + M3.5 coalescing model: many small // chunks land as O(ticks) events, not O(chunks) events. let ids: Vec = self.processes.keys().copied().collect(); for id in &ids { self.drain_byte_channel(*id); } // Poll each managed process; ids re-iterated to avoid // mutating the map while iterating. for id in ids { self.poll_one(id); self.maybe_restart(id); } // Probe the group reap ledger last so groups TERMed by this // tick's poll_one get their liveness checked from the very // next tick onward (Q#CM3). self.tick_reap_ledger(); } /// Probe every armed group: ESRCH → group gone, drop the entry; /// alive past its deadline → SIGKILL the group (once), then keep /// probing to ESRCH. Independent of managed-process records by /// design — this is what catches a TERM-ignoring descendant that /// survived its leader's clean exit with its output redirected /// (round-3 finding 1: neither leader state nor reader state can /// see that survivor; only group liveness can). fn tick_reap_ledger(&mut self) { let now = Instant::now(); self.reap_ledger.retain(|pgid, entry| { // ESRCH: no such group — done. Any other probe error is // also treated as "nothing left we can reach" (EPERM // cannot happen for our own children) so the ledger // cannot grow without bound. if nix::sys::signal::kill(Pid::from_raw(-*pgid), None).is_err() { return false; } if now >= entry.deadline && !entry.killed { let _ = nix::sys::signal::kill(Pid::from_raw(-*pgid), Some(Signal::SIGKILL)); entry.killed = true; } true }); } /// Drain the per-generation byte channel for `id` and emit at /// most one `Stdout` and one `Stderr` event into pending. Called /// from `tick()`. No-op if the process has no live runtime. fn drain_byte_channel(&mut self, id: ProcessId) { let drained = { let Some(proc) = self.processes.get(&id) else { return; }; let Some(rt) = proc.runtime.as_ref() else { return; }; match &rt.output_rx { RuntimeOutputRx::Bytes(byte_rx) => drain_raw_output(byte_rx), RuntimeOutputRx::Ansi(ansi_rx) => drain_ansi_output(ansi_rx), } }; if drained.is_empty() { return; } let now = Instant::now(); let queue = self.pending.entry(id).or_default(); for kind in drained { queue.push(ProcessEvent { id, kind, at: now }); } } /// Poll one process for exit. Transitions Running/Exiting → /// Terminated and emits the appropriate event. fn poll_one(&mut self, id: ProcessId) { let Some(proc) = self.processes.get_mut(&id) else { return; }; let started = match proc.state { ProcessState::Running { started, .. } => started, ProcessState::Exiting { signaled_at, .. } => signaled_at, _ => return, }; let Some(runtime) = proc.runtime.as_mut() else { return; }; let status = runtime.child.try_wait(); if matches!(status, Ok(None)) { return; } // Terminal from here on. Group leader-exit reap (Q#CM3): // TERM the remaining group and arm the reap ledger BEFORE // the final drain — a leader that exits leaving `sleep 60 &` // holding the merged pipe would otherwise burn the full // drain timeout and then block the reader join. Arming is // insert-if-absent, so a deadline already armed by an // explicit kill is not extended. let group_ctx = if proc.spec.group { i32::try_from(runtime.pid).ok().map(|pgid| { let _ = nix::sys::signal::kill(Pid::from_raw(-pgid), Some(Signal::SIGTERM)); let deadline = Instant::now() + self.group_term_grace; let entry = self.reap_ledger.entry(pgid).or_insert(GroupReap { deadline, killed: false, }); GroupDrainCtx { pgid, deadline: entry.deadline, } }) } else { None }; let now = Instant::now(); let final_output = final_drain_runtime(runtime, group_ctx); let (termination, event) = match status { Ok(Some(TermStatus::Exited(code))) => ( Termination::Exited { code, started, ended: now, }, ProcessEventKind::Exited { code }, ), Ok(Some(TermStatus::Signaled(signal))) => ( Termination::Signaled { signal: signal.clone(), started, ended: now, }, ProcessEventKind::Signaled { signal }, ), Err(e) => ( Termination::Crashed { error: e.clone(), ended: now, }, ProcessEventKind::Crashed { error: e }, ), // Guarded above; kept explicit so the match stays total. Ok(None) => return, }; proc.state = ProcessState::Terminated(termination); proc.runtime = None; append_process_events(&mut self.pending, id, final_output, now); self.pending.entry(id).or_default().push(ProcessEvent { id, kind: event, at: now, }); } /// Apply restart policy after `poll_one` may have transitioned /// the process to `Terminated`. fn maybe_restart(&mut self, id: ProcessId) { // Inert during and after shutdown: shutdown's own tick() // calls must not respawn a `restart = always` process // mid-teardown (round-4 finding 1). if self.shut_down { return; } let now = Instant::now(); let restart_now = { let Some(proc) = self.processes.get(&id) else { return; }; let ProcessState::Terminated(termination) = &proc.state else { return; }; let policy = proc.spec.restart; let should = match (policy, termination) { (RestartPolicy::Never, _) => false, (RestartPolicy::Always, _) | ( RestartPolicy::OnCrash, Termination::Signaled { .. } | Termination::Crashed { .. }, ) => true, (RestartPolicy::OnCrash, Termination::Exited { code, .. }) => *code != 0, }; if !should { return; } // Schedule the restart after `restart_backoff` from the // termination time; we don't synchronously block. match proc.next_restart_at { Some(at) => at <= now, None => false, } }; if restart_now { // Borrow mutably for the actual restart. let mut managed = self.processes.remove(&id).expect("checked existence above"); let attempt = managed.attempt_count + 1; self.pending.entry(id).or_default().push(ProcessEvent { id, kind: ProcessEventKind::Restarting { attempt }, at: now, }); let _ = self.start_generation(id, &mut managed, true); self.processes.insert(id, managed); } else { // Schedule a restart attempt for `restart_backoff` from // now if not yet scheduled. if let Some(proc) = self.processes.get_mut(&id) && matches!(proc.state, ProcessState::Terminated(_)) && !matches!(proc.spec.restart, RestartPolicy::Never) && proc.next_restart_at.is_none() { proc.next_restart_at = Some(now + self.restart_backoff); } } } /// Drain and return all events queued for `id` since the last /// call. Returns an empty vec for unknown ids and for known ids /// that haven't produced events yet. pub fn take_events(&mut self, id: ProcessId) -> Vec { self.pending.remove(&id).unwrap_or_default() } /// Drain every queued event across every process. Returns events /// in the order they were enqueued. Useful for `*processes*` /// log-style buffers and tests. pub fn take_all_events(&mut self) -> Vec { let mut all = Vec::new(); for (_id, mut evs) in std::mem::take(&mut self.pending) { all.append(&mut evs); } all.sort_by_key(|e| e.at); all } /// Current state of `id`, or `None` if the id is unknown. #[must_use] pub fn state(&self, id: ProcessId) -> Option<&ProcessState> { self.processes.get(&id).map(|p| &p.state) } /// Spec for `id`, or `None` if the id is unknown. #[must_use] pub fn spec(&self, id: ProcessId) -> Option<&ProcessSpec> { self.processes.get(&id).map(|p| &p.spec) } /// Iterator over every managed process id, in arbitrary order. pub fn ids(&self) -> impl Iterator + '_ { self.processes.keys().copied() } /// Forget about `id`. The process must already be terminated; /// otherwise this returns an error and leaves the process /// alone. Use [`Self::terminate`] + tick + `forget` to /// permanently remove a running process. pub fn forget(&mut self, id: ProcessId) -> Result<(), String> { let proc = self .processes .get(&id) .ok_or_else(|| format!("unknown process: {id}"))?; if !matches!(proc.state, ProcessState::Terminated(_)) { return Err(format!("process {id} is not terminated")); } self.processes.remove(&id); self.pending.remove(&id); Ok(()) } /// Send SIGTERM to every running process; wait up to the grace /// period for them to exit; SIGKILL anything still alive. /// Idempotent. Called automatically from `Drop`. pub fn shutdown(&mut self) { if self.shut_down { return; } self.shut_down = true; // SIGTERM phase. let ids: Vec = self.processes.keys().copied().collect(); for id in &ids { let _ = self.signal(*id, Signal::SIGTERM); } // Poll-loop with timeout. tick() is slightly heavier than // we need (it does restart accounting), but it's the // canonical exit-observation path. let deadline = Instant::now() + self.grace_period; while Instant::now() < deadline && self.any_running() { self.tick(); std::thread::sleep(Duration::from_millis(20)); } // SIGKILL anything left. for id in &ids { if let Some(proc) = self.processes.get(id) && matches!( proc.state, ProcessState::Running { .. } | ProcessState::Exiting { .. } ) { let _ = self.signal(*id, Signal::SIGKILL); } } // Editor exit owes group survivors no grace: force-kill every // outstanding reap-ledger entry now, then probe it to ESRCH in // the bounded loop below. Without this, a pre-deadline ledger // (leader exited promptly, TERM-ignoring group member alive) // would be silently discarded at Drop and leak the member // (Q#CM3, round-4 finding 1). for (pgid, entry) in &mut self.reap_ledger { let _ = nix::sys::signal::kill(Pid::from_raw(-*pgid), Some(Signal::SIGKILL)); entry.killed = true; } // Final reap loop. SIGKILL is delivered immediately by the // kernel; the child becomes a zombie until we reap. Bound // the wait so a pathological case can't hang the editor // exit forever. The tick also probes the reap ledger, so the // loop holds until force-killed groups observe ESRCH. let final_deadline = Instant::now() + Duration::from_secs(2); while Instant::now() < final_deadline && (self.any_running() || !self.reap_ledger.is_empty()) { self.tick(); std::thread::sleep(Duration::from_millis(20)); } } fn any_running(&self) -> bool { self.processes.values().any(|p| { matches!( p.state, ProcessState::Running { .. } | ProcessState::Starting | ProcessState::Exiting { .. } ) }) } /// Clone of a live generation's active-reader counter (see /// [`RuntimeHandles::active_readers`]). Unit tests grab it while /// the generation runs and assert zero after teardown. #[cfg(test)] fn active_reader_probe(&self, id: ProcessId) -> Option> { Some(Arc::clone( &self.processes.get(&id)?.runtime.as_ref()?.active_readers, )) } /// Number of armed reap-ledger entries. Test observability for /// the shutdown/drop-twin pins. #[cfg(test)] fn reap_ledger_len(&self) -> usize { self.reap_ledger.len() } } impl Drop for ProcessSupervisor { fn drop(&mut self) { self.shutdown(); } } // --------------------------------------------------------------------------- // Spawn machinery // --------------------------------------------------------------------------- /// Build a fresh runtime (handles + reader threads) per /// `spec`. Pulled out of [`ProcessSupervisor::start_generation`] so /// the supervisor itself is small and the pipe-vs-pty branching /// lives in one place. /// /// Lifecycle events (Started/Exited/Signaled/Crashed/Restarting) are /// emitted by the supervisor itself onto its unbounded /// `events_tx`; reader threads emit only byte chunks onto the /// per-generation bounded byte channel. T M6.2. fn build_runtime(spec: &ProcessSpec, id: ProcessId) -> Result { if spec.ansi_events && matches!(spec.mode, ProcessMode::Pipes) { return Err("process spawn: ansi=true requires pty mode; pipe-mode consumers receive raw stdout/stderr bytes".to_owned()); } if matches!(spec.mode, ProcessMode::Pty { .. }) { if matches!(spec.stdin, StdinMode::Null) { return Err( "process spawn: stdin=\"null\" requires pipe mode; a PTY has no separable stdin" .to_owned(), ); } if spec.group { return Err("process spawn: group=true requires pipe mode; PTY children already lead their own session and are signaled group-wide".to_owned()); } } match spec.mode { ProcessMode::Pipes => build_pipes_runtime(spec, id), ProcessMode::Pty { rows, cols, mode } => build_pty_runtime(spec, id, rows, cols, mode), } } fn build_pipes_runtime(spec: &ProcessSpec, _id: ProcessId) -> Result { use std::process::{Command, Stdio}; let mut cmd = Command::new(&spec.command); cmd.args(&spec.args) .stdin(match spec.stdin { StdinMode::Piped => Stdio::piped(), // Immediate EOF, no writer thread, zero close-after-spawn // race (Q#CM3). StdinMode::Null => Stdio::null(), }) .stdout(Stdio::piped()) .stderr(Stdio::piped()); if spec.group { // Fresh process group with the child as leader (pgid == pid). // Safe std API — no `unsafe`, no trampoline (stable 1.64). use std::os::unix::process::CommandExt; cmd.process_group(0); } if let Some(ref cwd) = spec.cwd { cmd.current_dir(cwd); } for (k, v) in &spec.env { cmd.env(k, v); } let mut child = cmd.spawn().map_err(|e| format!("spawn: {e}"))?; let pid = child.id(); let stdin = child .stdin .take() .map(|s| StdinWriter::spawn(Box::new(s) as Box)); let stdout = child.stdout.take(); let stderr = child.stderr.take(); let (byte_tx, byte_rx) = channel::bounded::(BYTE_CHUNK_CHANNEL_CAP); let cancel = Arc::new(AtomicBool::new(false)); let active_readers = Arc::new(AtomicUsize::new(0)); let mut readers = Vec::new(); if let Some(out) = stdout { readers.push(if spec.group { spawn_group_reader( byte_tx.clone(), Arc::clone(&cancel), out, ReaderKind::Stdout, Arc::clone(&active_readers), ) } else { spawn_reader( byte_tx.clone(), Arc::clone(&cancel), out, ReaderKind::Stdout, ) }); } if let Some(err) = stderr { readers.push(if spec.group { spawn_group_reader( byte_tx, Arc::clone(&cancel), err, ReaderKind::Stderr, Arc::clone(&active_readers), ) } else { spawn_reader(byte_tx, Arc::clone(&cancel), err, ReaderKind::Stderr) }); } Ok(RuntimeHandles { child: ChildHandle::Pipes(child), stdin, pid, readers, output_rx: RuntimeOutputRx::Bytes(byte_rx), cancel, active_readers, }) } fn build_pty_runtime( spec: &ProcessSpec, _id: ProcessId, rows: u16, cols: u16, mode: TerminalMode, ) -> Result { use portable_pty::PtySize; let pty_system = portable_pty::native_pty_system(); let pair = pty_system .openpty(PtySize { rows, cols, pixel_width: 0, pixel_height: 0, }) .map_err(|e| format!("openpty: {e}"))?; // Line discipline per spec §sec:repl-supervisor is applied by // wrapping the command in a /bin/sh trampoline that calls // `stty` before `exec`-ing the target binary. Canonical mode is // the kernel default for a freshly-allocated PTY on Linux/macOS, // so it requires no trampoline. if matches!(mode, TerminalMode::Raw) && !std::path::Path::new("/bin/sh").is_file() { return Err("pty spawn: raw terminal mode requires /bin/sh \ (the supervisor uses a /bin/sh trampoline to apply line \ discipline per spec §sec:repl-supervisor); install a \ sh-compatible shell at /bin/sh, or configure this \ process with TerminalMode::Canonical" .to_owned()); } let mut cmd = build_pty_command(spec, mode); if let Some(ref cwd) = spec.cwd { cmd.cwd(cwd); } for (k, v) in &spec.env { cmd.env(k, v); } let child = pair .slave .spawn_command(cmd) .map_err(|e| format!("pty spawn: {e}"))?; let pid = child .process_id() .ok_or_else(|| "pty pid missing".to_owned())?; // Drop the slave: `child` keeps it alive on its end. Holding it // ourselves is unnecessary and prevents EOF detection on the // master once the child exits. drop(pair.slave); let writer = pair .master .take_writer() .map_err(|e| format!("pty writer: {e}"))?; let reader = pair .master .try_clone_reader() .map_err(|e| format!("pty reader: {e}"))?; let (byte_tx, byte_rx) = channel::bounded::(BYTE_CHUNK_CHANNEL_CAP); let cancel = Arc::new(AtomicBool::new(false)); let mut readers = vec![spawn_reader( byte_tx, Arc::clone(&cancel), reader, ReaderKind::Stdout, )]; let output_rx = if spec.ansi_events { let (ansi_tx, ansi_rx) = channel::bounded::(ANSI_EVENT_CHANNEL_CAP); readers.push(spawn_ansi_parser( byte_rx, ansi_tx, Arc::clone(&cancel), spec.ansi_profile, )); RuntimeOutputRx::Ansi(ansi_rx) } else { RuntimeOutputRx::Bytes(byte_rx) }; Ok(RuntimeHandles { child: ChildHandle::Pty { child: Arc::new(Mutex::new(into_send_sync_child(child))), _master: pair.master, }, stdin: Some(StdinWriter::spawn(writer)), pid, readers, output_rx, cancel, // PTY readers are the blocking kind; the counter is only // maintained by group readers and stays zero here. active_readers: Arc::new(AtomicUsize::new(0)), }) } /// portable-pty's `Child` is `Send` but not necessarily `Sync`. /// Wrapping in a Mutex makes the supervisor's `try_wait` callable /// from the main thread without `unsafe`. The `Sync` bound on the /// supervisor's [`ChildHandle::Pty::child`] field is satisfied via /// `Arc>`. fn into_send_sync_child( child: Box, ) -> Box { child } /// Build the [`portable_pty::CommandBuilder`] for a PTY-mode child, /// applying the requested line discipline. /// /// # Why a `/bin/sh` trampoline (not a direct `tcsetattr`) /// /// portable-pty 0.9 exposes no `set_termios` and no pre-exec hook. /// `nix::sys::termios::tcsetattr` requires `AsFd`, and converting /// `MasterPty::as_raw_fd` (a `RawFd`) to `AsFd` requires /// `BorrowedFd::borrow_raw`, which is `unsafe`. pmacs's crate-level /// `unsafe_code = "forbid"` rule is a project-identity property /// (see `MEMORY.md` / `feedback_unsafe_code_posture.md`), not a /// negotiable lint, so we trampoline through `/bin/sh` instead: /// /// ```sh /// /bin/sh -c 'stty raw -echo /dev/null; exec "$@"' -- CMD ARGS... /// ``` /// /// # Why this is shell-injection-safe /// /// The argv-as-positional-parameters mechanism is the standard /// pattern (the same one `xargs -0` relies on). When you invoke /// `sh -c 'SCRIPT' -- ARG1 ARG2 ARG3`, the shell receives: /// /// - `SCRIPT` as the literal source code to execute /// - `--` as `$0` (the script name) /// - `ARG1`, `ARG2`, `ARG3` as the positional parameters `$1`, /// `$2`, `$3` /// /// Critically, the positional parameters are **literal data from /// the moment they enter `sh`'s argv**; the shell never re-parses /// them. `"$@"` then expands to `"$1" "$2" "$3"` with each parameter /// as a separate word, regardless of whether they contain spaces, /// quotes, semicolons, or any other shell metacharacters. There is /// no path through which user-controlled `spec.command` or /// `spec.args` can become shell tokens; they remain argv all the /// way through to `exec`. /// /// # Why the redirections /// /// `/dev/null`: silences `stty` errors. If the slave isn't a tty /// `stty` recognizes, we proceed to `exec` regardless and the child /// runs in the kernel default (canonical) instead — graceful /// degradation rather than a confusing failure mode. /// /// # Canonical mode /// /// `TerminalMode::Canonical` skips the trampoline entirely. A /// freshly-allocated PTY's kernel default on Linux/macOS is /// canonical + echo + isig, which is exactly the canonical-mode /// contract from spec §sec:repl-supervisor. Adding a no-op `stty` /// invocation would be churn. fn build_pty_command(spec: &ProcessSpec, mode: TerminalMode) -> portable_pty::CommandBuilder { use portable_pty::CommandBuilder; match mode { TerminalMode::Raw => { let mut cmd = CommandBuilder::new("/bin/sh"); cmd.arg("-c"); cmd.arg("stty raw -echo /dev/null; exec \"$@\""); cmd.arg("--"); cmd.arg(&spec.command); for arg in &spec.args { cmd.arg(arg); } cmd } TerminalMode::Canonical => { let mut cmd = CommandBuilder::new(&spec.command); for arg in &spec.args { cmd.arg(arg); } cmd } } } #[derive(Clone, Copy)] enum ReaderKind { Stdout, Stderr, } /// Spawn a reader thread that pulls [`BYTE_CHUNK_SIZE`] chunks off /// `read` and pushes them onto the per-generation bounded byte /// channel. T M6.2 / spec §sec:repl-streaming. /// /// Backpressure: when `byte_tx` is full (consumer fell behind), the /// reader's `send` blocks. The kernel pipe then fills, the child's /// `write` syscall blocks, and the producer rate is rate-limited to /// the consumer's drain rate — exactly the spec's stalling chain. /// /// Cancellation: blocked sends are pre-empted by `cancel`. Without /// this, a reader stuck in `send` because the consumer fell behind /// permanently would leak until OS-level pipe teardown reaches it /// (which only happens once the producer is reaped). The cancel /// flag is what makes "cancellation propagates to source" prompt. /// /// Exits on: EOF (`Ok(0)`), closed channel (consumer dropped), /// cancel flag set, or read error. fn spawn_reader( byte_tx: Sender, cancel: Arc, mut read: R, kind: ReaderKind, ) -> JoinHandle<()> { std::thread::spawn(move || { let mut buf = [0u8; BYTE_CHUNK_SIZE]; loop { if cancel.load(Ordering::Relaxed) { return; } match read.read(&mut buf) { Ok(0) => return, Ok(n) => { let mut payload: ByteChunk = (kind, buf[..n].to_vec()); loop { match byte_tx.send_timeout(payload, READER_SEND_POLL_INTERVAL) { Ok(()) => break, Err(crossbeam::channel::SendTimeoutError::Timeout(rejected)) => { if cancel.load(Ordering::Relaxed) { return; } payload = rejected; } Err(crossbeam::channel::SendTimeoutError::Disconnected(_)) => { return; } } } } Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} Err(_) => return, } } }) } /// RAII live-count for group reader threads: increments on /// construction, decrements on every exit path (panic included), so /// [`RuntimeHandles::active_readers`] reaching zero is a /// deterministic "thread ended, its read FD dropped" signal. struct ActiveReaderGuard(Arc); impl ActiveReaderGuard { fn new(counter: Arc) -> Self { counter.fetch_add(1, Ordering::Relaxed); Self(counter) } } impl Drop for ActiveReaderGuard { fn drop(&mut self) { self.0.fetch_sub(1, Ordering::Relaxed); } } /// Poll-based cancellable reader for `group = true` generations /// (Q#CM3). Unlike [`spawn_reader`], the fd is set nonblocking and /// every wait — for readability or for channel space — re-checks /// `cancel` each [`READER_SEND_POLL_INTERVAL`], with an extra check /// between poll and read/send, so `RuntimeHandles::Drop`'s retained /// join completes within one interval regardless of who still holds /// the pipe's write end (a setsid'd descendant, notably). Non-group /// consumers (REPL, LSP) keep the blocking [`spawn_reader`] they /// were tuned on — the M6.6 ingest gate; unifying is a named /// deferral in the compile-mode framing. fn spawn_group_reader( byte_tx: Sender, cancel: Arc, read: R, kind: ReaderKind, active: Arc, ) -> JoinHandle<()> where R: Read + std::os::fd::AsFd + Send + 'static, { std::thread::spawn(move || { let _guard = ActiveReaderGuard::new(active); let mut read = read; // nix 0.29's fcntl still takes a RawFd (poll takes BorrowedFd). let raw_fd = std::os::fd::AsRawFd::as_raw_fd(&read.as_fd()); if nix::fcntl::fcntl( raw_fd, nix::fcntl::FcntlArg::F_SETFL(nix::fcntl::OFlag::O_NONBLOCK), ) .is_err() { // Cannot go nonblocking (does not happen for pipe fds in // practice): exit rather than risk an uncancellable // blocking read. return; } let poll_timeout = nix::poll::PollTimeout::try_from(READER_SEND_POLL_INTERVAL) .unwrap_or(nix::poll::PollTimeout::MAX); let mut buf = [0u8; BYTE_CHUNK_SIZE]; loop { if cancel.load(Ordering::Relaxed) { return; } let ready = { let mut fds = [nix::poll::PollFd::new( read.as_fd(), nix::poll::PollFlags::POLLIN, )]; nix::poll::poll(&mut fds, poll_timeout) }; match ready { // Timeout or interrupt: loop around and re-check the // cancel flag. Ok(0) | Err(nix::errno::Errno::EINTR) => continue, Ok(_) => {} Err(_) => return, } if cancel.load(Ordering::Relaxed) { return; } match read.read(&mut buf) { Ok(0) => return, Ok(n) => { let mut payload: ByteChunk = (kind, buf[..n].to_vec()); loop { match byte_tx.send_timeout(payload, READER_SEND_POLL_INTERVAL) { Ok(()) => break, Err(crossbeam::channel::SendTimeoutError::Timeout(rejected)) => { if cancel.load(Ordering::Relaxed) { return; } payload = rejected; } Err(crossbeam::channel::SendTimeoutError::Disconnected(_)) => { return; } } } } Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {} Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} Err(_) => return, } } }) } fn drain_raw_output(byte_rx: &Receiver) -> Vec { let mut stdout_buf: Vec = Vec::new(); let mut stderr_buf: Vec = Vec::new(); while let Ok((kind, mut bytes)) = byte_rx.try_recv() { match kind { ReaderKind::Stdout => stdout_buf.append(&mut bytes), ReaderKind::Stderr => stderr_buf.append(&mut bytes), } } let mut out = Vec::with_capacity(2); if !stdout_buf.is_empty() { out.push(ProcessEventKind::Stdout(stdout_buf)); } if !stderr_buf.is_empty() { out.push(ProcessEventKind::Stderr(stderr_buf)); } out } fn drain_ansi_output(ansi_rx: &Receiver) -> Vec { let mut events: Vec = Vec::new(); while let Ok(mut batch) = ansi_rx.try_recv() { events.append(&mut batch); } if events.is_empty() { Vec::new() } else { vec![ProcessEventKind::Ansi(events)] } } fn drain_runtime_output(rt: &RuntimeHandles) -> Vec { match &rt.output_rx { RuntimeOutputRx::Bytes(byte_rx) => drain_raw_output(byte_rx), RuntimeOutputRx::Ansi(ansi_rx) => drain_ansi_output(ansi_rx), } } /// Context for a group-aware final drain (Q#CM3). Carries the reap /// ledger's deadline for this group: the drain enforces it from /// inside its loop because no other tick runs while the drain /// blocks the frame. #[derive(Clone, Copy)] struct GroupDrainCtx { pgid: i32, deadline: Instant, } fn final_drain_runtime(rt: &RuntimeHandles, group: Option) -> Vec { let deadline = Instant::now() + EXIT_OUTPUT_DRAIN_TIMEOUT; let mut out = Vec::new(); // Group drains get tighter bounds than the plain byte-flush // timeout (Q#CM3, round-4 finding 2 / round-5 revision): // - the ledger deadline is enforced in-loop — SIGKILL the group // at the grace bound; // - once the group probes ESRCH, readers get one quiescent // READER_SEND_POLL_INTERVAL to flush already-read and // kernel-buffered bytes; new data resets the window; // - independently, no group drain may pass the absolute cancel // deadline of ledger deadline + one poll interval — reaching // it cancels the readers even when an escaped (setsid'd) // writer still holds the pipe past its group's death. Honest // trailing output gets a bounded flush; escaped output may be // truncated. The retained join in RuntimeHandles::Drop then // completes within one further poll interval because group // readers are poll-based and observe the cancel flag. let mut group_killed = false; let mut last_data = Instant::now(); loop { let drained = drain_runtime_output(rt); let drained_any = !drained.is_empty(); out.extend(drained); if drained_any { last_data = Instant::now(); } if rt.readers.iter().all(std::thread::JoinHandle::is_finished) && !drained_any { return out; } if let Some(ctx) = &group { let now = Instant::now(); let group_alive = nix::sys::signal::kill(Pid::from_raw(-ctx.pgid), None).is_ok(); if group_alive && now >= ctx.deadline && !group_killed { let _ = nix::sys::signal::kill(Pid::from_raw(-ctx.pgid), Some(Signal::SIGKILL)); group_killed = true; } let quiesced = !group_alive && now.duration_since(last_data) >= READER_SEND_POLL_INTERVAL; if quiesced || now >= ctx.deadline + READER_SEND_POLL_INTERVAL { rt.cancel.store(true, Ordering::Relaxed); out.extend(drain_runtime_output(rt)); return out; } } if Instant::now() >= deadline { return out; } std::thread::sleep(Duration::from_millis(1)); } } fn append_process_events( pending: &mut HashMap>, id: ProcessId, kinds: Vec, at: Instant, ) { if kinds.is_empty() { return; } let queue = pending.entry(id).or_default(); for kind in kinds { queue.push(ProcessEvent { id, kind, at }); } } /// Spawn the ANSI parser worker for an ANSI-enabled PTY generation. /// /// The reader thread remains responsible for the 1 MiB PTY-read ceiling. /// This stage consumes those chunks, maintains parser state across chunk /// boundaries, and forwards structured events through a second bounded /// channel whose capacity represents the spec's 256 KiB parser→main /// ceiling. fn spawn_ansi_parser( byte_rx: Receiver, ansi_tx: Sender, cancel: Arc, profile: AnsiParserProfile, ) -> JoinHandle<()> { std::thread::spawn(move || { let mut parser = AnsiParser::with_profile(profile); loop { if cancel.load(Ordering::Relaxed) { return; } let (kind, bytes) = match byte_rx.recv_timeout(READER_SEND_POLL_INTERVAL) { Ok(chunk) => chunk, Err(crossbeam::channel::RecvTimeoutError::Timeout) => continue, Err(crossbeam::channel::RecvTimeoutError::Disconnected) => { let events = parser.finish(); if !events.is_empty() { let _ = send_ansi_batch(&ansi_tx, &cancel, events); } return; } }; if !matches!(kind, ReaderKind::Stdout) { continue; } let events = parser.feed(&bytes); if !events.is_empty() && !send_ansi_batch(&ansi_tx, &cancel, events) { return; } } }) } fn send_ansi_batch( ansi_tx: &Sender, cancel: &AtomicBool, mut events: AnsiBatch, ) -> bool { loop { match ansi_tx.send_timeout(events, READER_SEND_POLL_INTERVAL) { Ok(()) => return true, Err(crossbeam::channel::SendTimeoutError::Timeout(rejected)) => { if cancel.load(Ordering::Relaxed) { return false; } events = rejected; } Err(crossbeam::channel::SendTimeoutError::Disconnected(_)) => return false, } } } // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- #[cfg(test)] mod tests { use super::*; fn drain_until bool>( sup: &mut ProcessSupervisor, id: ProcessId, deadline: Duration, predicate: F, ) -> Vec { let stop = Instant::now() + deadline; let mut all = Vec::new(); while Instant::now() < stop { sup.tick(); let mut evs = sup.take_events(id); all.append(&mut evs); if predicate(&all) { return all; } std::thread::sleep(Duration::from_millis(10)); } all } fn has_exited(events: &[ProcessEvent]) -> bool { events.iter().any(|e| { matches!( e.kind, ProcessEventKind::Exited { .. } | ProcessEventKind::Signaled { .. } ) }) } #[test] fn pty_signal_names_are_canonical_across_libc_variants() { assert_eq!(canonicalize_pty_signal_name("Terminated"), "SIGTERM"); assert_eq!(canonicalize_pty_signal_name("Terminated: 15"), "SIGTERM"); assert_eq!(canonicalize_pty_signal_name("Killed: 9"), "SIGKILL"); assert_eq!( canonicalize_pty_signal_name("Unknown signal: 99"), "Unknown signal: 99" ); } #[test] fn terminal_transactional_spawn_failure_has_no_event_or_process_residue() { let mut supervisor = ProcessSupervisor::new(); let spec = ProcessSpec::new( "unpublished-terminal", "/definitely/not/a/real/pmacs-terminal-program", ); assert!(supervisor.spawn_terminal(spec).is_err()); supervisor.tick(); assert_eq!(supervisor.ids().count(), 0); assert!(supervisor.take_all_events().is_empty()); } #[test] fn spawn_pipes_lifecycle_started_then_exited() { let mut sup = ProcessSupervisor::new(); let mut spec = ProcessSpec::new("echo-test", "/bin/sh"); spec.args = vec!["-c".into(), "echo hello && exit 0".into()]; let id = sup.spawn(spec).expect("spawn"); let events = drain_until(&mut sup, id, Duration::from_secs(5), has_exited); assert!( events .iter() .any(|e| matches!(e.kind, ProcessEventKind::Started { .. })), "must observe Started" ); assert!( events .iter() .any(|e| matches!(&e.kind, ProcessEventKind::Stdout(b) if b.starts_with(b"hello"))), "must observe stdout 'hello'" ); assert!( events .iter() .any(|e| matches!(e.kind, ProcessEventKind::Exited { code: 0 })), "must observe Exited{{code:0}}" ); } /// Spawn a PTY child that leads its own session and stays alive /// until terminated, returning its id and OS pid. /// /// `/bin/sleep` directly rather than through a shell: a shell may /// place the command in a different foreground process group, and /// these tests assert the exact target the tty reports. fn spawn_live_pty(sup: &mut ProcessSupervisor, name: &str) -> (ProcessId, u32) { let mut spec = ProcessSpec::new(name, "/bin/sleep"); spec.args = vec!["30".into()]; spec.mode = ProcessMode::Pty { rows: 24, cols: 80, mode: TerminalMode::Canonical, }; let id = sup.spawn(spec).expect("spawn"); (id, spawn_started_pid(sup, id)) } /// The OS pid straight from the supervisor's own record, WITHOUT /// ticking. /// /// `drain_until` ticks, and a tick can observe a fast child's exit /// and transition the record out of `Running` — after which /// `signal` returns "is not running" and never reaches the /// diagnostic at all. Any test whose child exits promptly must read /// the pid this way. (Found by the parallel workspace sweep: the /// drain-based helper raced only under load.) fn record_pid(sup: &ProcessSupervisor, id: ProcessId) -> u32 { match sup.processes.get(&id).expect("record").state { ProcessState::Running { pid, .. } | ProcessState::Exiting { pid, .. } => pid, ProcessState::Starting => panic!("spawn has not reported a pid yet"), ProcessState::Terminated(_) => { panic!("the record already left Running; the pid is unavailable") } } } /// Drain until `Started` and return the OS pid it carries. Safe /// only for children that outlive the drain; see [`record_pid`]. fn spawn_started_pid(sup: &mut ProcessSupervisor, id: ProcessId) -> u32 { let evs = drain_until(sup, id, Duration::from_secs(5), |evs| { evs.iter() .any(|e| matches!(e.kind, ProcessEventKind::Started { .. })) }); evs.iter() .find_map(|e| match e.kind { ProcessEventKind::Started { pid } => Some(pid), _ => None, }) .expect("Started carries a pid") } /// Drive the production diagnostic until it observes the leader as /// exited, bounded by `timeout`. /// /// A fixed sleep is NOT proof of exit — on a loaded runner the child /// can still be live, which would turn these tests into false /// failures. This synchronises on the very observation under test. /// Each failing attempt leaves the record untouched, because the /// failure path returns before any bookkeeping (Q#PD2), so looping /// is side-effect free. fn terminate_until_leader_exited( sup: &mut ProcessSupervisor, id: ProcessId, timeout: Duration, ) -> String { let deadline = Instant::now() + timeout; loop { sup.force_next_kill_errno(nix::errno::Errno::EPERM); let err = sup.terminate(id).expect_err("injected EPERM must fail"); if err.contains("leader=exited(") { return err; } assert!( !err.contains("is not running"), "the record left Running before the diagnostic could run, so \ this test never exercised it: {err}" ); assert!( Instant::now() < deadline, "leader never observed as exited within {timeout:?}: {err}" ); std::thread::sleep(Duration::from_millis(10)); } } /// Q#PD1 acceptance 1 — a group-directed failure names the target, /// the branch that chose it, the expected group, the errno, and the /// leader's own state, as five separate facts. /// /// Asserted as an exact message against the pid the kernel actually /// assigned, so a hardcoded target could not satisfy it. The leader /// field is the one that matters: for a PTY the signal goes to the /// terminal's foreground group, a different entity from the spawned /// child whenever job control has moved the terminal. Three rejected /// designs for this code collapsed the two; the report keeps them /// apart, and here they are asserted to agree only because nothing /// has moved the terminal. #[test] fn a_group_directed_kill_failure_reports_target_and_leader_separately() { let mut sup = ProcessSupervisor::new(); let (id, pid) = spawn_live_pty(&mut sup, "diag-group"); sup.force_next_kill_errno(nix::errno::Errno::EPERM); let err = sup.terminate(id).expect_err("injected EPERM must fail"); let expected = format!( "kill: {} (target=-{pid} via tcgetpgrp, leader_pid={pid}, expected_group=-{pid}, leader=live)", nix::errno::Errno::EPERM ); assert_eq!( err, expected, "the report names the exact target the tty reported, the exact \ leader pid, and observes the leader as live" ); let _ = sup.signal(id, Signal::SIGKILL); } /// Q#PD1 acceptance 2 — a leader-directed failure records the /// fallback branch and a positive target, and omits the group field /// that would be meaningless for it. Exact message again. #[test] fn a_leader_directed_kill_failure_reports_the_fallback_branch() { let mut sup = ProcessSupervisor::new(); let mut spec = ProcessSpec::new("diag-leader", "/bin/sleep"); spec.args = vec!["30".into()]; let id = sup.spawn(spec).expect("spawn"); let pid = spawn_started_pid(&mut sup, id); sup.force_next_kill_errno(nix::errno::Errno::ESRCH); let err = sup.terminate(id).expect_err("injected ESRCH must fail"); let expected = format!( "kill: {} (target={pid} via leader-pid, leader_pid={pid}, leader=live)", nix::errno::Errno::ESRCH ); assert_eq!( err, expected, "a non-group pipe child targets its own pid, and the group \ field is omitted where it has no meaning" ); let _ = sup.signal(id, Signal::SIGKILL); } /// Q#PD1 acceptance 3 — every leader state renders distinctly. The /// `Unobservable` and `NoRuntime` arms cannot be produced by a real /// child on demand, so they are pinned directly; `live` and `exited` /// are pinned through the real path by the tests around this one. #[test] fn every_leader_observation_renders_distinctly() { assert_eq!( LeaderObservation::Exited(TermStatus::Exited(0)).render(), "exited(code 0)" ); assert_eq!( LeaderObservation::Exited(TermStatus::Signaled("SIGTERM".into())).render(), "exited(signal SIGTERM)" ); assert_eq!(LeaderObservation::Live.render(), "live"); assert_eq!( LeaderObservation::Unobservable("try_wait: boom".into()).render(), "unobservable(try_wait: boom)" ); assert_eq!(LeaderObservation::NoRuntime.render(), "no-runtime"); } /// Q#PD1 acceptance 3, exited arm through the REAL path — the leader /// has genuinely exited and the report carries its exact code, not /// merely "some exit". #[test] fn a_failure_after_the_child_exits_reports_the_leader_as_exited() { let mut sup = ProcessSupervisor::new(); let mut spec = ProcessSpec::new("diag-exited", "/bin/sh"); spec.args = vec!["-c".into(), "exit 3".into()]; let id = sup.spawn(spec).expect("spawn"); // NOT `spawn_started_pid`: draining ticks, and this child exits // immediately. let pid = record_pid(&sup, id); let err = terminate_until_leader_exited(&mut sup, id, Duration::from_secs(10)); let expected = format!( "kill: {} (target={pid} via leader-pid, leader_pid={pid}, leader=exited(code 3))", nix::errno::Errno::EPERM ); assert_eq!( err, expected, "the exact exit code is observed from the real child, not \ inferred from the errno" ); } /// Q#PD2 acceptance 4 — **the disposition is unchanged.** An /// injected failure still fails, and neither the state transition /// nor the reap-ledger arming runs. This is the assertion that /// separates a diagnostic from the tolerance rules three review /// rounds rejected; flipping any arm to `Ok` fails it. #[test] fn an_injected_failure_changes_no_state_and_arms_no_ledger() { let mut sup = ProcessSupervisor::new(); let mut spec = ProcessSpec::new("diag-disposition", "/bin/sh"); spec.args = vec!["-c".into(), "sleep 30".into()]; spec.group = true; let id = sup.spawn(spec).expect("spawn"); let pid = spawn_started_pid(&mut sup, id); assert!( sup.reap_ledger.is_empty(), "precondition: nothing armed before the attempt" ); sup.force_next_kill_errno(nix::errno::Errno::EPERM); let err = sup.terminate(id).expect_err("injected EPERM must fail"); let expected = format!( "kill: {} (target=-{pid} via group, leader_pid={pid}, expected_group=-{pid}, leader=live)", nix::errno::Errno::EPERM ); assert_eq!(err, expected, "a group=true pipe child reports via group"); assert!( matches!( sup.processes.get(&id).expect("record").state, ProcessState::Running { .. } ), "a failed kill must not transition the record to Exiting" ); assert!( sup.reap_ledger.is_empty(), "a failed kill must not arm the reap ledger" ); let _ = sup.signal(id, Signal::SIGKILL); } /// Q#PD3/Q#PD4 acceptance 5 — the diagnostic consults the REAL /// `ChildHandle::try_wait` on the REAL child, which reaps it and /// caches the status. `poll_one` must still emit exactly one exit /// event, carrying the exact code. /// /// A stubbed observation would bypass the double-`try_wait` path /// entirely and pin nothing, so the injection replaces the kill /// result only. #[test] fn observing_the_leader_does_not_consume_the_exit_event() { let mut sup = ProcessSupervisor::new(); let mut spec = ProcessSpec::new("diag-one-event", "/bin/sh"); spec.args = vec!["-c".into(), "exit 7".into()]; spec.mode = ProcessMode::Pty { rows: 24, cols: 80, mode: TerminalMode::Canonical, }; let id = sup.spawn(spec).expect("spawn"); // NOT `spawn_started_pid`: draining ticks, and a tick can reap // this immediately-exiting child before the diagnostic runs. let _ = record_pid(&sup, id); // Drives `observe_leader`, which try_waits the real PTY child // for the first time and reaps it. let err = terminate_until_leader_exited(&mut sup, id, Duration::from_secs(10)); assert!( err.contains("leader=exited(code 7)"), "the real handle was consulted and carries the exact code: {err}" ); // The supervisor's own try_wait must still see that status. let evs = drain_until(&mut sup, id, Duration::from_secs(5), has_exited); let terminal: Vec = evs .iter() .filter_map(|e| match e.kind { ProcessEventKind::Exited { code, .. } => Some(code), ProcessEventKind::Signaled { .. } => Some(-1), _ => None, }) .collect(); assert_eq!( terminal, vec![7], "exactly one terminal event survives the diagnostic's try_wait, \ carrying the child's real exit code" ); } #[test] fn signal_terminates_a_running_child() { let mut sup = ProcessSupervisor::new(); // `sleep 30` is long enough that the test definitely needs // to terminate it deliberately. let mut spec = ProcessSpec::new("sleeper", "/bin/sh"); spec.args = vec!["-c".into(), "sleep 30".into()]; let id = sup.spawn(spec).expect("spawn"); // Wait for Started so we have a pid. let _ = drain_until(&mut sup, id, Duration::from_secs(2), |evs| { evs.iter() .any(|e| matches!(e.kind, ProcessEventKind::Started { .. })) }); sup.terminate(id).expect("terminate"); let after = drain_until(&mut sup, id, Duration::from_secs(5), |evs| { evs.iter() .any(|e| matches!(e.kind, ProcessEventKind::Signaled { .. })) }); assert!( after .iter() .any(|e| matches!(e.kind, ProcessEventKind::Signaled { .. })), "SIGTERM should produce a Signaled event" ); } #[test] fn write_stdin_queues_without_blocking_when_child_never_reads() { let mut sup = ProcessSupervisor::new(); // The child never reads its stdin, so the kernel pipe buffer // (~64 KiB) fills almost immediately. The pre-writer-thread // implementation blocked the caller in `write_all` here — // which in the editor was the main thread, wedging the frame // loop whenever an LSP server fell behind on its stdin. let mut spec = ProcessSpec::new("stdin-ignorer", "/bin/sh"); spec.args = vec!["-c".into(), "sleep 30".into()]; let id = sup.spawn(spec).expect("spawn"); let _ = drain_until(&mut sup, id, Duration::from_secs(2), |evs| { evs.iter() .any(|e| matches!(e.kind, ProcessEventKind::Started { .. })) }); let payload = vec![b'x'; 1024 * 1024]; // 16x the pipe buffer let start = Instant::now(); sup.write_stdin(id, &payload).expect("queued write"); assert!( start.elapsed() < Duration::from_secs(2), "write_stdin must queue, not block on pipe backpressure (took {:?})", start.elapsed() ); sup.terminate(id).expect("terminate"); let _ = drain_until(&mut sup, id, Duration::from_secs(5), has_exited); } #[test] fn close_stdin_flushes_queued_bytes_before_eof() { let mut sup = ProcessSupervisor::new(); // `cat` echoes stdin and exits on EOF. Receiving the full // payload back followed by a clean exit proves the writer // thread drains its queue before dropping the pipe (the // flush-then-EOF contract `close_stdin` documents). let mut spec = ProcessSpec::new("cat-echo", "/bin/sh"); spec.args = vec!["-c".into(), "cat".into()]; let id = sup.spawn(spec).expect("spawn"); let _ = drain_until(&mut sup, id, Duration::from_secs(2), |evs| { evs.iter() .any(|e| matches!(e.kind, ProcessEventKind::Started { .. })) }); let payload = vec![b'y'; 256 * 1024]; sup.write_stdin(id, &payload).expect("queued write"); sup.close_stdin(id).expect("close stdin"); let evs = drain_until(&mut sup, id, Duration::from_secs(10), |evs| { let echoed: usize = evs .iter() .filter_map(|e| match &e.kind { ProcessEventKind::Stdout(b) => Some(b.len()), _ => None, }) .sum(); echoed >= 256 * 1024 && evs .iter() .any(|e| matches!(e.kind, ProcessEventKind::Exited { .. })) }); let echoed: usize = evs .iter() .filter_map(|e| match &e.kind { ProcessEventKind::Stdout(b) => Some(b.len()), _ => None, }) .sum(); assert_eq!( echoed, payload.len(), "child must receive every queued byte before EOF" ); assert!( evs.iter() .any(|e| matches!(e.kind, ProcessEventKind::Exited { code: 0 })), "EOF after drain must let the child exit cleanly" ); } #[test] fn restart_on_crash_respawns_after_nonzero_exit() { let mut sup = ProcessSupervisor::new(); sup.set_restart_backoff(Duration::from_millis(10)); let mut spec = ProcessSpec::new("crasher", "/bin/sh"); spec.args = vec!["-c".into(), "exit 7".into()]; spec.restart = RestartPolicy::OnCrash; let id = sup.spawn(spec).expect("spawn"); // Wait for at least one restart (Restarting + a second Started). let evs = drain_until(&mut sup, id, Duration::from_secs(5), |evs| { evs.iter() .filter(|e| matches!(e.kind, ProcessEventKind::Started { .. })) .count() >= 2 }); let started_count = evs .iter() .filter(|e| matches!(e.kind, ProcessEventKind::Started { .. })) .count(); assert!( started_count >= 2, "OnCrash restart should respawn after non-zero exit; saw Started count {started_count}" ); assert!( evs.iter() .any(|e| matches!(e.kind, ProcessEventKind::Restarting { .. })), "must emit Restarting" ); // Stop the loop before test exit to keep things tidy. sup.processes.get_mut(&id).unwrap().spec.restart = RestartPolicy::Never; } #[test] fn restart_never_does_not_respawn_after_clean_exit() { let mut sup = ProcessSupervisor::new(); let mut spec = ProcessSpec::new("oneshot", "/bin/sh"); spec.args = vec!["-c".into(), "exit 0".into()]; let id = sup.spawn(spec).expect("spawn"); let _ = drain_until(&mut sup, id, Duration::from_secs(2), has_exited); // Several more ticks; no restart should occur. for _ in 0..5 { sup.tick(); std::thread::sleep(Duration::from_millis(10)); } let starts = sup .take_events(id) .iter() .chain(sup.take_all_events().iter()) .filter(|e| matches!(e.kind, ProcessEventKind::Started { .. })) .count(); assert_eq!(starts, 0, "no further Started events expected"); let proc = sup.processes.get(&id).expect("still tracked"); assert!(matches!(proc.state, ProcessState::Terminated(_))); } #[test] fn drop_supervisor_kills_running_children() { // Spawn a long-running child, drop the supervisor, and // verify the child is gone (try sending signal 0 via nix: // ESRCH means already reaped). Bounded wait because zombie // reaping is asynchronous on some platforms. let pid = { let mut sup = ProcessSupervisor::new(); sup.set_grace_period(Duration::from_millis(200)); let mut spec = ProcessSpec::new("victim", "/bin/sh"); spec.args = vec!["-c".into(), "sleep 30".into()]; let id = sup.spawn(spec).expect("spawn"); // Drain until Started so we know the pid. let _ = drain_until(&mut sup, id, Duration::from_secs(2), |evs| { evs.iter() .any(|e| matches!(e.kind, ProcessEventKind::Started { .. })) }); let ProcessState::Running { pid, .. } = sup.state(id).cloned().unwrap() else { panic!("expected Running"); }; pid }; // sup drops here -> shutdown -> SIGTERM/SIGKILL // Give the kernel a brief moment to deliver the signal; the // bounded loop tolerates jitter. let nix_pid = Pid::from_raw(i32::try_from(pid).unwrap()); let dead_or_unknown = || { // signal 0 returns Ok if pid exists, ESRCH otherwise. // After Drop the child is reaped or dead; ESRCH is the // expected outcome. matches!( nix::sys::signal::kill(nix_pid, None), Err(nix::errno::Errno::ESRCH) ) }; let deadline = Instant::now() + Duration::from_secs(2); while !dead_or_unknown() && Instant::now() < deadline { std::thread::sleep(Duration::from_millis(20)); } assert!( dead_or_unknown(), "child pid {pid} should be reaped/gone after supervisor Drop" ); } #[test] fn pty_mode_child_sees_a_tty() { let mut sup = ProcessSupervisor::new(); let mut spec = ProcessSpec::new("ttytest", "/bin/sh"); spec.args = vec!["-c".into(), "tty".into()]; spec.mode = ProcessMode::default_pty(); let id = sup.spawn(spec).expect("spawn"); let evs = drain_until(&mut sup, id, Duration::from_secs(5), has_exited); // Concatenate all stdout chunks; tty(1) prints the path of // the controlling terminal, which on Linux/macOS starts with // /dev/pts/ or /dev/ttys. let mut out = Vec::new(); for e in &evs { if let ProcessEventKind::Stdout(bytes) = &e.kind { out.extend_from_slice(bytes); } } let s = String::from_utf8_lossy(&out); assert!( s.contains("/dev/pts/") || s.contains("/dev/ttys"), "tty(1) should report a pty path in PTY mode; got {s:?}" ); } fn collect_stdout(events: &[ProcessEvent]) -> String { let mut out = Vec::new(); for e in events { if let ProcessEventKind::Stdout(bytes) = &e.kind { out.extend_from_slice(bytes); } } String::from_utf8_lossy(&out).into_owned() } /// T M6.1: PTY child observes SIGWINCH when the supervisor /// resizes its window. The child traps WINCH and echoes the new /// dimensions; we resize and look for the marker on stdout. #[test] fn m6_1_pty_resize_delivers_sigwinch_to_child() { let mut sup = ProcessSupervisor::new(); let mut spec = ProcessSpec::new("winch-watch", "/bin/sh"); // Trap WINCH, print READY for synchronization, then loop on // a short sleep so SIGWINCH can interrupt and fire the trap. spec.args = vec![ "-c".into(), "trap 'echo RESIZED:$(stty size)' WINCH; \ echo READY; \ while :; do sleep 0.05; done" .into(), ]; spec.mode = ProcessMode::default_pty(); let id = sup.spawn(spec).expect("spawn"); // Wait for READY so we know the trap is installed before we // signal. let _ = drain_until(&mut sup, id, Duration::from_secs(5), |evs| { collect_stdout(evs).contains("READY") }); sup.resize_pty(id, 40, 120).expect("resize"); let evs = drain_until(&mut sup, id, Duration::from_secs(5), |evs| { collect_stdout(evs).contains("RESIZED:") }); let stdout = collect_stdout(&evs); // `stty size` prints "rows cols" on Linux, with possible // leading/trailing whitespace differences across platforms. assert!( stdout.contains("RESIZED:40 120") || stdout.contains("RESIZED: 40 120"), "child should observe SIGWINCH and report new size 40x120; \ collected stdout was: {stdout:?}" ); // Stop the loop so the test exits cleanly. let _ = sup.terminate(id); } /// T M6.1: a PTY-mode child that exits cleanly produces the same /// `Started` -> `Exited` lifecycle as a pipe-mode child, with /// reader threads joining on EOF and the supervisor reaching /// `Terminated` state. #[test] fn m6_1_pty_mode_lifecycle_started_then_exited() { let mut sup = ProcessSupervisor::new(); let mut spec = ProcessSpec::new("pty-exit", "/bin/sh"); spec.args = vec!["-c".into(), "echo done && exit 0".into()]; spec.mode = ProcessMode::default_pty(); let id = sup.spawn(spec).expect("spawn"); let events = drain_until(&mut sup, id, Duration::from_secs(5), has_exited); assert!( events .iter() .any(|e| matches!(e.kind, ProcessEventKind::Started { .. })), "must observe Started for PTY-mode child" ); assert!( events .iter() .any(|e| matches!(&e.kind, ProcessEventKind::Stdout(b) if b.windows(4).any(|w| w == b"done"))), "must observe stdout 'done'" ); assert!( events .iter() .any(|e| matches!(e.kind, ProcessEventKind::Exited { code: 0 })), "PTY-mode child should exit cleanly with code 0" ); let proc = sup.processes.get(&id).expect("still tracked"); assert!(matches!(proc.state, ProcessState::Terminated(_))); } /// T M6.1: `TerminalMode::Raw` (the default) produces a PTY where /// the kernel does not echo input. Verified by running `stty -a` /// inside the PTY and looking for `-echo` and `-icanon` in its /// output. #[test] fn m6_1_pty_raw_mode_disables_kernel_echo() { let mut sup = ProcessSupervisor::new(); let mut spec = ProcessSpec::new("raw-stty", "/bin/sh"); spec.args = vec!["-c".into(), "stty -a".into()]; spec.mode = ProcessMode::default_pty(); // Raw by default. let id = sup.spawn(spec).expect("spawn"); let evs = drain_until(&mut sup, id, Duration::from_secs(5), has_exited); let stdout = collect_stdout(&evs); assert!( stdout.contains("-echo"), "raw mode should disable echo; stty -a output was: {stdout:?}" ); assert!( stdout.contains("-icanon"), "raw mode should disable canonical input; stty -a output was: {stdout:?}" ); } /// T M6.1: `TerminalMode::Canonical` keeps the kernel default, /// where echo and canonical input are enabled. Mirrors the raw /// test in reverse. #[test] fn m6_1_pty_canonical_mode_keeps_kernel_echo() { let mut sup = ProcessSupervisor::new(); let mut spec = ProcessSpec::new("canon-stty", "/bin/sh"); spec.args = vec!["-c".into(), "stty -a".into()]; spec.mode = ProcessMode::Pty { rows: 24, cols: 80, mode: TerminalMode::Canonical, }; let id = sup.spawn(spec).expect("spawn"); let evs = drain_until(&mut sup, id, Duration::from_secs(5), has_exited); let stdout = collect_stdout(&evs); // Disambiguate `echo` from `-echo` (raw) and from longer flag // names like `iexten`. Word-boundary check via padded match. assert!( (stdout.contains(" echo ") || stdout.contains(" echo\n") || stdout.contains("\necho ") || stdout.starts_with("echo ")) && !stdout.contains("-echo "), "canonical mode should leave echo enabled (no `-echo` flag); \ stty -a output was: {stdout:?}" ); } /// Test helper: number of byte chunks currently buffered in the /// per-generation bounded channel for `id`. Used by M6.2 tests to /// observe backpressure saturation. fn byte_channel_len(sup: &ProcessSupervisor, id: ProcessId) -> usize { sup.processes .get(&id) .and_then(|p| p.runtime.as_ref()) .map_or(0, |rt| match &rt.output_rx { RuntimeOutputRx::Bytes(rx) => rx.len(), RuntimeOutputRx::Ansi(_) => 0, }) } /// T M6.2 acceptance bullet 1: the per-generation byte channel /// caps in-flight bytes at the spec's 1 MiB ceiling, so a /// saturating producer stalls in `write` rather than ballooning /// supervisor memory. Asserts (a) the bounded channel never /// exceeds its slot cap, (b) the producer is still alive after a /// pause-drain window (was actually backpressured, not just very /// slow), and (c) every byte is delivered once draining resumes. #[test] fn m6_2_pty_streaming_respects_byte_ceiling() { use nix::sys::signal::kill; use nix::unistd::Pid; // 10 MiB target: comfortably more than (1 MiB channel + 64 // KiB kernel pipe), so the producer must stall in write // rather than fitting the entire payload in the un-drained // buffers. const TOTAL: usize = 10 * 1024 * 1024; let mut sup = ProcessSupervisor::new(); let mut spec = ProcessSpec::new("byte-flood", "/bin/sh"); spec.args = vec!["-c".into(), format!("head -c {TOTAL} /dev/zero")]; let id = sup.spawn(spec).expect("spawn"); // Read pid synchronously --- spawn → start_generation // already set the state to Running. We deliberately do NOT // call drain_until(Started) here: that path ticks the // supervisor in a loop, which drains the byte channel and // unsticks the reader, defeating the saturation observation // we are about to make. let pid = match sup.state(id).expect("tracked") { ProcessState::Running { pid, .. } => *pid, s => panic!("expected Running immediately after spawn; got {s:?}"), }; // Pause draining: no `tick()` calls. The reader thread // saturates the bounded channel, the kernel pipe fills, the // producer stalls in `write`. std::thread::sleep(Duration::from_millis(200)); // (a) bounded channel never exceeds the slot cap. let in_flight = byte_channel_len(&sup, id); assert!( in_flight <= BYTE_CHUNK_CHANNEL_CAP, "byte channel must be bounded by {BYTE_CHUNK_CHANNEL_CAP} \ slots; observed {in_flight}" ); assert!( in_flight > 0, "after 200 ms producing {TOTAL} bytes without draining, \ the channel should have data; got {in_flight}" ); // (b) the producer is still alive --- having NOT delivered // 10 MiB through a ~1 MiB ceiling means it is stalled in // write. We use an OS-level liveness check (kill(0)) rather // than the supervisor's cached state, since the cached // state is only updated by `tick()` and ticking would drain // the channel. let live = kill( Pid::from_raw(i32::try_from(pid).expect("pid fits i32")), None, ) .is_ok(); assert!( live, "producer (pid {pid}) should still be alive (stalled in write); \ without backpressure, head would have written 10 MiB and exited" ); // (c) drain to completion; verify exact byte count is // preserved across the backpressure-release boundary. let mut total = 0usize; let deadline = Instant::now() + Duration::from_secs(15); while total < TOTAL && Instant::now() < deadline { sup.tick(); for ev in sup.take_events(id) { if let ProcessEventKind::Stdout(b) = ev.kind { total += b.len(); } } std::thread::sleep(Duration::from_millis(5)); } assert_eq!( total, TOTAL, "expected {TOTAL} bytes after backpressure releases; got {total}" ); } /// T M6.2 / M3.5 coalescing: many in-flight chunks present at /// the same tick produce one coalesced `Stdout` event, not one /// event per chunk. The supervisor concatenates per-process /// bytes into a single event per tick (per kind). #[test] fn m6_2_pty_streaming_coalesces_per_tick() { let mut sup = ProcessSupervisor::new(); let mut spec = ProcessSpec::new("chunky-stream", "/bin/sh"); // 1 MiB of zeros from /dev/zero. The reader thread reads in // [`BYTE_CHUNK_SIZE`] (8 KiB) chunks --- ~128 reads --- all // queued onto the bounded channel within microseconds of // each other. Without coalescing, those would surface as // ~128 separate `Stdout` events; with coalescing, they // merge into a small handful (one per tick that drains // them). The trailing `END` marker is purely a // synchronization tag. spec.args = vec!["-c".into(), "head -c 1048576 /dev/zero; echo END".into()]; let id = sup.spawn(spec).expect("spawn"); let evs = drain_until(&mut sup, id, Duration::from_secs(10), |evs| { evs.iter().any( |e| matches!(&e.kind, ProcessEventKind::Stdout(b) if b.windows(3).any(|w| w == b"END")), ) }); let stdout_event_count = evs .iter() .filter(|e| matches!(e.kind, ProcessEventKind::Stdout(_))) .count(); let total_bytes: usize = evs .iter() .filter_map(|e| match &e.kind { ProcessEventKind::Stdout(b) => Some(b.len()), _ => None, }) .sum(); // No-loss assertion: every byte arrives (1 MiB + "END\n"). assert!( total_bytes >= 1_048_580, "expected ≥ 1 048 580 stdout bytes (1 MiB + END\\n); got {total_bytes}" ); // Coalescing assertion: ~128 underlying reads collapse to a // small handful of events. A loose ceiling of 16 absorbs // scheduler jitter and the eventual END-marker tick; the // typical observed count is 1--3. An un-coalesced path // would emit ≥ 128 events. assert!( stdout_event_count <= 16, "too many stdout events ({stdout_event_count}); coalescing \ not engaged --- expected O(ticks), not O(reads)" ); } #[test] fn m6_2_ansi_enabled_pty_emits_structured_events() { let mut sup = ProcessSupervisor::new(); let mut spec = ProcessSpec::new("ansi-stream", "/bin/sh"); spec.args = vec!["-c".into(), "printf '\\033[31mhi\\033[0m\\n'".into()]; spec.mode = ProcessMode::Pty { rows: 24, cols: 80, mode: TerminalMode::Canonical, }; spec.ansi_events = true; let id = sup.spawn(spec).expect("spawn ansi pty"); let evs = drain_until(&mut sup, id, Duration::from_secs(5), has_exited); assert!( evs.iter() .all(|e| !matches!(e.kind, ProcessEventKind::Stdout(_))), "ansi-enabled PTY should not surface raw stdout events: {evs:?}" ); let mut saw_red = false; let mut saw_text = false; for ev in evs { if let ProcessEventKind::Ansi(events) = ev.kind { for event in events { match event { AnsiEvent::SetStyle(style) if style.fg == crate::cell::Color::Indexed(1) => { saw_red = true; } AnsiEvent::Text(text) if text.contains("hi") => { saw_text = true; } _ => {} } } } } assert!(saw_red, "expected structured red SetStyle event"); assert!(saw_text, "expected structured text event"); } #[test] fn m6_2_ansi_parser_worker_exits_when_reader_channel_closes() { let (byte_tx, byte_rx) = channel::bounded::(1); let (ansi_tx, _ansi_rx) = channel::bounded::(1); let cancel = Arc::new(AtomicBool::new(false)); let handle = spawn_ansi_parser( byte_rx, ansi_tx, Arc::clone(&cancel), AnsiParserProfile::LineOriented, ); drop(byte_tx); let deadline = Instant::now() + Duration::from_millis(500); while Instant::now() < deadline && !handle.is_finished() { std::thread::sleep(Duration::from_millis(5)); } if !handle.is_finished() { cancel.store(true, Ordering::Relaxed); } assert!( handle.is_finished(), "ANSI parser worker should exit once the byte reader channel closes" ); handle.join().expect("parser worker join"); } /// T M6.2 acceptance bullet 2: stream cancellation propagates to /// the source. A long-lived producer with the consumer paused /// (so the reader is blocked in `send`) still terminates /// promptly when the supervisor is shut down --- the /// `RuntimeHandles::Drop` cancel-flag wake-out unblocks the /// reader before the join, and `shutdown` reaps the producer /// within the grace window. #[test] fn m6_2_pty_streaming_cancellation_propagates_to_child() { use std::sync::mpsc; // Run the supervisor in a worker thread so the test can // bound how long the cancellation path takes via a oneshot. let (done_tx, done_rx) = mpsc::channel(); let handle = std::thread::spawn(move || { let mut sup = ProcessSupervisor::new(); sup.set_grace_period(Duration::from_millis(300)); let mut spec = ProcessSpec::new("forever-flood", "/bin/sh"); // Continuous writer; SIGTERM kills it (no signal handler). spec.args = vec!["-c".into(), "while :; do printf 'X'; done".into()]; let id = sup.spawn(spec).expect("spawn"); // Wait for the bounded channel to saturate. We are NOT // ticking, so once the channel is full the reader // thread is blocked in `send_timeout`. let saturate_deadline = Instant::now() + Duration::from_secs(3); while Instant::now() < saturate_deadline && byte_channel_len(&sup, id) < BYTE_CHUNK_CHANNEL_CAP { std::thread::sleep(Duration::from_millis(10)); } assert_eq!( byte_channel_len(&sup, id), BYTE_CHUNK_CHANNEL_CAP, "channel should saturate when consumer doesn't drain" ); // Drop the supervisor. Its `Drop` runs `shutdown`: // SIGTERM, tick (which drains and unblocks the reader), // possibly SIGKILL, then `RuntimeHandles::Drop` which // sets the cancel flag and joins the reader. Any of // these mechanisms suffices to unblock the reader; the // assertion is that shutdown completes within a bound. drop(sup); let _ = done_tx.send(()); }); done_rx.recv_timeout(Duration::from_secs(5)).expect( "supervisor drop should complete within 5s --- if hung, \ cancellation is not propagating to a reader blocked in \ send (per-generation cancel flag is required)", ); handle.join().expect("test thread should exit cleanly"); } // ----------------------------------------------------------------- // Compile-mode group lifecycle (Q#CM3; framing acceptance 34) // ----------------------------------------------------------------- fn sh_group_spec(label: &str, script: &str) -> ProcessSpec { let mut spec = ProcessSpec::new(label, "/bin/sh"); spec.args = vec!["-c".into(), script.to_owned()]; spec.stdin = StdinMode::Null; spec.group = true; spec } fn started_pid(events: &[ProcessEvent]) -> Option { events.iter().find_map(|e| match e.kind { ProcessEventKind::Started { pid } => Some(pid), _ => None, }) } fn stdout_contains(events: &[ProcessEvent], needle: &[u8]) -> bool { let mut all = Vec::new(); for e in events { if let ProcessEventKind::Stdout(b) = &e.kind { all.extend_from_slice(b); } } all.windows(needle.len()).any(|w| w == needle) } fn pid_alive(pid: i32) -> bool { nix::sys::signal::kill(Pid::from_raw(pid), None).is_ok() } /// Process group of `pid` via `ps` (portable across Linux and /// macOS CI — the previous /proc//stat read has no macOS /// equivalent; `ps -o pgid=` avoids widening the nix feature set /// with `process` for `getpgid`). fn pgid_of(pid: u32) -> i32 { let out = std::process::Command::new("ps") .args(["-o", "pgid=", "-p", &pid.to_string()]) .output() .expect("run ps"); String::from_utf8_lossy(&out.stdout) .trim() .parse() .expect("pgid parses") } /// True when `name` resolves on PATH. Fixture-dependency gate: /// the setsid escape-hatch test needs util-linux's setsid(1), /// absent on macOS — skip per-test rather than fail (the /// `m6_5_repl_acceptance` selective-skip precedent). fn binary_available(name: &str) -> bool { std::process::Command::new("which") .arg(name) .output() .is_ok_and(|o| o.status.success()) } /// Fixture: background a TERM-ignoring survivor and let the /// leader exit only after the survivor's trap is INSTALLED /// (readiness file). Without the gate, a slow scheduler (macOS /// CI, observed) can deliver the leader-exit group-TERM before /// the subshell's `trap` runs — killing the "survivor": flaky /// red for tests that need it alive, vacuous green for tests /// that assert its death. `redirect` sheds the survivor's /// stdout/stderr (the acceptance-8 shape); without it the /// survivor keeps fd1 (the acceptance-9 shape). Returns /// (script, pidfile). fn survivor_script(dir: &std::path::Path, redirect: bool) -> (String, std::path::PathBuf) { let pidfile = dir.join("pid"); let ready = dir.join("ready"); let redirect_part = if redirect { "exec >/dev/null 2>&1; " } else { "" }; let script = format!( "( trap '' TERM; : > {ready}; {redirect_part}sleep 30 ) & echo $! > {pid}; \ while [ ! -e {ready} ]; do sleep 0.01; done", ready = ready.display(), pid = pidfile.display(), ); (script, pidfile) } /// Poll `path` until it holds a parseable pid. Fixture scripts /// write descendant pids there. fn wait_pidfile(path: &std::path::Path) -> i32 { let stop = Instant::now() + Duration::from_secs(5); while Instant::now() < stop { if let Ok(s) = std::fs::read_to_string(path) && let Ok(pid) = s.trim().parse::() { return pid; } std::thread::sleep(Duration::from_millis(10)); } panic!("pidfile {} never appeared", path.display()); } #[test] fn stdin_null_yields_immediate_eof() { let mut sup = ProcessSupervisor::new(); // `cat` exits only at stdin EOF; under piped stdin this test // would hang until the drain deadline killed it. (Framing // acceptance 34 / round-1 finding 3.) let spec = sh_group_spec("eof-test", "cat; echo done"); let id = sup.spawn(spec).expect("spawn"); let events = drain_until(&mut sup, id, Duration::from_secs(5), has_exited); assert!( events .iter() .any(|e| matches!(e.kind, ProcessEventKind::Exited { code: 0 })), "cat must see EOF and exit 0; events: {events:?}" ); assert!( stdout_contains(&events, b"done"), "post-cat echo must run; events: {events:?}" ); } #[test] fn group_true_spawns_distinct_process_group() { let mut sup = ProcessSupervisor::new(); let id = sup .spawn(sh_group_spec("group-test", "sleep 30")) .expect("spawn"); let events = drain_until(&mut sup, id, Duration::from_secs(2), |evs| { started_pid(evs).is_some() }); let pid = started_pid(&events).expect("Started event"); assert_eq!( pgid_of(pid), i32::try_from(pid).unwrap(), "group child must lead its own process group (pgid == pid)" ); // Control: a non-group child inherits the test process's // group instead of leading its own. let mut plain = ProcessSpec::new("plain", "/bin/sh"); plain.args = vec!["-c".into(), "sleep 30".into()]; let plain_id = sup.spawn(plain).expect("spawn plain"); let plain_events = drain_until(&mut sup, plain_id, Duration::from_secs(2), |evs| { started_pid(evs).is_some() }); let plain_pid = started_pid(&plain_events).expect("Started event"); assert_ne!( pgid_of(plain_pid), i32::try_from(plain_pid).unwrap(), "non-group child must not lead its own group" ); sup.terminate(id).ok(); sup.terminate(plain_id).ok(); let _ = drain_until(&mut sup, id, Duration::from_secs(5), has_exited); let _ = drain_until(&mut sup, plain_id, Duration::from_secs(5), has_exited); } #[test] fn terminate_group_escalates_to_sigkill_on_term_trapping_child() { let mut sup = ProcessSupervisor::new(); sup.set_group_term_grace(Duration::from_millis(200)); // Readiness echo: terminating before the trap is installed // would let plain SIGTERM win and vacuously pass. let id = sup .spawn(sh_group_spec( "trap-test", "trap '' TERM; echo ready; sleep 30", )) .expect("spawn"); let _ = drain_until(&mut sup, id, Duration::from_secs(2), |evs| { stdout_contains(evs, b"ready") }); let t0 = Instant::now(); sup.terminate(id).expect("terminate"); let events = drain_until(&mut sup, id, Duration::from_secs(5), has_exited); let elapsed = t0.elapsed(); assert!( events.iter().any(|e| matches!( &e.kind, ProcessEventKind::Signaled { signal } if signal == "SIGKILL" )), "TERM-trapping child must fall to the ledger's SIGKILL; events: {events:?}" ); assert!( elapsed < Duration::from_millis(1500), "escalation must land near the 200ms grace, not the 2s drain timeout; took {elapsed:?}" ); } #[test] fn liveness_probe_reaps_term_ignoring_survivor_after_leader_exit() { // Unit twin of framing acceptance 8: the survivor ignores // TERM *and* sheds its stdout/stderr, so the leader's // terminal event arrives and the readers finish — only the // ledger's kill(-pgid, 0) probe can catch it. let dir = tempfile::tempdir().expect("tempdir"); let mut sup = ProcessSupervisor::new(); sup.set_group_term_grace(Duration::from_millis(200)); let (script, pidfile) = survivor_script(dir.path(), true); let id = sup .spawn(sh_group_spec("survivor", &script)) .expect("spawn"); let events = drain_until(&mut sup, id, Duration::from_secs(5), has_exited); assert!( events .iter() .any(|e| matches!(e.kind, ProcessEventKind::Exited { code: 0 })), "leader must exit cleanly; events: {events:?}" ); let survivor = wait_pidfile(&pidfile); // The ledger fires on subsequent ticks — keep ticking. let stop = Instant::now() + Duration::from_secs(3); while Instant::now() < stop && pid_alive(survivor) { sup.tick(); std::thread::sleep(Duration::from_millis(10)); } assert!( !pid_alive(survivor), "TERM-ignoring redirected survivor must be SIGKILLed by the ledger probe" ); // Ledger converges to empty once the group probes ESRCH. let stop = Instant::now() + Duration::from_secs(2); while Instant::now() < stop && sup.reap_ledger_len() > 0 { sup.tick(); std::thread::sleep(Duration::from_millis(10)); } assert_eq!(sup.reap_ledger_len(), 0, "ledger must drain to empty"); } #[test] fn repeated_terminate_does_not_extend_ledger_deadline() { let mut sup = ProcessSupervisor::new(); sup.set_group_term_grace(Duration::from_millis(500)); let id = sup .spawn(sh_group_spec( "re-term", "trap '' TERM; echo ready; sleep 30", )) .expect("spawn"); let _ = drain_until(&mut sup, id, Duration::from_secs(2), |evs| { stdout_contains(evs, b"ready") }); let t0 = Instant::now(); sup.terminate(id).expect("terminate"); // Re-terminate at half the grace window: with plain // HashMap::insert arming, this would reset the 500ms clock // and push SIGKILL past 800ms. std::thread::sleep(Duration::from_millis(300)); sup.tick(); sup.terminate(id).expect("re-terminate"); let events = drain_until(&mut sup, id, Duration::from_secs(5), has_exited); let elapsed = t0.elapsed(); assert!( events.iter().any(|e| matches!( &e.kind, ProcessEventKind::Signaled { signal } if signal == "SIGKILL" )), "must escalate; events: {events:?}" ); assert!( elapsed < Duration::from_millis(750), "earliest deadline must win: SIGKILL by ~500ms, not 800ms; took {elapsed:?}" ); } #[test] fn shutdown_force_kills_outstanding_ledger_groups() { // Drop-twin of framing acceptance 8 (round-4 finding 1): the // grace is long enough that the ledger cannot fire on its // own — only shutdown's force-kill can reap the survivor. let dir = tempfile::tempdir().expect("tempdir"); let mut sup = ProcessSupervisor::new(); sup.set_group_term_grace(Duration::from_secs(30)); let (script, pidfile) = survivor_script(dir.path(), true); let id = sup .spawn(sh_group_spec("survivor", &script)) .expect("spawn"); let _ = drain_until(&mut sup, id, Duration::from_secs(5), has_exited); let survivor = wait_pidfile(&pidfile); assert!(pid_alive(survivor), "survivor alive pre-shutdown"); assert!(sup.reap_ledger_len() > 0, "ledger armed pre-shutdown"); sup.shutdown(); assert!( !pid_alive(survivor), "shutdown must force-kill outstanding ledger groups" ); assert_eq!( sup.reap_ledger_len(), 0, "shutdown must probe forced kills to ESRCH" ); } #[test] fn maybe_restart_inert_once_shut_down() { let mut sup = ProcessSupervisor::new(); sup.set_restart_backoff(Duration::from_millis(30)); let mut spec = ProcessSpec::new("restarter", "/bin/sh"); spec.args = vec!["-c".into(), "echo x".into()]; spec.restart = RestartPolicy::Always; let id = sup.spawn(spec).expect("spawn"); // Prove the policy is live: observe at least one restart. let events = drain_until(&mut sup, id, Duration::from_secs(5), |evs| { evs.iter() .any(|e| matches!(e.kind, ProcessEventKind::Restarting { .. })) }); assert!( events .iter() .any(|e| matches!(e.kind, ProcessEventKind::Restarting { .. })), "restart=always must restart pre-shutdown; events: {events:?}" ); sup.shutdown(); let _ = sup.take_events(id); // Give a reset restart-backoff window plenty of room, then // confirm no respawn happened during or after teardown. for _ in 0..8 { sup.tick(); std::thread::sleep(Duration::from_millis(20)); } let after = sup.take_events(id); assert!( !after.iter().any(|e| matches!( e.kind, ProcessEventKind::Restarting { .. } | ProcessEventKind::Started { .. } )), "restart accounting must be inert once shut down; events: {after:?}" ); } #[test] fn leader_exit_reap_bounds_drain_with_pipe_holding_descendant() { // Unit twin of framing acceptance 9: the descendant ignores // TERM and KEEPS fd1, so the readers stay alive and the old // drain would block ~2s per EXIT_OUTPUT_DRAIN_TIMEOUT (and // then the join would hang). In-drain ledger enforcement // SIGKILLs at the grace bound instead. Readiness-gated so an // early leader-exit TERM can't reap the holder and let the // bound hold vacuously. let dir = tempfile::tempdir().expect("tempdir"); let mut sup = ProcessSupervisor::new(); sup.set_group_term_grace(Duration::from_millis(300)); let (script, _pidfile) = survivor_script(dir.path(), false); let id = sup.spawn(sh_group_spec("holder", &script)).expect("spawn"); let stop = Instant::now() + Duration::from_secs(5); let mut max_tick = Duration::ZERO; let mut events = Vec::new(); while Instant::now() < stop && !has_exited(&events) { let t = Instant::now(); sup.tick(); max_tick = max_tick.max(t.elapsed()); events.append(&mut sup.take_events(id)); std::thread::sleep(Duration::from_millis(10)); } assert!( has_exited(&events), "leader exit must be observed; events: {events:?}" ); assert!( max_tick < Duration::from_millis(1200), "the blocking tick must be bounded by ~grace + 2 poll intervals, \ not the 2s drain timeout; max tick {max_tick:?}" ); } #[test] fn setsid_escapee_is_not_reaped_and_teardown_reclaims_readers() { // The setsid'd descendant leaves the group (the deliberate // daemonization escape hatch) while inheriting fd1, so it // holds the pipe after its old group is ESRCH. The // quiescence/cancel cap must bound the drain, the retained // joins must complete, and the per-runtime active-reader // count must return to zero — across repeated cycles, so // nothing accumulates. if !binary_available("setsid") { // util-linux's setsid(1) is absent on macOS CI; the // escape hatch is a Linux-production behavior. Skip // rather than fail — the other group-lifecycle tests // still run everywhere. eprintln!("skipping: setsid(1) not on PATH"); return; } let dir = tempfile::tempdir().expect("tempdir"); let mut escapees = Vec::new(); for round in 0..3 { let pidfile = dir.path().join(format!("pid{round}")); let mut sup = ProcessSupervisor::new(); sup.set_group_term_grace(Duration::from_millis(300)); // Do not let the group leader exit until the background child has // completed `setsid` and published its pid. Without this readiness // gate, teardown can TERM the old process group before `setsid` // runs; the child then dies before creating the pidfile (a race // exposed consistently by the Ubuntu 20260714 runner image). let script = format!( "setsid /bin/sh -c 'echo $$ > {pid}; exec sleep 30' & \ while [ ! -s {pid} ]; do sleep 0.01; done; echo started", pid = pidfile.display() ); let id = sup.spawn(sh_group_spec("escapee", &script)).expect("spawn"); let ready = drain_until(&mut sup, id, Duration::from_secs(2), |evs| { started_pid(evs).is_some() }); assert!(started_pid(&ready).is_some(), "Started must arrive"); let probe = sup.active_reader_probe(id).expect("live runtime probe"); let t0 = Instant::now(); let events = drain_until(&mut sup, id, Duration::from_secs(5), has_exited); let elapsed = t0.elapsed(); assert!( has_exited(&events), "leader exit must be observed; events: {events:?}" ); assert!( elapsed < Duration::from_millis(1500), "escaped-writer drain must be cancelled at the cap, \ not ride the 2s timeout; took {elapsed:?}" ); assert_eq!( probe.load(Ordering::Relaxed), 0, "reader threads must have ended and dropped their FDs" ); let escapee = wait_pidfile(&pidfile); assert!( pid_alive(escapee), "setsid escapee must NOT be reaped (deliberate escape hatch)" ); escapees.push(escapee); } // Fixture owns the escapees the supervisor deliberately // does not: kill them explicitly. for pid in escapees { let _ = nix::sys::signal::kill(Pid::from_raw(pid), Some(Signal::SIGKILL)); } } #[test] fn group_and_null_stdin_rejected_under_pty() { let mut sup = ProcessSupervisor::new(); let mut spec = ProcessSpec::new("pty-null", "/bin/sh"); spec.mode = ProcessMode::default_pty(); spec.stdin = StdinMode::Null; let err = sup .spawn(spec) .expect_err("stdin=null must be rejected under pty"); assert!( err.contains("pipe mode"), "error points at pipe mode: {err}" ); let mut spec = ProcessSpec::new("pty-group", "/bin/sh"); spec.mode = ProcessMode::default_pty(); spec.group = true; let err = sup .spawn(spec) .expect_err("group=true must be rejected under pty"); assert!( err.contains("pipe mode"), "error points at pipe mode: {err}" ); } }