neuwirth
/
pmacs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #235 from levineuwirth/lsp-file-watch-d3 

feat(lsp): D3 — the file watcher stops sleeping and walks once per scan (#233)
This commit is contained in:
Levi Neuwirth 2026-08-11 16:12:18 +00:00 committed by GitHub
parent add0ba1a20 ac02952c91
commit 122b8e8ce9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 2507 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
builtin/runtime/fs.lua
View File

@ -106,6 +106,7 @@ end
local READ_DIR_OPTS = { supersede = true, tolerant = true }
local STAT_OPTS = { supersede = true }
local WALK_TREE_OPTS = { supersede = true }
-- Two result shapes, chosen by `opts.tolerant` (dired Q#DR6):
--
@ -130,6 +131,22 @@ function fs.read_dir(path, opts)
return build_handle(id)
end
-- walk_tree(base [, opts]) -> handle; await -> { <entry>, ... } where
-- each entry is the read_dir shape but `name` is a BASE-RELATIVE path
-- ("sub/dir/file.txt") and the listing covers the whole tree as ONE
-- job (issue #233 D3). Symlinks are recorded, never traversed; an
-- unreadable subdirectory is skipped with its subtree; only the root
-- failing to open fails the walk. Directory entries are included
-- (kind "dir") --- consumers that only want files filter on kind.
function fs.walk_tree(base, opts)
if type(base) ~= "string" then
error("pmacs.fs.walk_tree: base must be a string, got " .. type(base))
end
local key = read_opts(opts, "pmacs.fs.walk_tree", WALK_TREE_OPTS)
local id = async_mod._dispatch_fs_walk_tree(base, key)
return build_handle(id)
end
function fs.stat(path, opts)
if type(path) ~= "string" then
error("pmacs.fs.stat: path must be a string, got " .. type(path))

491
builtin/runtime/lsp.lua
View File

@ -1910,22 +1910,33 @@ local function repull_for_attachments(sid, request_fn)
end
end
-- T M4.5 — workspace file watching (workspace/didChangeWatchedFiles).
-- T M4.5 / issue #233 D3 — workspace file watching
-- (workspace/didChangeWatchedFiles).
--
-- Servers register watchers dynamically via client/registerCapability.
-- pmacs has no kernel file-watch, so each registration runs a polling
-- snapshot-diff coroutine: walk the base dir into a { relpath = sig }
-- map and, every tick, diff against the previous map to emit per-file
-- created/changed/deleted FileEvents (filtered by the glob and the
-- WatchKind bitmask), batched into one notification. Coarser than an
-- inotify bridge but accurate; a watcher self-cancels when the server
-- dies or the capability is unregistered.
-- pmacs has no kernel file-watch, so watching is a polling
-- snapshot-diff — but scheduled, not slept (D3 framing, approved
-- 2026-08-11): one SCAN GROUP per (server, base) owns a retained
-- { relpath = sig } snapshot, and a single `process.after-tick`
-- subscription drives every group's cadence off
-- `pmacs.editor.monotonic_ms` (autosave's Q#AS2 idiom). Waiting
-- allocates no job and holds no pool thread; a due group runs ONE
-- `pmacs.fs.walk_tree` job for the whole tree, diffs in Lua, and
-- routes per-file created/changed/deleted FileEvents through each
-- member watcher's glob and WatchKind mask, deduped into one
-- notification. Quiet scans back the interval off to a cap; any
-- change resets it. Groups retire when their last member leaves or
-- their server dies, cancelling an in-flight walk cooperatively.
local FILE_WATCH_INTERVAL_MS = 250
local FILE_WATCH_BACKOFF_CAP_MS = 4000
-- file_watchers[tostring(sid)][registrationId] = list of watch records
-- ({ cancelled = bool, form = "relative"|"absolute", _sleep = handle? }),
-- one per glob watcher.
-- ({ cancelled, form = "relative"|"absolute", kind_mask, match_subject,
-- baseline_epoch, group }), one per glob watcher. `baseline_epoch`
-- is nil until the first snapshot whose WALK STARTED after the record
-- joined its group — the registration-epoch rule that keeps a joiner
-- from receiving events for files that predate it.
local file_watchers = {}
-- WatchKind is a bitmask (Create=1, Change=2, Delete=4); test it
@ -2029,108 +2040,328 @@ local function glob_matcher(glob)
end
end
-- Recursively list files under `base` → { relpath = sig }. `sig`
-- folds size+mtime+kind so a content/metadata change flips it.
-- Symlinks are recorded, not traversed (loop-safe). Awaits fs
-- primitives, so call from inside an async coroutine.
local function scan_tree(base, matches)
local out = {}
local function walk(dir, rel_prefix)
local ok, entries = pcall(function()
return pmacs.fs.read_dir(dir):await()
end)
if not ok or not entries then return end
for _, e in ipairs(entries) do
local rel = (rel_prefix == "") and e.name or (rel_prefix .. "/" .. e.name)
if e.kind == "dir" then
walk(dir .. "/" .. e.name, rel)
elseif matches(rel) then
out[rel] = table.concat({
tostring(e.size), tostring(e.mtime),
tostring(e.mtime_nsec), tostring(e.kind),
}, "|")
end
end
end
walk(base, "")
return out
end
local FC_CREATED, FC_CHANGED, FC_DELETED = 1, 2, 3
local function start_file_watcher(sid, base, glob, kind_mask, record)
-- Per LSP, a plain-string glob matches the file's ABSOLUTE path,
-- while a RelativePattern's pattern is relative to its base — the
-- record's `form` (from resolve_watcher) picks the match subject.
-- scan_tree always walks in relative terms; only the string handed
-- to the matcher changes.
local match_glob = glob_matcher(glob)
local matches = match_glob
if record.form == "absolute" then
matches = function(rel)
return match_glob(base .. "/" .. rel)
-- Join a base-relative path under its base. The filesystem root is
-- special-cased the way `dired`'s handler already spells it (the
-- `(dir == "/") and "" or dir` idiom): a naive `base .. "/" .. rel`
-- at `/` yields `//path` — the implementation-defined POSIX spelling
-- — and `file:////path` once a URI wraps it.
local function join_under(base, rel)
if base == "/" then return "/" .. rel end
return base .. "/" .. rel
end
-- Build a record's match predicate over a base-RELATIVE path. Per LSP
-- (and #234's P1 review), a plain-string glob matches the file's
-- ABSOLUTE path while a RelativePattern's pattern is relative to its
-- base — the record's `form` picks the subject; the walk itself is
-- always relative.
local function make_matcher(base, pat, form)
local match_glob = glob_matcher(pat)
if form == "absolute" then
return function(rel)
return match_glob(join_under(base, rel))
end
end
pmacs.async(function()
local prev = scan_tree(base, matches)
while not record.cancelled and server_is_live(sid) do
local sh = pmacs.workers.sleep(FILE_WATCH_INTERVAL_MS)
record._sleep = sh
pcall(function() sh:await() end)
record._sleep = nil
if record.cancelled or not server_is_live(sid) then break end
return match_glob
end
local cur = scan_tree(base, matches)
-- The seam that makes the recheck below WITNESSABLE. `scan_tree`
-- suspends on `read_dir` once per directory, and the race is a
-- cancel arriving during one of those suspensions --- which no
-- arrangement of real timing can be made to happen on demand.
-- Same reason `git.lua` exposes `_deliver_status`: the contract is
-- about an interleaving the caller does not choose. Unset in
-- production, so this costs one nil test per tick.
-- `cur` is handed over so a test can cancel on THE SCAN THAT
-- OBSERVED a given change. Cancelling on any other scan is not a
-- witness: the loop would break at the post-sleep check on the
-- next iteration and emit nothing anyway, so the assertion would
-- pass with the recheck below deleted.
if pmacs.lsp._after_scan_for_tests then
pcall(pmacs.lsp._after_scan_for_tests, record, cur)
-- The event URI for one change, as `finish_group_scan` emits it. A
-- named function rather than an inline concat so the root-boundary
-- witness can drive the PRODUCTION construction at base "/" — a base
-- no fixture can walk for real.
local function watch_change_uri(base, rel)
return file_uri_for(join_under(base, rel))
end
-- Exposed for the root-boundary witness (the `_deliver_status`
-- pattern): the exact functions the watcher matches subjects and
-- builds URIs with. Test-only by convention; production never reads
-- them back.
pmacs.lsp._watch_matcher_for_tests = make_matcher
pmacs.lsp._watch_change_uri_for_tests = watch_change_uri
-- scan_groups[skey .. "\0" .. base] = one scan group per
-- (server, base): the shared snapshot, the members it serves, and the
-- D3 state machine — single-flight (`in_flight`), completion-advanced
-- deadlines (`next_scan_at`), a scan `generation` that rejects stale
-- completions, `rescan_queued` for joins landing mid-walk, the
-- backoff `interval`, and the `failure_reported` dedup latch.
local scan_groups = {}
local function group_key(skey, base)
return skey .. "\0" .. base
end
-- Retire a group: forget it and cancel any in-flight walk
-- cooperatively. The walk's completion is then rejected by the
-- identity/generation check in `finish_group_scan`, so a retired
-- group's state can never be written again.
local function retire_group(group)
scan_groups[group.key] = nil
if group.in_flight and group.in_flight.handle then
pcall(function()
group.in_flight.handle:cancel()
end)
end
end
-- One scan's completion. `scan_members` is the membership captured at
-- scan START — a watcher joining mid-walk is not in it and waits for
-- its queued baseline scan. Three disjoint arms (framing round 3):
-- stale/retired, live non-success, success.
local function finish_group_scan(group, gen, scan_members, ok, result)
-- Stale/retired arm: the group was retired (or superseded under the
-- same key) while the walk was parked. Nothing here may touch a
-- successor's state — #234's P2 recheck, applied at group scope.
if scan_groups[group.key] ~= group then return end
if not group.in_flight or group.in_flight.generation ~= gen then return end
group.in_flight = nil
local now = pmacs.editor.monotonic_ms()
if not ok then
-- Live non-success arm: no snapshot, no epoch, no emit; the prior
-- snapshot and backoff interval survive. A queued join still gets
-- its immediate baseline attempt; otherwise reschedule normally.
-- `Handle:await()` raises { tag = "cancelled" } (R45) for the
-- intentional `workers.cancel-at-point` outcome, which stays
-- quiet; a failure is reported once per distinct message until a
-- success clears the latch.
local tag = type(result) == "table" and result.tag or nil
if tag ~= "cancelled" then
local msg = type(result) == "table"
and tostring(result.message or result.tag)
or tostring(result)
if group.failure_reported ~= msg then
group.failure_reported = msg
local report = "lsp: file watch scan failed for " .. group.base .. ": " .. msg
if pmacs.error then pcall(pmacs.error, report) end
pcall(pmacs.editor.set_status, report)
end
-- RECHECKED AFTER THE SCAN, not only after the sleep (review P2).
-- The coroutine is suspended for most of a tick with `_sleep`
-- already cleared, so a cancel landing there sets `cancelled` and
-- has no sleep to interrupt. Without this line the resumed scan
-- runs on to `did_change_watched_files` below and a SUPERSEDED
-- watcher emits one last batch under its OLD pattern. One batch is
-- enough: it is a wrong-pattern notification the server acts on.
if record.cancelled or not server_is_live(sid) then break end
local changes = {}
for rel, sig in pairs(cur) do
local was = prev[rel]
if was == nil then
if kind_has(kind_mask, 1) then
changes[#changes + 1] =
{ uri = file_uri_for(base .. "/" .. rel), type = FC_CREATED }
end
if group.rescan_queued then
group.rescan_queued = false
group.next_scan_at = now
else
group.next_scan_at = now + group.interval
end
return
end
-- Success arm.
group.failure_reported = nil
local cur = {}
for _, e in ipairs(result) do
if e.kind ~= "dir" then
cur[e.name] = table.concat({
tostring(e.size), tostring(e.mtime),
tostring(e.mtime_nsec), tostring(e.kind),
}, "|")
end
end
-- The seam that makes cancel-during-scan WITNESSABLE (#234's P2
-- device, per member): the race is a cancel landing while the walk
-- job is out, which no arrangement of real timing produces on
-- demand. Unset in production. Handed the snapshot so a test can
-- cancel on THE SCAN THAT OBSERVED a given change; the delivery
-- loop below rechecks `cancelled` per member.
if pmacs.lsp._after_scan_for_tests then
for _, m in ipairs(scan_members) do
pcall(pmacs.lsp._after_scan_for_tests, m, cur)
end
end
local changes = {}
local prev = group.snapshot
if prev then
-- Diff once; route per member. `bit` is the WatchKind the event
-- needs (Create=1, Change=2, Delete=4).
local diff = {}
for rel, sig in pairs(cur) do
local was = prev[rel]
if was == nil then
diff[#diff + 1] = { rel = rel, type = FC_CREATED, bit = 1 }
elseif was ~= sig then
diff[#diff + 1] = { rel = rel, type = FC_CHANGED, bit = 2 }
end
end
for rel in pairs(prev) do
if cur[rel] == nil then
diff[#diff + 1] = { rel = rel, type = FC_DELETED, bit = 4 }
end
end
-- A member delivers only with a baseline (the registration-epoch
-- rule: no events for files that predate the join) and only while
-- uncancelled. Changes are deduped by (path, type) into the
-- server's single notification.
local emitted = {}
for _, m in ipairs(scan_members) do
if not m.cancelled and m.baseline_epoch ~= nil then
for _, d in ipairs(diff) do
if kind_has(m.kind_mask, d.bit) and m.match_subject(d.rel) then
local key = d.type .. " " .. d.rel
if not emitted[key] then
emitted[key] = true
changes[#changes + 1] = {
uri = watch_change_uri(group.base, d.rel),
type = d.type,
}
end
end
elseif was ~= sig and kind_has(kind_mask, 2) then
changes[#changes + 1] =
{ uri = file_uri_for(base .. "/" .. rel), type = FC_CHANGED }
end
end
for rel in pairs(prev) do
if cur[rel] == nil and kind_has(kind_mask, 4) then
changes[#changes + 1] =
{ uri = file_uri_for(base .. "/" .. rel), type = FC_DELETED }
end
end
end
group.snapshot = cur
group.epoch = group.epoch + 1
-- Baseline assignment: exactly the members captured at scan start
-- with no baseline yet — for them this walk is the first one that
-- STARTED after their join.
for _, m in ipairs(scan_members) do
if not m.cancelled and m.baseline_epoch == nil then
m.baseline_epoch = group.epoch
end
end
if #changes > 0 then
pcall(pmacs.lsp.did_change_watched_files, group.sid, changes)
group.interval = FILE_WATCH_INTERVAL_MS
else
-- A quiet scan backs off toward the cap; waiting costs nothing
-- under the after-tick cadence, so this bounds scan frequency,
-- not sleep-job length (there are no sleeps).
group.interval = math.min(group.interval * 2, FILE_WATCH_BACKOFF_CAP_MS)
end
-- Sweep members cancelled outside cancel_watch_records (the seam,
-- or a supersede that raced the walk); an empty group retires.
local live = {}
for _, m in ipairs(group.members) do
if not m.cancelled then live[#live + 1] = m end
end
group.members = live
if #group.members == 0 then
retire_group(group)
return
end
if group.rescan_queued then
group.rescan_queued = false
group.next_scan_at = now
else
group.next_scan_at = now + group.interval
end
end
-- Start one walk for `group`. Single-flight is the caller's contract
-- (the tick skips in-flight groups; joins queue instead) — this only
-- stamps the generation and captures the delivery membership.
local function start_group_scan(group)
group.generation = group.generation + 1
local gen = group.generation
local scan_members = {}
for i, m in ipairs(group.members) do
scan_members[i] = m
end
local handle = pmacs.fs.walk_tree(group.base)
group.in_flight = {
generation = gen,
started_at = pmacs.editor.monotonic_ms(),
handle = handle,
}
pmacs.async(function()
local ok, result = pcall(function()
return handle:await()
end)
finish_group_scan(group, gen, scan_members, ok, result)
end)
end
-- The cadence (D3 framing; autosave's Q#AS2 idiom). One after-tick
-- subscription drives every group: a clock read and a compare per
-- frame, no job and no pool thread while waiting. Installed once and
-- guarded rather than removed, because `pmacs.hook.remove` does not
-- exist (the P3 gap).
local watch_tick_installed = false
local function ensure_watch_tick()
if watch_tick_installed then return end
watch_tick_installed = true
pmacs.hook.add("process.after-tick", function()
if next(scan_groups) == nil then return end
local now = pmacs.editor.monotonic_ms()
for _, group in pairs(scan_groups) do
if not server_is_live(group.sid) then
retire_group(group)
elseif not group.in_flight and now >= group.next_scan_at then
start_group_scan(group)
end
if #changes > 0 then
pcall(pmacs.lsp.did_change_watched_files, sid, changes)
end
prev = cur
end
end)
end
-- Join `record` to its (server, base) group, creating the group on
-- first use. Joins WAKE the group (framing round 2): the deadline
-- pulls to now, or — mid-walk — exactly one immediate follow-up scan
-- is queued, so a joiner's baseline is at most the current walk's
-- remainder plus one walk away, never a backoff cap. The
-- join-triggered scan does not reset the backoff curve; only observed
-- changes do.
local function join_group(sid, skey, base, record)
ensure_watch_tick()
local key = group_key(skey, base)
local group = scan_groups[key]
if not group then
group = {
key = key,
sid = sid,
skey = skey,
base = base,
members = {},
snapshot = nil,
epoch = 0,
generation = 0,
in_flight = nil,
rescan_queued = false,
next_scan_at = 0,
interval = FILE_WATCH_INTERVAL_MS,
failure_reported = nil,
}
scan_groups[key] = group
end
group.members[#group.members + 1] = record
record.group = group
if group.in_flight then
group.rescan_queued = true
else
group.next_scan_at = 0
end
end
-- The workspace directory the registering server actually serves
-- (Q#D3-3): its spec `root_uri` — verbatim, nil when the server never
-- asked for a root — then its `cwd`. Configured roots and custom
-- resolvers (texlab's Q#LX2) are already folded into the spec, which
-- is why this must NOT be a fresh `pmacs.project.detect`.
local function server_workspace_dir(sid)
local skey = tostring(sid)
local ok, rows = pcall(pmacs.lsp.list)
if not ok then return nil end
for _, row in ipairs(rows) do
if tostring(row.id) == skey then
if row.root_uri then
local p = pmacs.lsp.path_for_uri(row.root_uri)
if p then return p end
end
if row.cwd then return row.cwd end
return nil
end
end
return nil
end
-- Resolve a GlobPattern (string | { baseUri, pattern }) to
-- (base_dir, pattern, form). The form must travel with the pair: a
-- RelativePattern's pattern is relative to its baseUri, and dropping
@ -2152,23 +2383,52 @@ local function resolve_watcher(sid, gp)
return pmacs.lsp.path_for_uri(gp.baseUri), gp.pattern or "**", "relative"
end
if type(gp) == "string" then
local form = (gp:sub(1, 1) == "/") and "absolute" or "relative"
-- Q#D3-3: the server's own workspace first. The
-- attachment-directory guess remains only for a server with
-- neither root_uri nor cwd — and even there it must be
-- DETERMINISTIC (review blocker): `pairs` order is hash order, so
-- the fallback takes the lexicographically smallest attachment
-- directory rather than whichever record iteration yields first.
-- REACHABLE, not defensive (review round 2 corrected round 1's
-- "unreachable" claim here): `pmacs.lsp.spawn` accepts a spec
-- with neither field, and for a MARKERLESS file `ensure_server`
-- adopts such a live server because its `root_uri` and the
-- attach's `key_uri` are both nil — after which any number of
-- buffers in different directories can attach to it.
local dir = server_workspace_dir(sid)
if dir then return dir, gp, form end
local fallback = nil
for _, rec in pairs(attachments) do
if rec.server == sid and rec.uri then
local p = pmacs.lsp.path_for_uri(rec.uri)
local dir = p and p:match("^(.*)/[^/]*$")
if dir then
return dir, gp, (gp:sub(1, 1) == "/") and "absolute" or "relative"
end
local d = p and p:match("^(.*)/[^/]*$")
-- A file at the filesystem root leaves the capture empty.
if d == "" then d = "/" end
if d and (fallback == nil or d < fallback) then fallback = d end
end
end
if fallback then return fallback, gp, form end
end
return nil, nil, nil
end
-- Cancel records and leave their groups; a group losing its last
-- member retires (its in-flight walk cancelled cooperatively).
local function cancel_watch_records(recs)
for _, r in ipairs(recs or {}) do
r.cancelled = true
if r._sleep then pcall(function() r._sleep:cancel() end) end
local group = r.group
if group then
r.group = nil
for i, m in ipairs(group.members) do
if m == r then
table.remove(group.members, i)
break
end
end
if #group.members == 0 then retire_group(group) end
end
end
end
@ -2177,20 +2437,29 @@ local function register_file_watchers(sid, registrations)
file_watchers[skey] = file_watchers[skey] or {}
for _, reg in ipairs(registrations or {}) do
if reg.method == "workspace/didChangeWatchedFiles" then
-- Re-registering a live id supersedes it (rust-analyzer does
-- this): cancel the outgoing records first, because the table
-- write below drops the only reference to them and an
-- uncancelled record polls until the server dies.
cancel_watch_records(file_watchers[skey][reg.id])
local outgoing = file_watchers[skey][reg.id]
local recs = {}
for _, w in ipairs((reg.registerOptions or {}).watchers or {}) do
local base, pat, form = resolve_watcher(sid, w.globPattern)
if base and pat then
local r = { cancelled = false, form = form }
local r = {
cancelled = false,
form = form,
kind_mask = w.kind or 7,
match_subject = make_matcher(base, pat, form),
baseline_epoch = nil,
}
recs[#recs + 1] = r
start_file_watcher(sid, base, pat, w.kind or 7, r)
join_group(sid, skey, base, r)
end
end
-- Re-registering a live id supersedes it (rust-analyzer does
-- this, #234's D2). The successors joined FIRST, so a same-base
-- group stays alive across the hand-over — its in-flight walk
-- is not torn down, and the joiners already queued their
-- baseline. An id whose successors watch a different base still
-- retires the old group when its last member leaves here.
cancel_watch_records(outgoing)
file_watchers[skey][reg.id] = recs
end
end
@ -2225,7 +2494,7 @@ end
-- family for every attached document so the matching store
-- (`pmacs.inlay_hint` / `pmacs.semantic_tokens`) stays fresh.
-- * `client/registerCapability` / `client/unregisterCapability` —
-- start/stop the file-watch coroutines for any
-- join/leave the file-watch scan groups for any
-- `workspace/didChangeWatchedFiles` registration; reply `null`.
--
-- Only servers in `attachments` are drained, so a test (or package)

112
docs/active-work.md
View File

@ -247,8 +247,116 @@ their bite results. Durable facts are absorbed in
unmerged until this was resolved; #234 merged first and #227 followed
the same day (`b867f64`), refreshed and re-gated on the merged base.
**D3 — the polling cost — is the remainder, and the user has ruled it
is next (2026-08-11).** No branch and no framing yet. What is known,
**D3 — the polling cost — PR #235 OPEN**
(https://github.com/levineuwirth/pmacs/pull/235, opened 2026-08-11 at
`db24abb`, the implementation commit after two pre-commit review
rounds). **Branch `lsp-file-watch-d3`** (base
`githubsucks/main` @ `add0ba1`; the remote ref is authoritative), with
**framing `docs/lsp-file-watch-d3-framing.md`, revision 4, APPROVED
2026-08-11 with the four rulings adopted as proposed** (honest ⋯N bar;
no exclusions; server root then cwd then attachment fallback;
constants). **IMPLEMENTED on the branch**: `pmacs.fs.walk_tree` (one
cancellable job per scan, eight Rust unit tests), the group scheduler
in `lsp.lua` (after-tick cadence, single-flight state machine with
the round-3 non-success partition, registration epochs, backoff,
retirement), and eighteen acceptance tests — the six #234 tests
byte-unchanged plus twelve witnesses, each mutation-verified.
Implementation-time facts and review-round records worth keeping:
- **Retirement is deliberately double-enforced** (the unregister path
and the post-scan sweep), and the mutation pass proved it: biting
either copy alone is masked by the other; only biting both goes red.
The sweep exists for seam-cancelled members, the unregister path for
idle groups whose next scan may be seconds away.
- **Round three, post-PR: two witness overclaims, and the PR's first
CI red — my own fixed-duration pump.** The mid-walk bound (60) could
not tell a deleted per-entry poll from the real code — the cancel
lands two files into a 41-file directory, so the per-DIRECTORY poll
stops a poll-less walk at 44; the bound is now 40 against an
expected exactly-35, and the entry-poll-only bite goes red at 44.
The retirement helper accepted `cancel_requested` on an active row —
a request, not settlement; it now waits for a `cancelled`
COMPLETION. And all five CI test legs failed deterministically where
sixteen local cores stayed green: `d3_pump(1600)` wrote the
discriminating file before the held walk even STARTED on a
3-thread pool (8×1200 ms sleeps drain in ~3.6 s of waves), folding
it into the baseline. The drain is now an observable condition
(a post-join walk completed and none active), the sleeps are 800 ms,
and the three saturation tests plus the whole family were re-run
green under `taskset -c 0-3` — the CI pool shape, reproduced
locally. **A fixed-duration pump against pool-dependent timing is a
core-count assumption in disguise.**
- **A second pre-commit round found three more** (implementation
review, not framing): mid-walk cancellation was unwitnessed — both
Rust cancel tests pre-cancelled and the acceptance test cancelled a
QUEUED walk, so deleting the internal polls left everything green
(a `cfg(test)` entry hook now cancels at an exact entry boundary
and asserts the walk stopped NEAR it, and the retirement witness
holds a walk in flight across the unregister and asserts the job
settles cancelled); the "unreachable fallback" claim was WRONG — a
manual `pmacs.lsp.spawn` may omit both `cwd` and `root_uri`, and
`ensure_server` adopts such a server for markerless files (nil ==
nil), so the deterministic minimum now has a five-directory
through-the-server witness (five, because with two the build's hash
order coincided with the lexicographic answer and the first-pairs
bite survived); and the root-boundary joins gained a witness
through the exported production matcher/URI functions, since no
fixture can walk `/` for real.
- **A pre-commit review round found four blockers**, fixed before
anything was committed: empty-tree cancellation (the entry loops
never run, so a pre-cancelled walk returned empty SUCCESS — the
deletion-storm shape the non-success arm exists to prevent); the
attachment fallback was still `pairs`-order nondeterministic (now
lexicographic-minimum; this round also called it unreachable, which
round two above DISPROVED via the manual-spawn adoption path); a
filesystem-root base joined as `//path` (now `join_under`, dired's
idiom); and two test probes defaulted on error, so a broken pair
could compare equal and lie green (every probe now `expect`s).
- **A stray marker high in the tree re-roots every markerless fixture
under it.** An empty `/tmp/.git` (leftover from the #233
investigation, since removed by the user) made project detection
root tempdir fixtures at `/tmp`, which under Q#D3-3 the watcher then
faithfully watched. Any machine can grow one; a markerless-fixture
red that looks like a watcher bug may be an ancestor marker.
Framing and lane were committed
at the branch's first commit so the document stayed portable during
review. **Review
round 3 (2026-08-11) found the live group's non-success transition
missing**: every job is user-cancellable and `Handle:await()` raises on
cancel/failure, so an uncaught result could leave `in_flight` set forever.
Revision 4 partitions completion into success, stale/retired, and live
non-success: live cancel/failure commits no snapshot or epoch, preserves
the prior snapshot/backoff, clears in-flight, honors a queued baseline or
reschedules, and visibly deduplicates failures. It also corrects the
queued-baseline bound: a mid-walk join waits for the current walk's
remainder plus its own follow-up walk, never a backoff cap. Two round-3
witnesses cover live cancel and failure. **Review round 2 (2026-08-11)
found the scheduler underspecified**: a joining watcher must force an
immediate baseline scan (a backed-off group would otherwise fold
post-registration files into the baseline — and a baseline is now
only a snapshot whose WALK STARTED after the join); the group gained
a defined state machine (single-flight per group, deadlines advanced
from completion, stale completions rejected by generation, retirement
that cooperatively cancels the walk — cancellation joining
`walk_tree`'s contract); and Q#D3-1's `⋯1` was an overclaim — the
accurate bar is absence at idle plus one attributable job per
concurrently due (server, base) group. Six round-2 witnesses joined
the plan. **Review round 1 (2026-08-11) found five
findings and revision 1 did not survive it** — the promised idle
state was impossible (`workers.sleep` is a pool-thread-holding
running job the indicator counts; revision 2 replaces the sleep loop
with autosave's Q#AS2 after-tick cadence), the scan root must be the
server's own `root_uri`/`cwd` (not `pmacs.project.detect`, which
texlab's Q#LX2 proves wrong), coalescing gained registration-epoch
delivery semantics with two new witnesses, the exclusion default
became **none** (any unconditional skip deviates from the registered
glob contract, and `walk_tree` removes the job-count economics that
motivated it), and the cost arithmetic was corrected to this
checkout (1,326 jobs/tick for rust-analyzer's six watchers; revised
steady state is zero jobs at idle). The four rulings (Q#D3-1..4)
were ADOPTED AS PROPOSED at the 2026-08-11 approval and are recorded
in the framing's status block. What was known before framing,
verified while framing D1/D2:
- After #234 the watcher is *correct* but still walks: `walk` recurses

17
docs/ci-red-signatures.md
View File

@ -570,6 +570,23 @@ the lane's only `pmacs-gpu` addition is the arm that went red.
The next agent to touch this row should reproduce at 1-in-10 and
instrument which side closes the pipe, rather than re-running for green.
**Fourth occurrence — D3 file-watch scheduler (PR #235), 2026-08-11,
local (Linux), at the gate's SWEEP step** (`cargo test --workspace
--no-fail-fast`, default features — U3's flavor, this time with the
fragments captured). All three required fragments verified against the
durable gate log
(`pmacs-fdccc423/gate-logs/20260811T150651Z-1481359/08-sweep.log`):
`transient sequence must attach: Attach(Handshake(Io(Os { code: 32,
kind: BrokenPipe, message: "Broken pipe" })))`, `attach.rs:1680`.
242/243 in the target; the same sweep had passed twice earlier the same
day on materially the same tree (the diff between runs was a test file
and docs — **no `pmacs-gpu` code, no wire, no protocol**, the
strongest non-attribution shape this row has had). Ambient context,
recorded not asserted: load average ~5.2 and four leaked
`pmacs --daemon` processes resident. Consistent with the established
~1-in-10-under-load rate; adds no new mechanism evidence. The
retirement bar is unchanged.
### U2 — `m6_1_pty_raw_mode_disables_kernel_echo`, THIRD known occurrence
**Corrected 2026-08-09 after review.** A previous edit of this row

474
docs/lsp-file-watch-d3-framing.md Normal file
View File

@ -0,0 +1,474 @@
# LSP file watcher D3 — the polling cost — framing
**Status: revision 4 — APPROVED 2026-08-11.** The user's own review
pass (round 3, absorbed below) closed the state machine; on the
soundness confirmation the four rulings were adopted as proposed:
**Q#D3-1** the honest bar — absent at idle, one attributable job per
concurrently due group; **Q#D3-2** no exclusions by default;
**Q#D3-3** server `root_uri` → `cwd` → attachment fallback;
**Q#D3-4** constants, no config keys.
Continues issue #233, which stays open until this lane closes it. D1
and D2 — matching correctness and the re-registration leak — merged as
**#234** (`ae84d58`); this frames the remainder the user ruled next on
2026-08-11: the watcher is now *correct* and still walks everything,
every tick, forever.
## Review round 1 — five findings, and what each changed
Revision 1 was reviewed 2026-08-11 and did not survive it. Recorded
here because three of the five are cases where the framing reasoned
from the wrong mechanism, which is exactly what a framing review
exists to catch before code does.
- **P1 — the promised idle state was impossible.** Revision 1
promised "one brief `walk_tree` blip every four seconds"; but
`pmacs.workers.sleep` allocates a **running job for its whole
duration** (`dispatch_sleep`, `src/async_runtime.rs:1022-1037` —
the job sleeps in 1 ms slices on a **pool thread**), and
`activity_summary` counts every `Running` job
(`src/async_runtime.rs:1570`). A 4 s backoff sleep would render as a
*constant* `⋯1 sleep 4000ms`, and filtering sleeps from the
indicator would touch the instrument this lane declares out of
scope. **Revision 2 removed the sleep from the design entirely**
(see "The cadence" below) — the fix is the codebase's own idiom,
not a new mechanism.
- **P1 — the scan root must be the server's, not a freshly detected
one.** Revision 1 proposed `pmacs.project.detect` for the
string-form base. Server rooting honours **configured strings and
custom resolvers first** (`project_root_for`,
`builtin/runtime/lsp.lua:783`), and the bundled texlab entry
documents why its root *cannot* be `project.detect` (Q#LX2,
`lsp.lua:284` — texlab wants the document root, not the repository
root). The server's own `root_uri` and `cwd` are already exposed
(`pmacs.lsp.list`, `src/lua_bindings/mod.rs:10985-11002`;
`root_uri` is the spec field verbatim, nil when the server never
asked for a root). **Revision 2 rooted the scan at what the
registering server actually serves**: `root_uri` → `cwd` →
attached-file directory, in that order.
- **P1 — coalescing needs registration-epoch semantics.** Revision 1
said "route the shared diff through every watcher" without defining
which watcher set owns a diff. A watcher registered between two
snapshots would receive a false CREATED for a file that predates
its registration; membership changing during a walk recreates
either #234's stale-watcher batch or the same pre-registration
event. **Revision 2 defined shared snapshots with per-watcher
baselines**, refined by round 2 below.
- **P1 — "VCS-only exclusion is safest" was wrong.** A hard skip
silently ignores a server that legitimately registers `.git/HEAD`
or `**/.git/**`, and glob semantics mean even `**/*.rs` *can* match
under `.git/` — so any unconditional exclusion is a deviation from
the registered contract, not a safe default. Revision 1 also
overweighted the win: exclusion was a **job-count** lever when every
directory was a separate job, and the walk primitive removes that
economics. **Revision 2 defaults to no unconditional exclusion.**
- **P2 — the arithmetic used the issue's machine, not this one.**
With D = 220 on this checkout it is 220 `read_dir` jobs **plus one
sleep** per watcher per tick — 221, or **1,326 across
rust-analyzer's six watchers** — and revision 1's proposed steady
state was itself two jobs (sleep + walk), not one. Corrected
throughout.
## Review round 2 — the scheduler was underspecified
Round 2 (2026-08-11) closed the round-1 findings and found the
cadence's own semantics missing: revision 2 said *when* scans become
due but not what happens when due-ness, in-flight walks, joins, and
retirement collide.
- **P1 — a joining watcher must force an immediate baseline scan.**
Revision 2's baseline was "the first snapshot completed after
join" — but a backed-off group's next snapshot can be 4 s away, so
a file created after registration and before that delayed snapshot
would fold into the baseline and never be reported. Today,
registration begins its initial scan immediately (`lsp.lua:2074`);
the coalesced design must preserve that. **A join pulls the group's
`next_scan_at` to now; if a walk is already in flight, exactly one
immediate follow-up scan is queued.** The baseline is sharpened to
match: a snapshot serves as a watcher's baseline only if its **walk
started after the join** — an in-flight walk may have passed a
directory before a pre-join file appeared there, and using its
snapshot as a baseline would turn that file into a false CREATED on
the next diff.
- **P1 — single-flight and retirement were undefined.** The document
itself establishes that walks can outlive their interval, so
"every due group starts a scan" permits overlapping walks —
restoring multiple jobs, completing snapshots out of order, and
making the epoch state ambiguous. **The group scheduler below is
now a defined state machine**: one in-flight scan per group,
deadlines advanced from completion, stale completions rejected,
and retirement (last member gone, or server death) that
cooperatively cancels the walk — which puts cancellation into
`walk_tree`'s contract and tests.
- **P2 — `⋯1` was an overclaim.** Groups are keyed by
(server, base), so several can be due on the same frame, and
per-group single-flight still permits `⋯N`. **The bar is restated
accurately**: absence at idle; while scans run, one attributable
job per concurrently due group. Global serialization is offered as
the alternative under Q#D3-1 if `⋯1` must be guaranteed.
## Review round 3 — live non-success closes the state machine
Round 3 (2026-08-11) accepted all three round-2 corrections and found
one transition still absent: revision 3 specified successful and stale
scan completions, but not a cancellation or failure while the group
itself remains live.
- **P1 — a live cancelled/failed walk needs a terminal transition.**
Every job is user-cancellable from `*workers*`, and `Handle:await()`
raises structured `cancelled` and `failed` outcomes. Without a caught
non-success path, the group can retain `in_flight = true` forever and
silently stop watching. **Revision 4 partitions completion into
success, stale/retired, and live non-success** (below). A live
cancellation or failure commits no snapshot or epoch, emits nothing,
preserves the previous snapshot and backoff curve, clears in-flight,
and either serves a queued join immediately or schedules the next
attempt normally. Failures are reported visibly and deduplicated per
group; cancellation is an intentional user/runtime outcome and stays
quiet.
- **P2 — the queued-baseline latency bound was one walk short.** A
join just after a walk starts waits for the remainder of that walk
*and* its own follow-up walk before the baseline snapshot completes.
Revision 4 states the two boundaries separately: the baseline walk
starts when the current walk completes, and its snapshot completes
after that follow-up walk — never after the backoff cap.
## Verified against the tree at `add0ba1`
Every claim below was read or measured this session (revisions 2–4
re-verified their corrections against the code).
- Each registered watcher is its own coroutine looping
`sleep(FILE_WATCH_INTERVAL_MS)` → `scan_tree` (`lsp.lua:1924`,
`:2074-2083`); the interval is 250 ms. Registration's initial scan
runs immediately (`:2074-2075`).
- `scan_tree` awaits `pmacs.fs.read_dir` once **per directory**
(`:2038-2041`), one async job each. `walk` recurses
unconditionally; `matches` gates only whether an entry is
*recorded*.
- **A sleep is a pool-occupying running job.** `dispatch_sleep`
dispatches `run_sleep` onto the worker pool
(`async_runtime.rs:1022-1037`), so every sleeping watcher holds one
of the pool's `available_parallelism - 1` threads for the full
interval — the hazard `autosave.lua:133-139` documents in so many
words. Twelve pre-#234 rust-analyzer watchers were twelve
mostly-sleeping pool threads.
- **Jobs per tick per watcher = 1 sleep + D read_dirs.** On this
checkout D = 220, so 221 per watcher and **1,326 per tick for
rust-analyzer's six watchers**.
- **250 ms is a floor, not a period.** The awaits are sequential, so
a tree whose walk takes longer than the interval makes the
effective period scan-bound — the loop never idles. The issue
measured ~270 ms on a two-directory tree; D round trips dominate on
real trees.
- Measured on this checkout: **220 directories, of which `.git` is
177 — over 80 % of the walk**. (The issue's machine carried an
in-tree `target/`: 187 directories, 46 outside `.git`+`target`.
This machine exports an external `CARGO_TARGET_DIR` and still pays
`.git`.)
- **The string-form base is nondeterministic, found while framing:**
`resolve_watcher`'s string arm takes the directory of the FIRST
attachment `pairs()` happens to yield (`:2150-2173`) — table
order, not a chosen root. #234 made matching correct *per base*;
**which** base is still accidental.
- **The codebase already has a no-job cadence idiom with five
adopters.** `process.after-tick` fires every frame, including idle
frames (the run loops tick on a frame *timeout*), and
`pmacs.editor.monotonic_ms` is the clock built for exactly such
loops (`lua_bindings/mod.rs:13833`). `autosave.lua`'s Q#AS2 sweep
is the model: one clock read and one compare per frame, no job, no
pool thread.
- **`pmacs.hook.remove` does not exist** (the P3 prerequisite gap,
`docs/agent-handoff.md` §1a) — an after-tick subscription is
permanent, so the scheduler installs **once** and early-returns
when it owns no groups, exactly as autosave's does when disabled.
- **Cooperative cancellation is the established job shape**: every
job body in `async_runtime.rs` polls `cancel.is_cancelled()` at
its work boundaries (`run_sleep` per slice, the others per unit);
`walk_tree` polling between directory reads inherits the pattern.
- **No `notify`/inotify dependency in the tree** and **no
ignore-list infrastructure to reuse** — both re-verified, both
carried from the D1/D2 framing.
## What §9 asks of this lane
Not "quiet the modeline." The indicator is the instrument that found
this, and the churn it shows is real; quieting it is explicitly out
of bounds. The lane's job is to make the background work **small,
attributable, and honest**: at idle there should *be* no running
background work to report, and while scans run each should be one job
named for its root.
## The cadence — after-tick deadlines, not sleeps (round 1)
The per-watcher sleep loop is replaced by the Q#AS2 idiom: one
`process.after-tick` subscription owns every scan group's schedule.
Per frame it reads `pmacs.editor.monotonic_ms` once and compares each
group's `next_scan_at`. Waiting allocates **no job and no pool
thread** and renders **no indicator segment** — `activity_summary`
returns `None` at zero by contract. While a scan runs, the indicator
honestly shows its job.
The subscription installs once at module load and early-returns when
no groups exist — it cannot be removed, because `pmacs.hook.remove`
does not exist, and a guard is the house answer (autosave's
`enabled` check).
### The group state machine (rounds 2 and 3)
Per group — keyed (server, base) — the scheduler holds
`next_scan_at`, the backoff interval, an **in-flight record** (handle,
generation, and start time), a **scan generation counter**, a
`rescan_queued` bit, and the last reported walk failure for
deduplication.
- **Single-flight.** The after-tick check skips a group whose walk is
in flight; a group cannot become due against itself. Concurrent
walks, out-of-order snapshots, and ambiguous epochs are therefore
unrepresentable, not merely avoided.
- **Deadlines advance from completion.** On scan completion,
`next_scan_at = completion time + current interval`. A walk that
outlives its interval degrades to back-to-back scans with a full
interval between them — never to overlap.
- **Stale completions are rejected.** Each scan carries its group's
generation at start; a completion whose group is retired, or whose
generation is not the group's current one, is dropped before any
state write or emit — #234's P2 recheck, applied at group scope.
- **Current completions have three disjoint outcomes (round 3).** The
coroutine catches `Handle:await()` so its structured outcome cannot
bypass group cleanup. A successful current completion clears
in-flight, commits the snapshot and epoch, routes its diff, updates
the backoff curve, and clears the failure-dedup latch. A
**cancelled or failed completion while the group is still live**
also clears in-flight, but commits no snapshot or epoch, emits
nothing, and preserves the previous snapshot and backoff interval.
It consumes `rescan_queued` by starting exactly one immediate scan;
otherwise it sets `next_scan_at = completion time + current
interval`. Failure is surfaced through the existing LSP status/error
reporting shape once per distinct `(group, error)` until a success;
cancellation stays quiet because `workers.cancel-at-point` makes it
an intentional outcome. Retirement is the third arm: its cancelled
completion is stale by construction and reaches none of this live
state.
- **Joins wake the group.** A watcher joining sets
`next_scan_at = now`. If a walk is in flight, `rescan_queued` is
set instead, and completion of the current walk starts **exactly
one** immediate follow-up scan. The joiner's baseline is the first
snapshot whose **walk started after its join** (see below), so the
baseline walk starts as soon as the current walk completes and its
snapshot completes after that one follow-up walk — never a backoff
cap away. The join-triggered scan does **not** reset the backoff
curve; only observed changes do.
- **Retirement.** When the last member leaves (unregistration, or
supersession with no successor) or the server dies, the group
retires: the in-flight walk's job is **cancelled cooperatively**,
its completion is rejected by the generation rule, and the group's
schedule entry and snapshots are dropped. A re-registration that
replaces members keeps the group alive — the superseded members
are cancelled per #234's D2 and the new members join as above.
## The scan root (round 1)
For a string-form (bare `*.txt` / absolute) registration the base
becomes, in order: the server's **`root_uri`** (spec verbatim — nil
when the server never asked for a root), the server's **`cwd`**, and
only then the attached-file directory. These describe the workspace
the registering server actually serves — including configured roots
and custom resolvers like texlab's, which `pmacs.project.detect` can
never reproduce (Q#LX2). This replaces the `pairs()`-order accident
with a deterministic, server-owned answer. `RelativePattern`s keep
their own `baseUri`, unchanged.
## Coalescing, with registration epochs (rounds 1 and 2)
One scan group per (server, base). The group's scanner records
**all** files (the matcher moves from scan time to diff time); each
completed scan increments the group's **snapshot epoch**.
Delivery semantics:
- Each watcher's **baseline is the first snapshot whose walk started
after it joined** — an in-flight walk may have passed a directory
before a pre-join file appeared there, so its snapshot cannot serve
as a baseline (round 2). A watcher receives diffs only between
snapshots at or after its baseline. A file created after the
group's previous snapshot but before a watcher joined therefore
produces **no event for that watcher** — folded into its baseline,
exactly as the initial scan folds pre-existing files today.
- **Membership for delivery is captured at scan start**; a watcher
joining mid-walk waits for its queued baseline scan.
- **Cancellation is rechecked per watcher at emit time** — #234's P2
rule per member: a watcher superseded or unregistered during the
walk emits nothing, and its replacement has no baseline yet, so it
emits nothing either.
- Changes passing a watcher's matcher and kind mask are deduped by
`(uri, type)` into the server's single
`workspace/didChangeWatchedFiles` notification, as today.
## The walk primitive
`pmacs.fs.walk_tree(base)` — the whole recursive walk as **one job**
instead of one per directory: 220 `read_dir` jobs per scan on this
repo become 1. The indicator shows one purpose (`walk_tree <root>`).
An additive fs binding plus its async-runtime job; **no wire change**
(fs bindings are not the frontend protocol) and no new crate. Two
contract clauses, each with its own Rust tests:
- **Symlinks are recorded, not traversed** — `scan_tree`'s
loop-safety, preserved.
- **Cancellation is cooperative and prompt**: the job polls its
cancel token between directory reads (the established
`async_runtime.rs` job shape), so group retirement mid-walk stops
the walk instead of orphaning it.
## Exclusions (round 1) — none by default
Glob semantics make any unconditional skip a contract deviation:
`**/*.rs` compiles with a separator-spanning prefix, so it *can*
match under `.git/`, and a server may register `.git/HEAD` outright
(branch-watching tools do). The only semantics-preserving default is
**no unconditional exclusion**, and with the walk primitive the
economics support it: exclusion was worth 80 % of the *job count*
when every directory was a job; inside one `walk_tree` job it is only
readdir syscalls, and the whole 220-directory walk is a few
milliseconds of one pool thread per scan.
The option space, for Q#D3-2: (a) no unconditional exclusion — the
proposed default; (b) **opt-in** exclusion through configuration, for
users with pathological trees, framed explicitly as a
watcher-contract trade; (c) matcher-aware pruning — skip a subtree
only when *no* active watcher's pattern can match under it — sound
but almost never fires against real registrations, because
`**/`-leading globs can match anywhere; (d) a hard built-in VCS
skip, which revision 1 called "safest" and is not: it is (b) without
the opt-in.
## Idle backoff
The interval doubles while consecutive scans observe no change,
capped at 4 s; any change batch resets it to 250 ms. Under the
after-tick cadence a longer interval costs *nothing* while waiting —
backoff bounds **scan frequency**, not sleep-job length. Worst-case
latency for an external change at idle equals the cap **except at
registration, where the join rule forces an immediate baseline**
(round 2). LSP imposes no latency bound, and edits made through
pmacs never depended on the watcher (the server sees `didChange`).
The watcher exists for git checkouts, generated files, and other
editors.
## Deliberately staged separately — kernel notification
`notify` (inotify / FSEvents / kqueue) eliminates polling. Also: a
new dependency, a new Rust subsystem, a Lua binding, a platform
matrix, and an interaction with §9's ownership model. Staged as its
own framing — not because it is wrong but because everything above is
a pure win it does not obsolete (a kernel watcher still needs the
initial scan and a polling fallback), and a new-crate decision
deserves its own review.
## Proposed shape — Stage 1
After-tick cadence with the group state machine + walk primitive +
coalescing-with-epochs + backoff; no exclusions by default;
server-owned scan root.
At rest on this repo with rust-analyzer attached: **from 1,326 jobs
per scan-bound tick (six of them pool-thread-holding sleeps) to zero
jobs at idle**, with one `walk_tree` job per group for the few
milliseconds each scan actually runs — at most every 250 ms under
activity and every 4 s at rest, immediately once at registration.
## The rulings — adopted as proposed at approval (2026-08-11)
- **Q#D3-1 — the acceptance bar, stated accurately (round 2).** At
idle the indicator is **absent** (no running job exists —
`activity_summary`'s `None`-at-zero contract). While scans run it
shows **one attributable job per concurrently due group** — `⋯N`
when N (server, base) groups are due on the same frame, each named
for its root; a typical single-project session has one group.
**Adopted.** The alternative — a global scan queue guaranteeing
`⋯1` at the cost of coupling one server's scan latency to
another's tree size — was declined.
- **Q#D3-2 — exclusions.** **Adopted: none by default**, with opt-in
exclusion as a documented contract trade (option b) if a user ever
asks.
- **Q#D3-3 — the scan root.** **Adopted: server `root_uri` → server
`cwd` → attached-file directory** (the fallback itself made
deterministic in implementation review: lexicographically smallest
attachment directory). This widens the watched tree for servers
with a real root — a behavioural change to a path real servers
exercise, accepted as such.
- **Q#D3-4 — interval, cap, and backoff curve.** **Adopted:
constants until someone asks.** The D1/D2 framing refused a knob
for a defect; with D3 the cadence is a designed mechanism, so keys
would be defensible — but more registry surface is coherence cost
nobody has yet paid for a reason.
## Verification sketch
- **Idle witness:** with a server attached, watchers registered, and
no file activity, `activity_summary` settles to `None` (the absent
segment) between scans — the strongest form of the job-count
claim, and unwritable under the sleep design.
- **Scan-cost witness:** one scan allocates O(1) jobs, not
O(directories), on a tree with enough directories to discriminate.
- **Join-wakes witness (round 2):** with a group backed off at the
cap, register a new watcher — a scan starts immediately
(timestamps through the group seam); after that baseline settles, a
newly created file is reported to the joiner.
- **No-overlap witness (round 2):** with a walk deliberately held
in flight past its interval (through the group seam or by
withholding the completion pump), the group allocates **no second
walk job**; deadlines resume from completion.
- **Retirement witness (round 2):** the last member unregisters
mid-walk — the walk's job is cancelled, its completion is
rejected (no emit, no state write), and the group's schedule entry
is gone; a server death takes the same path.
- **Live-cancel witness (round 3):** cancel an in-flight walk while
members remain (the `workers.cancel-at-point` outcome) — no snapshot
or epoch commits, the previous snapshot remains, in-flight clears,
and the group scans again on schedule; repeat with a queued join and
its baseline scan starts immediately.
- **Live-failure witness (round 3):** force a walk failure through the
group completion seam — no snapshot or epoch commits, the prior
snapshot remains, the group retries, one distinct failure is
reported once, and a later success clears the dedup latch.
- **Queued-baseline witness (round 2):** a watcher joining mid-walk
gets exactly one immediate follow-up scan, and its baseline is
that scan, not the walk that was in flight at join.
- **Epoch witness (registration between snapshots):** create a file
after the group's snapshot, then register a second watcher, then
let a scan complete — the old watcher receives CREATED, the new
one receives **nothing** for that file, and does receive events
for files created after its baseline.
- **Backoff witness:** quiet scans lengthen the gap between scans
and one change resets it — observed through scan timestamps at the
seam, not through sleep purposes (there are none).
- **Root witness:** a server with a configured root watches that
root, not the attached file's directory; texlab's resolver shape
is the fixture model.
- **Contract preservation:** all six existing `m4_24` watcher tests
stay **byte-unchanged** and green.
- `walk_tree` Rust unit tests: symlinks recorded-not-traversed,
cooperative cancellation observed mid-walk, signature parity with
the Lua walk it replaces.
- Each new behaviour is mutation-tested against the defect it
guards.
## Coherence impact (§20)
- **Journey steps:** none added; step 5 is unchanged.
- **Interaction islands:** none.
- **Config registry:** none by default; Q#D3-2/Q#D3-4 could add keys
and are flagged as such.
- **Background-work attribution (§9):** at idle there is genuinely no
running background work, and the indicator's absence is then a true
statement rather than a filtered one; each scan is one job named
for its root. The ownership *model* remains §9 Stage 2's work. As a
side effect the watcher stops holding pool threads while waiting.
## Gates
`./scripts/gate --acceptance m4_acceptance`. No `--protocol` — no
wire change, no `PROTOCOL_VERSION` bump; `walk_tree` is an fs
binding, not a protocol message.

38
src/async_runtime.rs
View File

@ -317,6 +317,10 @@ pub enum JobKind {
Parse,
/// `dispatch_fs_read_dir` --- directory enumeration ([T M8.1]).
FsReadDir,
/// `dispatch_fs_walk_tree` --- one whole-tree enumeration as a
/// single job (issue #233 D3: the watcher's per-directory job
/// storm collapsed into one purpose per scan).
FsWalkTree,
/// `dispatch_fs_stat` --- single-path metadata ([T M8.1]).
FsStat,
/// `dispatch_fs_rename` --- atomic rename ([T M8.1]).
@ -355,6 +359,7 @@ impl JobKind {
JobKind::Grep => "grep",
JobKind::Parse => "parse",
JobKind::FsReadDir => "fs_read_dir",
JobKind::FsWalkTree => "fs_walk_tree",
JobKind::FsStat => "fs_stat",
JobKind::FsRename => "fs_rename",
JobKind::FsChmod => "fs_chmod",
@ -1185,6 +1190,29 @@ impl AsyncRuntime {
id
}
/// Dispatch a `walk_tree(base)` job: the whole recursive tree as
/// ONE job, entry names base-relative (issue #233 D3). The reply
/// reuses [`ReplyKind::ReadDir`] --- the payload shape is
/// identical and the Lua boundary needs no second conversion;
/// [`JobKind::FsWalkTree`] still separates the two for the
/// `*workers*` label and the purpose string. Cancellation and
/// tolerance are [`crate::fs::walk_tree_blocking`]'s contract.
pub fn dispatch_fs_walk_tree(&self, base: PathBuf, supersede: Option<&str>) -> JobId {
let (id, cancel) = self.allocate(JobSpec {
kind: JobKind::FsWalkTree,
supersede,
stream: None,
resource: None,
purpose: format!("walk_tree {}", base.display()),
});
let bus = self.workers.clone();
self.pool.dispatch(move |_pool| {
let kind = run_fs_walk_tree(&cancel, &base);
let _ = bus.send(ASYNC_REPLY_TOPIC, &WorkerReply { job_id: id, kind });
});
id
}
/// Dispatch a `stat(path)` job. Returns one [`FsDirEntry`] of
/// metadata for `path`. T M8.1.
pub fn dispatch_fs_stat(&self, path: PathBuf, supersede: Option<&str>) -> JobId {
@ -1759,6 +1787,16 @@ fn run_fs_read_dir(
}
}
fn run_fs_walk_tree(cancel: &CancellationToken, base: &Path) -> ReplyKind {
match crate::fs::walk_tree_blocking(base, cancel) {
Ok(listing) => ReplyKind::ReadDir(listing),
Err(FsError::Cancelled) => ReplyKind::Cancelled,
Err(e @ (FsError::Io { .. } | FsError::NonUtf8Path { .. })) => {
ReplyKind::Error(e.to_string())
}
}
}
fn run_fs_stat(cancel: &CancellationToken, path: &Path) -> ReplyKind {
match stat_blocking(path, cancel) {
Ok(entry) => ReplyKind::Stat(entry),

111
src/bin/pmacs_fake_lsp.rs
View File

@ -88,6 +88,8 @@ fn main() {
let mut stdout = io::stdout().lock();
let mut crashed_after_init = false;
let mut open_docs: HashMap<String, String> = HashMap::new();
// `filewatchjoin` / `filewatchretire` mid-session triggers.
let mut didchange_count: u32 = 0;
// `fullonly` observability: counts /full responses (rid-1, rid-2…).
let mut full_count: u32 = 0;
loop {
@ -354,6 +356,58 @@ fn main() {
});
write_frame(&mut stdout, &req);
}
// Issue #233 D3: `filewatchjoin` registers one watcher at
// initialized and a SECOND one — same base, different
// pattern — on the first didChange it receives. The
// didChange is the test's trigger for a mid-session join,
// which is what the join-wakes / registration-epoch /
// queued-baseline witnesses need and no
// at-initialized-only mode can produce.
("initialized", _) if mode == "filewatchjoin" => {
let base = std::env::var("PMACS_FAKE_LSP_WATCH_BASE").unwrap_or_default();
let req = serde_json::json!({
"jsonrpc": "2.0",
"id": 9310,
"method": "client/registerCapability",
"params": { "registrations": [{
"id": "watch-j1",
"method": "workspace/didChangeWatchedFiles",
"registerOptions": { "watchers": [{
"globPattern": {
"baseUri": format!("file://{base}"),
"pattern": "**/*.aaa"
},
"kind": 7
}] }
}] }
});
write_frame(&mut stdout, &req);
}
// Issue #233 D3: `filewatchretire` registers at
// initialized, UNREGISTERS on the first didChange (the
// group's last member leaves — retirement), and
// re-registers on the second (a fresh group whose
// baseline folds whatever happened while unwatched).
("initialized", _) if mode == "filewatchretire" => {
let base = std::env::var("PMACS_FAKE_LSP_WATCH_BASE").unwrap_or_default();
let req = serde_json::json!({
"jsonrpc": "2.0",
"id": 9320,
"method": "client/registerCapability",
"params": { "registrations": [{
"id": "watch-r1",
"method": "workspace/didChangeWatchedFiles",
"registerOptions": { "watchers": [{
"globPattern": {
"baseUri": format!("file://{base}"),
"pattern": "**/*.txt"
},
"kind": 7
}] }
}] }
});
write_frame(&mut stdout, &req);
}
// Issue #233 review P1 guard: `filewatchbare` registers a
// BARE STRING with no base and no leading `/` — `*.txt`.
// The string arm and the `filewatchflat` arm below carry the
@ -563,6 +617,63 @@ fn main() {
}
}
("textDocument/didOpen" | "textDocument/didChange", _) => {
if method == "textDocument/didChange" {
didchange_count += 1;
let base = std::env::var("PMACS_FAKE_LSP_WATCH_BASE").unwrap_or_default();
// `filewatchjoin` / `filewatchretire`: a didChange
// is the test's mid-session trigger; see the
// `initialized` arms above.
if mode == "filewatchjoin" && didchange_count == 1 {
let req = serde_json::json!({
"jsonrpc": "2.0",
"id": 9311,
"method": "client/registerCapability",
"params": { "registrations": [{
"id": "watch-j2",
"method": "workspace/didChangeWatchedFiles",
"registerOptions": { "watchers": [{
"globPattern": {
"baseUri": format!("file://{base}"),
"pattern": "**/*.bbb"
},
"kind": 7
}] }
}] }
});
write_frame(&mut stdout, &req);
}
if mode == "filewatchretire" && didchange_count == 1 {
let req = serde_json::json!({
"jsonrpc": "2.0",
"id": 9321,
"method": "client/unregisterCapability",
"params": { "unregisterations": [{
"id": "watch-r1",
"method": "workspace/didChangeWatchedFiles"
}] }
});
write_frame(&mut stdout, &req);
}
if mode == "filewatchretire" && didchange_count == 2 {
let req = serde_json::json!({
"jsonrpc": "2.0",
"id": 9322,
"method": "client/registerCapability",
"params": { "registrations": [{
"id": "watch-r2",
"method": "workspace/didChangeWatchedFiles",
"registerOptions": { "watchers": [{
"globPattern": {
"baseUri": format!("file://{base}"),
"pattern": "**/*.txt"
},
"kind": 7
}] }
}] }
});
write_frame(&mut stdout, &req);
}
}
let uri = params
.get("textDocument")
.and_then(|t| t.get("uri"))

323
src/fs.rs
View File

@ -377,6 +377,144 @@ pub fn read_dir_blocking(
})
}
/// Recursively walk `base` into ONE flat listing whose entry names are
/// base-RELATIVE paths (`sub/dir/file.txt`). Issue #233 D3: the LSP
/// file-watcher scan used to be one `read_dir` job per directory per
/// tick; this is the whole tree as one job.
///
/// Contract, in the D3 framing's terms:
/// - **Symlinks are recorded, never traversed.** `lstat` kind
/// `Symlink` regardless of target, so a link cycle cannot loop the
/// walk. A dangling or unreadable link target leaves the entry in
/// the listing with `symlink_target = None`, the same tolerance
/// [`read_dir_blocking`] applies.
/// - **Cancellation is cooperative and prompt**: the token is polled
/// once per directory and every [`READDIR_CANCEL_POLL_EVERY`]
/// entries within one --- the cadence [`read_dir_blocking`] set.
/// - **The root failing to open fails the walk** (the caller's
/// live-failure arm owns it); an unreadable SUBDIRECTORY is skipped
/// with its whole subtree, which is the Lua `scan_tree` `pcall`
/// behaviour this primitive replaces. A non-UTF-8 entry name is
/// skipped the same way: such a path cannot become a watcher URI,
/// and one weird name must not take the walk down.
///
/// Directory entries appear in the listing (kind `dir`) so a consumer
/// can see structure; the watcher filters them out when building
/// signatures, as its Lua walk always did.
pub fn walk_tree_blocking(
base: &Path,
cancel: &CancellationToken,
) -> Result<FsDirListing, FsError> {
// Checked BEFORE opening and again before returning, not only
// inside the entry loops: an empty tree never enters a loop, so a
// pre-cancelled queued walk would otherwise return an empty
// SUCCESS — which the scheduler's success arm would commit as a
// snapshot and diff into a deletion storm, the exact outcome the
// live non-success arm exists to prevent. (And a missing root
// must report Cancelled, not its Io error, for the same reason.)
if cancel.is_cancelled() {
return Err(FsError::Cancelled);
}
let root = std::fs::read_dir(base).map_err(|source| FsError::Io {
path: base.display().to_string(),
source,
})?;
let mut out: Vec<FsDirEntry> = Vec::new();
// Subdirectories discovered but not yet walked, with their
// base-relative prefixes. LIFO order --- traversal order is not
// part of the contract; the consumer diffs a map.
let mut pending: Vec<(std::path::PathBuf, String)> = Vec::new();
walk_one_dir(root, "", cancel, &mut out, &mut pending)?;
while let Some((dir, prefix)) = pending.pop() {
if cancel.is_cancelled() {
return Err(FsError::Cancelled);
}
// Subtree skip on an unreadable directory: scan_tree's pcall.
let Ok(iter) = std::fs::read_dir(&dir) else {
continue;
};
walk_one_dir(iter, &prefix, cancel, &mut out, &mut pending)?;
}
if cancel.is_cancelled() {
return Err(FsError::Cancelled);
}
Ok(FsDirListing {
entries: out,
errors: None,
})
}
#[cfg(test)]
thread_local! {
/// Test-only seam: invoked after every entry [`walk_one_dir`]
/// records. A MID-walk cancellation is an interleaving no real
/// timing produces on demand (the `_after_scan_for_tests`
/// justification, at this layer): the hook lets a test flip the
/// cancel token at an exact entry boundary and assert the walk
/// stopped NEAR it --- which is what discriminates the internal
/// polls from the entry/exit checks alone. `None` outside the one
/// test that arms it.
static WALK_ENTRY_HOOK: std::cell::RefCell<Option<Box<dyn FnMut()>>> =
const { std::cell::RefCell::new(None) };
}
/// One directory's worth of [`walk_tree_blocking`]: record every
/// representable entry under its base-relative name and queue child
/// directories. Only cancellation propagates as an error --- every
/// per-entry failure is a skip, per the walk's tolerance contract.
fn walk_one_dir(
iter: std::fs::ReadDir,
prefix: &str,
cancel: &CancellationToken,
out: &mut Vec<FsDirEntry>,
pending: &mut Vec<(std::path::PathBuf, String)>,
) -> Result<(), FsError> {
for (i, entry_result) in iter.enumerate() {
if i % READDIR_CANCEL_POLL_EVERY == 0 && cancel.is_cancelled() {
return Err(FsError::Cancelled);
}
let Ok(entry) = entry_result else { continue };
let Some(name) = entry.file_name().to_str().map(ToOwned::to_owned) else {
continue;
};
let rel = if prefix.is_empty() {
name
} else {
format!("{prefix}/{name}")
};
let entry_path = entry.path();
let Ok(metadata) = std::fs::symlink_metadata(&entry_path) else {
continue;
};
let kind = classify(&metadata);
if matches!(kind, FsEntryKind::Dir) {
pending.push((entry_path.clone(), rel.clone()));
}
let mut symlink_target = None;
if matches!(kind, FsEntryKind::Symlink)
&& let Ok(target) = std::fs::read_link(&entry_path)
{
symlink_target = target.to_str().map(ToOwned::to_owned);
}
out.push(FsDirEntry {
name: rel,
kind,
size: metadata.len(),
mtime_secs: mtime_to_unix_secs(&metadata),
mtime_nsec: mtime_to_unix_nsec(&metadata),
mode: mode_bits(&metadata),
symlink_target,
});
#[cfg(test)]
WALK_ENTRY_HOOK.with(|h| {
if let Some(hook) = h.borrow_mut().as_mut() {
hook();
}
});
}
Ok(())
}
/// Route one per-entry failure: append it to the tolerant channel, or
/// propagate it when the caller asked for the fatal contract.
///
@ -930,4 +1068,189 @@ mod tests {
other => panic!("expected NonUtf8Path, got {other:?}"),
}
}
#[test]
fn walk_tree_flattens_with_relative_names() {
let td = tempfile::tempdir().expect("tempdir");
std::fs::write(td.path().join("a.txt"), b"hello").expect("write");
std::fs::create_dir(td.path().join("sub")).expect("mkdir");
std::fs::write(td.path().join("sub/b.txt"), b"xy").expect("write");
std::fs::create_dir(td.path().join("sub/deep")).expect("mkdir");
std::fs::write(td.path().join("sub/deep/c.txt"), b"z").expect("write");
let listing = walk_tree_blocking(td.path(), &token()).expect("walk");
assert!(listing.errors.is_none());
let mut names: Vec<&str> = listing.entries.iter().map(|e| e.name.as_str()).collect();
names.sort_unstable();
assert_eq!(
names,
vec!["a.txt", "sub", "sub/b.txt", "sub/deep", "sub/deep/c.txt"]
);
let c = listing
.entries
.iter()
.find(|e| e.name == "sub/deep/c.txt")
.unwrap();
assert_eq!(c.kind, FsEntryKind::File);
assert_eq!(c.size, 1);
let sub = listing.entries.iter().find(|e| e.name == "sub").unwrap();
assert_eq!(sub.kind, FsEntryKind::Dir);
}
#[test]
fn walk_tree_records_symlink_without_traversing() {
let td = tempfile::tempdir().expect("tempdir");
std::fs::write(td.path().join("real.txt"), b"x").expect("write");
// A link back to the root: traversal would loop forever, so
// completing at all is half the witness.
symlink(td.path(), td.path().join("loop")).expect("symlink");
let listing = walk_tree_blocking(td.path(), &token()).expect("walk");
let link = listing.entries.iter().find(|e| e.name == "loop").unwrap();
assert_eq!(link.kind, FsEntryKind::Symlink);
assert!(
!listing.entries.iter().any(|e| e.name.starts_with("loop/")),
"a symlinked directory must be recorded, never entered"
);
}
#[test]
fn walk_tree_skips_unreadable_subdirectory_and_keeps_the_rest() {
use std::os::unix::fs::PermissionsExt;
let td = tempfile::tempdir().expect("tempdir");
std::fs::write(td.path().join("a.txt"), b"x").expect("write");
std::fs::create_dir(td.path().join("locked")).expect("mkdir");
std::fs::write(td.path().join("locked/hidden.txt"), b"x").expect("write");
std::fs::set_permissions(
td.path().join("locked"),
std::fs::Permissions::from_mode(0o000),
)
.expect("chmod 000");
let result = walk_tree_blocking(td.path(), &token());
// Restore before asserting so the tempdir can be removed even
// if an assertion fails.
std::fs::set_permissions(
td.path().join("locked"),
std::fs::Permissions::from_mode(0o755),
)
.expect("chmod back");
let listing = result.expect("walk must survive an unreadable subdirectory");
let names: Vec<&str> = listing.entries.iter().map(|e| e.name.as_str()).collect();
assert!(names.contains(&"a.txt"));
assert!(
names.contains(&"locked"),
"the dir entry itself is representable"
);
assert!(
!names.contains(&"locked/hidden.txt"),
"the unreadable subtree is skipped, matching scan_tree's pcall"
);
}
#[test]
fn walk_tree_polls_cancellation_token() {
let td = tempfile::tempdir().expect("tempdir");
std::fs::write(td.path().join("a.txt"), b"x").expect("write");
let cancel = token();
cancel.cancel();
let err = walk_tree_blocking(td.path(), &cancel).expect_err("must observe cancel");
assert!(matches!(err, FsError::Cancelled));
}
/// Review blocker: the entry loops never run on an EMPTY tree, so
/// without the entry/exit checks a pre-cancelled queued walk
/// returned an empty SUCCESS — which the scheduler would commit as
/// a snapshot and diff into a deletion storm. A missing root must
/// likewise report Cancelled, not its Io error.
#[test]
fn walk_tree_pre_cancelled_is_cancelled_even_with_no_entries() {
let empty = tempfile::tempdir().expect("tempdir");
let cancel = token();
cancel.cancel();
let err = walk_tree_blocking(empty.path(), &cancel)
.expect_err("empty tree must still observe cancel");
assert!(matches!(err, FsError::Cancelled));
let err = walk_tree_blocking(&empty.path().join("absent"), &cancel)
.expect_err("missing root must still observe cancel");
assert!(
matches!(err, FsError::Cancelled),
"cancellation outranks the root error: got {err:?}"
);
}
/// Review blocker: both prior cancellation tests pre-cancelled, so
/// deleting the INTERNAL polls (per directory, and every
/// [`READDIR_CANCEL_POLL_EVERY`] entries) left every test green —
/// the walk ran to completion and only the exit check fired. This
/// cancels MID-walk through the test hook and asserts the walk
/// stopped near the cancellation point, not at the end.
#[test]
fn walk_tree_observes_cancellation_mid_walk() {
let td = tempfile::tempdir().expect("tempdir");
for d in 0..3 {
let dir = td.path().join(format!("d{d}"));
std::fs::create_dir(&dir).expect("mkdir");
for i in 0..41 {
std::fs::write(dir.join(format!("f{i:02}.txt")), b"x").expect("write");
}
}
let cancel = token();
let seen = std::rc::Rc::new(std::cell::Cell::new(0usize));
{
let seen = seen.clone();
let cancel = cancel.clone();
WALK_ENTRY_HOOK.with(|h| {
*h.borrow_mut() = Some(Box::new(move || {
let n = seen.get() + 1;
seen.set(n);
if n == 5 {
cancel.cancel();
}
}));
});
}
let result = walk_tree_blocking(td.path(), &cancel);
WALK_ENTRY_HOOK.with(|h| *h.borrow_mut() = None);
assert!(
matches!(result, Err(FsError::Cancelled)),
"mid-walk cancel must surface as Cancelled"
);
// 126 entries total (3 dirs + 123 files). The cancel lands at
// entry 5 (root's three dir entries, then two files), so the
// per-entry poll stops the walk at exactly 35 = 3 + 32 (one
// READDIR_CANCEL_POLL_EVERY stride). The bound sits BELOW 44:
// with the per-entry poll deleted, the 41-file directory runs
// to completion and the per-directory poll catches at 44 --- a
// bound of 60 could not tell the two apart (review round 3).
assert!(
seen.get() < 40,
"the walk must stop within one poll stride of the cancel \
({} of 126 entries processed; expected 35)",
seen.get()
);
}
#[test]
fn walk_tree_fails_when_the_root_cannot_open() {
let td = tempfile::tempdir().expect("tempdir");
let err = walk_tree_blocking(&td.path().join("absent"), &token())
.expect_err("missing root must fail the walk");
assert!(matches!(err, FsError::Io { .. }));
}
#[test]
fn walk_tree_matches_read_dir_on_a_flat_directory() {
let td = tempfile::tempdir().expect("tempdir");
std::fs::write(td.path().join("a.txt"), b"hello").expect("write");
std::fs::write(td.path().join("b.md"), b"xy").expect("write");
let mut walked = walk_tree_blocking(td.path(), &token())
.expect("walk")
.entries;
let mut listed = read_dir_fatal(td.path(), &token()).expect("read_dir");
walked.sort_by(|a, b| a.name.cmp(&b.name));
listed.sort_by(|a, b| a.name.cmp(&b.name));
// At depth zero the relative name IS the basename, so the two
// primitives must agree entry-for-entry --- the signature
// parity the D3 framing requires.
assert_eq!(walked, listed);
}
}

10
src/lua_bindings/mod.rs
View File

@ -7364,6 +7364,16 @@ pub fn install_async(
)?;
}
{
let rt = runtime.clone();
async_mod.set(
"_dispatch_fs_walk_tree",
lua.create_function(move |_, (base, key): (String, Option<String>)| {
Ok(rt.dispatch_fs_walk_tree(std::path::PathBuf::from(base), key.as_deref()))
})?,
)?;
}
{
let rt = runtime.clone();
async_mod.set(

1027
tests/m4_acceptance.rs
View File

File diff suppressed because it is too large Load Diff