From 4a60e4c85805e23e819afa2bd617eac7f73bc2ee Mon Sep 17 00:00:00 2001 From: Levi Neuwirth Date: Tue, 14 Jul 2026 14:33:06 +0100 Subject: [PATCH 1/4] feat(highlight): shebang-based language detection for extensionless scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extension detection missed extensionless scripts — `scripts/deploy`, git hooks, `configure`, and `scripts/bite` itself — so they got neither highlighting nor an LSP server. Add a first-line shebang fallback. - New `pmacs.parse.language_from_shebang(buf)` (builtin/runtime/syntax.lua): sniffs the first line (capped at 256 bytes), maps the interpreter's basename to a language, and resolves the `#!/usr/bin/env python3` indirection (skipping env's own `-S`/flags and `VAR=val` assignments). Backed by `pmacs.parse.shebangs`, a user-extensible map seeded with the interpreters pmacs can act on: sh-family -> bash, python* -> python, node -> javascript, lua* -> lua. - Wired as a strict *fallback* on both resolution paths: syntax.lua's grammar attach (`language_for_path or language_from_shebang`) and lsp.lua's `buffer_language` (grammar -> filetypes -> shebang). A recognized extension always wins, so a `.py`/`.sh` file is never re-classified by a stray shebang. - Cross-language, not shell-only: `#!/usr/bin/env python` /`node` /`lua` resolve too. Special filenames (`.bashrc`, `Dockerfile`, `Makefile`) are intentionally deferred until there are grammars behind them. Bite-verified acceptance (tests/m4_acceptance.rs): - m4_shebang_resolver_maps_interpreters — the mapping incl. env indirection and `env -S`; non-shebangs and unmapped interpreters (ruby) resolve to nil. - m4_shebang_extensionless_script_resolves_bash — opening an extensionless `#!/bin/sh` script resolves to bash on BOTH paths: lsp.lua's `active_buffer_language()` and a settled bash parse tree (grammar attach). Reachable only via the shebang, since the file has no extension. - m4_shebang_does_not_override_extension — a `.py` file opening with `#!/bin/sh` still resolves to python (extension precedence). Gates green: fmt; clippy -D warnings; test --lib; --features crdt; m4_acceptance --skip basedpyright; GPU; full workspace sweep; git diff --check. Change is Lua-only plus the acceptance tests. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01YJ9FQ832QwftJXCD9LeFan --- builtin/runtime/lsp.lua | 8 ++- builtin/runtime/syntax.lua | 68 +++++++++++++++++++++ tests/m4_acceptance.rs | 121 +++++++++++++++++++++++++++++++++++++ 3 files changed, 195 insertions(+), 2 deletions(-) diff --git a/builtin/runtime/lsp.lua b/builtin/runtime/lsp.lua index 6607423..4961657 100644 --- a/builtin/runtime/lsp.lua +++ b/builtin/runtime/lsp.lua @@ -374,11 +374,15 @@ local function buffer_language(buf) if not ok or not path then return nil end -- Grammar-backed detection first (keeps rust/.rs etc. exactly as -- before); fall back to the LSP-only filetype map so languages - -- with a server but no tree-sitter grammar (Python) still attach. + -- with a server but no tree-sitter grammar (Python) still attach; + -- finally, for an extensionless file, sniff a `#!interp` shebang so + -- e.g. an extensionless `#!/bin/sh` script still attaches its server. local lang = pmacs.parse.language_for_path(path) if lang then return lang end local ext = path:match("%.([%w_]+)$") - return ext and pmacs.lsp.filetypes[ext] or nil + local by_ext = ext and pmacs.lsp.filetypes[ext] + if by_ext then return by_ext end + return pmacs.parse.language_from_shebang(buf) end -- Public: the per-buffer language chain. Auto-pairing resolves -- relevance against the buffer its typed-edit record names — which a diff --git a/builtin/runtime/syntax.lua b/builtin/runtime/syntax.lua index 5f7fcb8..ae73b76 100644 --- a/builtin/runtime/syntax.lua +++ b/builtin/runtime/syntax.lua @@ -44,6 +44,72 @@ function pmacs.parse._dispatch(buf, lang) return job_id end +-- Shebang → language detection ------------------------------------------ +-- +-- Extension detection (`language_for_path`) misses extensionless scripts +-- (`scripts/deploy`, git hooks, `configure`), which are the common case +-- for shell. This fills that gap by sniffing the first line: `#!interp` +-- maps the interpreter's basename to a language. It is a *fallback* — +-- every caller tries extension detection first, so a `.py`/`.sh` file is +-- never re-classified by a stray shebang. Deliberately does not cover +-- special filenames (`.bashrc`, `Dockerfile`): those wait on grammars for +-- the languages behind them. +-- +-- The map is user-extensible from init.lua, e.g. +-- `pmacs.parse.shebangs.ruby = "ruby"`. Only interpreters whose language +-- pmacs can act on (grammar and/or LSP config) are seeded; an entry whose +-- language has neither is harmless but inert. +pmacs.parse.shebangs = pmacs.parse.shebangs or { + sh = "bash", bash = "bash", dash = "bash", ash = "bash", + ksh = "bash", mksh = "bash", zsh = "bash", + python = "python", python2 = "python", python3 = "python", + pypy = "python", pypy3 = "python", + node = "javascript", nodejs = "javascript", + lua = "lua", luajit = "lua", +} + +-- First line of `buf` (up to 256 bytes), sans trailing newline, or nil. +-- 256 bytes is well past any real shebang and keeps the slice cheap on +-- the hot path (callers only reach here when extension detection missed). +local function first_line(buf) + local ok_len, n = pcall(function() return buf:len() end) + if not ok_len or not n or n <= 0 then return nil end + if n > 256 then n = 256 end + local ok, s = pcall(function() return buf:slice(0, n) end) + if not ok or type(s) ~= "string" or #s == 0 then return nil end + local nl = s:find("\n", 1, true) + if nl then s = s:sub(1, nl - 1) end + return s +end + +-- Resolve the interpreter basename from a shebang line, resolving the +-- `env` indirection (`#!/usr/bin/env python3` → `python3`, skipping +-- `env`'s own options / `VAR=val` assignments). Returns the language +-- name from `pmacs.parse.shebangs`, or nil. +function pmacs.parse.language_from_shebang(buf) + if not buf then return nil end + local line = first_line(buf) + if not line or line:sub(1, 2) ~= "#!" then return nil end + local rest = line:sub(3):gsub("^%s+", "") + local first = rest:match("^(%S+)") + if not first then return nil end + local base = first:match("([^/]+)$") or first + if base == "env" then + base = nil + for tok in rest:gmatch("%S+") do + -- Skip the `env` path itself, its flags (`-S`, `--split-string`), + -- and inline `VAR=value` assignments; the first bare word left is + -- the real interpreter. + if tok ~= first and tok:sub(1, 1) ~= "-" and not tok:find("=", 1, true) then + base = tok:match("([^/]+)$") or tok + break + end + end + if not base then return nil end + end + return pmacs.parse.shebangs[base] +end + -- Set of buffer ids that already have a highlight overlay -- attached, keyed by raw id (number). A buffer that opens, gets -- highlights, gets killed, and is reopened needs a fresh overlay @@ -56,6 +122,7 @@ local function attach_for_active_buffer() local path = buf:name() if not path then return end local lang = pmacs.parse.language_for_path(path) + or pmacs.parse.language_from_shebang(buf) if not lang then return end pmacs.parse._dispatch(buf, lang) -- T M4.3: install the syntax-highlight overlay for this buffer. @@ -111,6 +178,7 @@ local function reparse_active_buffer_after_edit() local path = buf:name() if not path then return end local lang = pmacs.parse.language_for_path(path) + or pmacs.parse.language_from_shebang(buf) if not lang then return end pmacs.parse._dispatch(buf, lang) end diff --git a/tests/m4_acceptance.rs b/tests/m4_acceptance.rs index c648bcc..2d84aa1 100644 --- a/tests/m4_acceptance.rs +++ b/tests/m4_acceptance.rs @@ -5849,6 +5849,127 @@ fn m4_12_default_bundle_wires_bash() { assert_eq!(probe.get::("grammar_bats").unwrap(), "bash"); } +/// Shebang detection: `pmacs.parse.language_from_shebang` maps the +/// interpreter basename (resolving the `#!/usr/bin/env` indirection) to a +/// language, and returns nil for non-shebangs and unmapped interpreters. +/// This is the fallback that lets extensionless scripts (`scripts/deploy`, +/// git hooks, `configure`) resolve a language at all — extension +/// detection misses them. +#[test] +fn m4_shebang_resolver_maps_interpreters() { + use pmacs::editor::EditorState; + let s = EditorState::new(); + let resolve = |first_line: &str| -> Option { + s.lua_host + .lua() + .load(format!( + "local b = pmacs.window.buffer() + if b:len() > 0 then b:delete(0, b:len()) end + b:insert(0, {first_line:?}) + return pmacs.parse.language_from_shebang(b)" + )) + .eval() + .expect("resolve shebang") + }; + for (line, want) in [ + ("#!/bin/sh\n", "bash"), + ("#!/bin/bash -e\n", "bash"), + ("#! /bin/zsh\n", "bash"), + ("#!/usr/bin/env bash\n", "bash"), + ("#!/usr/bin/env python3\n", "python"), + ("#!/usr/bin/env -S python3 -u\n", "python"), + ("#!/usr/bin/node\n", "javascript"), + ("#!/usr/bin/env lua\n", "lua"), + ] { + assert_eq!(resolve(line).as_deref(), Some(want), "{line:?}"); + } + for line in [ + "echo hi\n", + "# just a comment\n", + "#!/usr/bin/env ruby\n", // interpreter not in the seeded map + "\n", + "", + ] { + assert_eq!(resolve(line), None, "{line:?}"); + } +} + +/// End-to-end: opening an extensionless `#!/bin/sh` script resolves to +/// `bash` on both paths — `lsp.lua`'s `buffer_language` chain (so the +/// server would attach) and `syntax.lua`'s grammar attach (so a bash +/// parse tree is produced). `pmacs.lsp.config` is emptied first so the +/// real bash-language-server isn't spawned; grammar detection is +/// independent of the LSP config. +#[test] +fn m4_shebang_extensionless_script_resolves_bash() { + use pmacs::editor::EditorState; + let mut s = EditorState::new(); + let dir = tempfile::tempdir().expect("tempdir"); + let hook = dir.path().join("pre-commit"); // no extension + std::fs::write(&hook, b"#!/bin/sh\nset -e\necho building\n").expect("write"); + let hook_disp = hook.display(); + + s.lua_host + .lua() + .load(format!( + "pmacs.lsp.config = {{}} + pmacs.buffer.find_or_open('{hook_disp}')" + )) + .exec() + .expect("open extensionless shebang script"); + + let lsp_lang: Option = s + .lua_host + .lua() + .load("return pmacs.lsp.active_buffer_language()") + .eval() + .expect("lsp language"); + assert_eq!( + lsp_lang.as_deref(), + Some("bash"), + "extensionless #!/bin/sh resolves to bash for LSP" + ); + + pump_async(&mut s, |st| current_tree_language(st).is_some()); + assert_eq!( + current_tree_language(&s).as_deref(), + Some("bash"), + "extensionless #!/bin/sh gets a bash parse tree" + ); +} + +/// Precedence: a recognized extension always wins over file content, so a +/// `.py` file that happens to open with `#!/bin/sh` still resolves to +/// python — the shebang is consulted only when extension detection misses. +#[test] +fn m4_shebang_does_not_override_extension() { + use pmacs::editor::EditorState; + let s = EditorState::new(); + let dir = tempfile::tempdir().expect("tempdir"); + let f = dir.path().join("tool.py"); + std::fs::write(&f, b"#!/bin/sh\nprint('hi')\n").expect("write"); + let f_disp = f.display(); + s.lua_host + .lua() + .load(format!( + "pmacs.lsp.config = {{}} + pmacs.buffer.find_or_open('{f_disp}')" + )) + .exec() + .expect("open .py with a shell shebang"); + let lang: Option = s + .lua_host + .lua() + .load("return pmacs.lsp.active_buffer_language()") + .eval() + .expect("language"); + assert_eq!( + lang.as_deref(), + Some("python"), + ".py extension wins over a #!/bin/sh shebang" + ); +} + /// Typing-perf: the default bundle coalesces full-document /// `didChange` notifications instead of sending one per keystroke /// (each send copies the whole buffer several times and writes From f300b77533bf1c5008a491d670546c99eb7b249d Mon Sep 17 00:00:00 2001 From: Levi Neuwirth Date: Tue, 14 Jul 2026 15:02:59 +0100 Subject: [PATCH 2/4] =?UTF-8?q?fix(highlight):=20PR=20#116=20round=201=20?= =?UTF-8?q?=E2=80=94=20shebang=20precedence,=20pinned=20grammar,=20env=20o?= =?UTF-8?q?perands?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review findings, all in builtin/runtime/syntax.lua. 1. [P1] Syntax bypassed extension precedence and grammar availability. attach_for_active_buffer resolved `language_for_path or shebang`, but language_for_path knows only grammar-backed extensions — so a `.py` file opening with `#!/bin/sh` fell through to the shebang and got a bash parse tree, and an extensionless `#!/usr/bin/env python3` script dispatched "python" (no grammar) and raised "unknown language". A new resolve_active_language walks the full precedence chain — grammar extension -> LSP filetype map -> shebang — consulting the shebang only when the extension is unrecognized (a recognized non-grammar extension like .py is authoritative). Dispatch is then gated on pmacs.parse._has_language(lang), so grammarless languages are skipped silently. The extension parts stay keyed on buf:name() (unchanged from before), so path-less buffers that resolve a grammar by name — e.g. generated markdown buffers — are unaffected. 2. [P2] Editing an open script's shebang left parsing/highlighting stale. The after-edit path re-sniffed the mutable shebang: sh -> python raised "unknown language" while leaving the old bash tree, and sh -> lua swapped the parse tree under a highlight overlay still holding the original grammar's query. Reparse now uses the language pinned at first attach (parse_lang_by_buffer), never re-resolving — a language change needs a close/reopen, as it does for extensions. 3. [P2] `env` options with operands were mistaken for interpreters. `#!/usr/bin/env -u FOO python3` skipped `-u` but took `FOO`. The env walk now skips the operand of the operand-consuming GNU-env options (-u/--unset, -C/--chdir, -a/--argv0) before selecting the interpreter. -S/--split-string stays excluded (its string carries the interpreter). Tests (bite-verified against pre-fix syntax.lua — each fails without its fix; scripts/bite HEAD builtin/runtime/syntax.lua): - m4_shebang_does_not_override_extension now also asserts _has_view is false (no bash grammar tree for a `.py` + `#!/bin/sh`), not only the LSP language. - m4_shebang_extensionless_grammarless_language_is_silent — extensionless python resolves for LSP, gets no grammar view, and records no error. - m4_shebang_edit_keeps_pinned_grammar — rewriting a `#!/bin/sh` script's shebang to lua keeps the bash tree and reports no error. - m4_shebang_resolver_maps_interpreters — added the env-operand cases (`-u FOO`, `-C /tmp`, combined). Gates: fmt; clippy -D warnings; m4_acceptance --skip basedpyright; GPU; git diff --check all green. The only sweep failure is the pre-existing editor::composition_overhead_under_ten_percent render microbenchmark (ratio hovers at the 1.10 cutoff; flakes ~1/3 even isolated single- threaded, already asserted-off on macOS) — a pure-Rust render loop this Lua-only change cannot touch. Change is Lua-only plus the acceptance tests. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01YJ9FQ832QwftJXCD9LeFan --- builtin/runtime/syntax.lua | 71 ++++++++++++++++---- tests/m4_acceptance.rs | 128 +++++++++++++++++++++++++++++++++++-- 2 files changed, 182 insertions(+), 17 deletions(-) diff --git a/builtin/runtime/syntax.lua b/builtin/runtime/syntax.lua index ae73b76..cfe71ed 100644 --- a/builtin/runtime/syntax.lua +++ b/builtin/runtime/syntax.lua @@ -96,11 +96,29 @@ function pmacs.parse.language_from_shebang(buf) local base = first:match("([^/]+)$") or first if base == "env" then base = nil + local seen_env = false + local skip_next = false for tok in rest:gmatch("%S+") do - -- Skip the `env` path itself, its flags (`-S`, `--split-string`), - -- and inline `VAR=value` assignments; the first bare word left is - -- the real interpreter. - if tok ~= first and tok:sub(1, 1) ~= "-" and not tok:find("=", 1, true) then + if not seen_env then + seen_env = true -- the `env` path token itself + elseif skip_next then + skip_next = false -- the operand consumed by the previous option + elseif tok:find("=", 1, true) then + -- `VAR=value` env assignment, or a `--long=value` option: both + -- self-contained, skip. + elseif tok:sub(1, 1) == "-" then + -- An option. A few GNU-env short options and their long forms + -- consume the *next* token as an operand (`-u NAME`, `-C DIR`, + -- `-a NAME`); skip that operand too, or its value is mistaken for + -- the interpreter. `-S`/`--split-string` is deliberately absent: + -- the string it introduces contains the interpreter, which the + -- walk then picks up. An option with an attached operand + -- (`-uNAME`) is one self-contained token and needs no skip. + if tok == "-u" or tok == "-C" or tok == "-a" + or tok == "--unset" or tok == "--chdir" or tok == "--argv0" then + skip_next = true + end + else base = tok:match("([^/]+)$") or tok break end @@ -116,14 +134,37 @@ end -- attach; the kill path clears the entry below if/when it lands. local highlighted_buffers = {} +-- Filetype-aware language resolution for the active buffer, in +-- precedence order: grammar extension → LSP filetype map → shebang. The +-- shebang is consulted ONLY when the extension is unrecognized (a known +-- non-grammar extension like `.py` must not fall through to a stray +-- `#!/bin/sh` and be misparsed as bash). Keyed on `buf:name()` for the +-- extension parts (matching the historical behavior — path-less buffers +-- that resolve a grammar by name keep working); the shebang reads buffer +-- content directly. +local function resolve_active_language(buf) + local name = buf:name() + if name then + local grammar = pmacs.parse.language_for_path(name) + if grammar then return grammar end + local ext = name:match("%.([%w_]+)$") + local by_ext = ext and pmacs.lsp and pmacs.lsp.filetypes and pmacs.lsp.filetypes[ext] + -- A recognized (even non-grammar) extension is authoritative; do not + -- consult the shebang for it. + if by_ext then return by_ext end + end + return pmacs.parse.language_from_shebang(buf) +end + local function attach_for_active_buffer() local buf = pmacs.window.buffer() if not buf then return end - local path = buf:name() - if not path then return end - local lang = pmacs.parse.language_for_path(path) - or pmacs.parse.language_from_shebang(buf) - if not lang then return end + -- Gate dispatch on `_has_language`: the chain above also resolves + -- languages with no grammar (python, javascript), and dispatching one + -- would raise "unknown language" (caught, but noise) — and an + -- extensionless script must never get a wrong-grammar parse tree. + local lang = resolve_active_language(buf) + if not lang or not pmacs.parse._has_language(lang) then return end pmacs.parse._dispatch(buf, lang) -- T M4.3: install the syntax-highlight overlay for this buffer. -- Idempotent --- repeated calls for the same buffer are a no-op @@ -175,10 +216,14 @@ local function reparse_active_buffer_after_edit() if not pmacs.parse._has_view(buf) then return end local pending = pmacs.parse._pending_edits(buf) if not pending or pending == 0 then return end - local path = buf:name() - if not path then return end - local lang = pmacs.parse.language_for_path(path) - or pmacs.parse.language_from_shebang(buf) + -- Reparse with the language pinned when the view was first attached --- + -- never re-resolve from the path or (mutable) shebang. The Rust side is + -- "first language wins"; re-sniffing a shebang the user just edited + -- would either raise "unknown language" (sh → python) or swap the parse + -- tree to a new grammar while the highlight overlay still holds the + -- original grammar's query (sh → lua). Language changes need a + -- close/reopen, exactly as they do for a renamed extension. + local lang = parse_lang_by_buffer[tostring(buf)] if not lang then return end pmacs.parse._dispatch(buf, lang) end diff --git a/tests/m4_acceptance.rs b/tests/m4_acceptance.rs index 2d84aa1..9dacd83 100644 --- a/tests/m4_acceptance.rs +++ b/tests/m4_acceptance.rs @@ -5878,6 +5878,11 @@ fn m4_shebang_resolver_maps_interpreters() { ("#!/usr/bin/env bash\n", "bash"), ("#!/usr/bin/env python3\n", "python"), ("#!/usr/bin/env -S python3 -u\n", "python"), + // GNU-env options that consume an operand must not have the + // operand mistaken for the interpreter. + ("#!/usr/bin/env -u FOO python3\n", "python"), + ("#!/usr/bin/env -C /tmp python3\n", "python"), + ("#!/usr/bin/env -u FOO -C /tmp node\n", "javascript"), ("#!/usr/bin/node\n", "javascript"), ("#!/usr/bin/env lua\n", "lua"), ] { @@ -5957,17 +5962,132 @@ fn m4_shebang_does_not_override_extension() { )) .exec() .expect("open .py with a shell shebang"); - let lang: Option = s + // Both the LSP language *and* the grammar decision must respect the + // extension: python for LSP, and NO grammar parse view (python has no + // grammar) — not a bash tree installed from the `#!/bin/sh` line. + // `_has_view` is set synchronously by `_dispatch`, so no pump is + // needed; without the precedence fix the shebang would have dispatched + // bash and this would be true. + let (lang, has_view): (Option, bool) = s .lua_host .lua() - .load("return pmacs.lsp.active_buffer_language()") + .load( + "return pmacs.lsp.active_buffer_language(), + pmacs.parse._has_view(pmacs.window.buffer())", + ) .eval() - .expect("language"); + .expect("language + view"); assert_eq!( lang.as_deref(), Some("python"), - ".py extension wins over a #!/bin/sh shebang" + ".py extension wins over a #!/bin/sh shebang (LSP)" ); + assert!( + !has_view, + ".py file must not get a grammar parse view from a #!/bin/sh line" + ); +} + +/// Finding-1 gate: an extensionless `#!/usr/bin/env python3` script +/// resolves to python for LSP, but python has no grammar — syntax must +/// skip it *silently*. Without the `_has_language` gate, `_dispatch` +/// raises "unknown language: python" (caught by the after-load pcall and +/// reported through `pmacs.error`), which we assert does NOT happen. +#[test] +fn m4_shebang_extensionless_grammarless_language_is_silent() { + use pmacs::editor::EditorState; + let s = EditorState::new(); + let dir = tempfile::tempdir().expect("tempdir"); + let f = dir.path().join("generate"); // no extension + std::fs::write(&f, b"#!/usr/bin/env python3\nprint('hi')\n").expect("write"); + let f_disp = f.display(); + s.lua_host + .lua() + .load(format!( + "pmacs.lsp.config = {{}} + _G.__errs = {{}} + local real = pmacs.error + pmacs.error = function(m) table.insert(_G.__errs, m) end + pmacs.buffer.find_or_open('{f_disp}')" + )) + .exec() + .expect("open extensionless python script"); + let (lang, has_view, errs): (Option, bool, i64) = s + .lua_host + .lua() + .load( + "return pmacs.lsp.active_buffer_language(), + pmacs.parse._has_view(pmacs.window.buffer()), + #_G.__errs", + ) + .eval() + .expect("probe"); + assert_eq!(lang.as_deref(), Some("python"), "python resolves for LSP"); + assert!( + !has_view, + "no grammar parse view for a grammarless language" + ); + assert_eq!(errs, 0, "no 'unknown language' error reported"); +} + +/// Finding-2 pin: editing an open extensionless script's shebang must not +/// re-switch the parse grammar. A `#!/bin/sh` script attaches the bash +/// grammar; rewriting its shebang to lua and firing after-edit must keep +/// the bash tree (the pinned grammar) rather than swap in lua under the +/// stale highlight overlay — and must not error. +#[test] +fn m4_shebang_edit_keeps_pinned_grammar() { + use pmacs::editor::EditorState; + let mut s = EditorState::new(); + let dir = tempfile::tempdir().expect("tempdir"); + let hook = dir.path().join("deploy"); // no extension + std::fs::write(&hook, b"#!/bin/sh\necho one\n").expect("write"); + let hook_disp = hook.display(); + s.lua_host + .lua() + .load(format!( + "pmacs.lsp.config = {{}} + _G.__errs = {{}} + local real = pmacs.error + pmacs.error = function(m) table.insert(_G.__errs, m) end + pmacs.buffer.find_or_open('{hook_disp}')" + )) + .exec() + .expect("open extensionless shell script"); + pump_async(&mut s, |st| { + current_tree_language(st).as_deref() == Some("bash") + }); + + // Rewrite the first line to a lua shebang, then fire after-edit. + s.lua_host + .lua() + .load( + "local b = pmacs.window.buffer() + local text = b:slice(0, b:len()) + local first_len = (text:find('\\n', 1, true) or 1) - 1 + b:replace(0, first_len, '#!/usr/bin/env lua') + pmacs.hook.run('buffer.after-edit')", + ) + .exec() + .expect("rewrite shebang to lua"); + // Let the reparse settle (manual ticks: the tree stays bash with the + // pin, so a `pump_async` for a language *change* would time out). + for _ in 0..64 { + s.tick_async(); + std::thread::sleep(Duration::from_millis(2)); + } + assert_eq!( + current_tree_language(&s).as_deref(), + Some("bash"), + "editing the shebang must not re-switch the pinned grammar" + ); + let errs: i64 = s + .lua_host + .lua() + .load("return #_G.__errs") + .eval() + .expect("errs"); + assert_eq!(errs, 0, "reparse with the pinned grammar reports no error"); } /// Typing-perf: the default bundle coalesces full-document From 558d00020f7714e108e2b95442048647f79048d3 Mon Sep 17 00:00:00 2001 From: Levi Neuwirth Date: Tue, 14 Jul 2026 15:20:22 +0100 Subject: [PATCH 3/4] =?UTF-8?q?fix(highlight):=20PR=20#116=20round=202=20?= =?UTF-8?q?=E2=80=94=20pin=20grammar=20across=20switch,=20attached=20env?= =?UTF-8?q?=20-S?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-ups from review, both in builtin/runtime/syntax.lua. 1. [P2] Buffer switching bypassed the pinned grammar. after-edit already reparsed the pinned language, but the after-switch reattach path (attach_for_active_buffer) re-resolved from scratch — so open an extensionless `#!/bin/sh` (bash), edit its shebang to lua, switch away and back, and the grammar flipped to lua while the LSP side kept its bash attachment (lsp.lua's after-switch reuses the existing record). attach_for_active_buffer now reuses the language pinned at first attach whenever a parse view already exists; only a first-seen buffer resolves. A language change still needs a close/reopen, matching both the after-edit behavior and how extensions work. 2. [P2] Attached `env -S`/`--split-string` forms failed. The walk skipped the whole option token, but for split-string the interpreter rides inside it: `-Spython3`, `-vSpython3` (after no-operand short flags i/v/0), and `--split-string=python3` all resolved to nil (the last was also eaten by the earlier `=` branch). The env walk now extracts the interpreter from the attached value (`^-[iv0]*S(.+)$` / `^--split-string=(.+)$`); the separated forms (`-S python3`) still work by walking on to the next token. Tests (bite-verified against the round-1 syntax.lua — both fail there; scripts/bite HEAD builtin/runtime/syntax.lua): - m4_shebang_edit_keeps_pinned_grammar now adds a switch-away/back cycle (via pmacs.window.switch_buffer, which fires after-switch synchronously) and asserts the tree stays bash. - m4_shebang_resolver_maps_interpreters adds the attached split-string cases (`-Spython3`, `--split-string=python3`, `-vSpython3`). Gates: fmt; clippy -D warnings; m4_acceptance --skip basedpyright; GPU; git diff --check green. Only-known-flake caveat as round 1 (editor::composition_overhead_under_ten_percent perf microbenchmark, unrelated to this Lua change). Change is Lua-only plus the acceptance tests. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01YJ9FQ832QwftJXCD9LeFan --- builtin/runtime/syntax.lua | 44 +++++++++++++++++++++++++++----------- tests/m4_acceptance.rs | 32 +++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/builtin/runtime/syntax.lua b/builtin/runtime/syntax.lua index cfe71ed..dd97689 100644 --- a/builtin/runtime/syntax.lua +++ b/builtin/runtime/syntax.lua @@ -99,21 +99,34 @@ function pmacs.parse.language_from_shebang(buf) local seen_env = false local skip_next = false for tok in rest:gmatch("%S+") do + -- `-S`/`--split-string` introduces a string whose FIRST word is the + -- interpreter. GNU env accepts that value ATTACHED — `-Spython3`, + -- `-vSpython3` (after no-operand short flags i/v/0), or + -- `--split-string=python3` — where the interpreter rides inside the + -- option token. Extract it; the separated forms (`-S python3`) are + -- handled by simply walking on to the next token. + local split_attached = + tok:match("^%-[iv0]*S(.+)$") or tok:match("^%-%-split%-string=(.+)$") if not seen_env then seen_env = true -- the `env` path token itself elseif skip_next then skip_next = false -- the operand consumed by the previous option + elseif split_attached then + local word = split_attached:match("^(%S+)") + base = word and (word:match("([^/]+)$") or word) + break + elseif tok == "-S" or tok == "--split-string" then + -- Separated split-string: the next token starts the string, i.e. + -- the interpreter — keep walking. elseif tok:find("=", 1, true) then - -- `VAR=value` env assignment, or a `--long=value` option: both + -- `VAR=value` env assignment, or another `--long=value` option: -- self-contained, skip. elseif tok:sub(1, 1) == "-" then -- An option. A few GNU-env short options and their long forms -- consume the *next* token as an operand (`-u NAME`, `-C DIR`, -- `-a NAME`); skip that operand too, or its value is mistaken for - -- the interpreter. `-S`/`--split-string` is deliberately absent: - -- the string it introduces contains the interpreter, which the - -- walk then picks up. An option with an attached operand - -- (`-uNAME`) is one self-contained token and needs no skip. + -- the interpreter. An option with an attached operand (`-uNAME`) + -- is one self-contained token and needs no skip. if tok == "-u" or tok == "-C" or tok == "-a" or tok == "--unset" or tok == "--chdir" or tok == "--argv0" then skip_next = true @@ -159,11 +172,18 @@ end local function attach_for_active_buffer() local buf = pmacs.window.buffer() if not buf then return end - -- Gate dispatch on `_has_language`: the chain above also resolves - -- languages with no grammar (python, javascript), and dispatching one - -- would raise "unknown language" (caught, but noise) — and an - -- extensionless script must never get a wrong-grammar parse tree. - local lang = resolve_active_language(buf) + local key = tostring(buf) + -- Reuse the language pinned at first attach if this buffer already has + -- a parse view. A switch-away/back re-runs this hook (via after-switch); + -- re-resolving there would re-sniff a shebang the user has since edited + -- and silently swap the grammar — and diverge from the LSP side, which + -- keeps its existing attachment across the switch. A first-seen buffer + -- (no view yet) resolves normally. Gate dispatch on `_has_language`: the + -- resolution chain also yields languages with no grammar (python, + -- javascript), and dispatching one would raise "unknown language" + -- (caught, but noise) and never gives a wrong-grammar tree. + local lang = pmacs.parse._has_view(buf) and parse_lang_by_buffer[key] + or resolve_active_language(buf) if not lang or not pmacs.parse._has_language(lang) then return end pmacs.parse._dispatch(buf, lang) -- T M4.3: install the syntax-highlight overlay for this buffer. @@ -174,8 +194,8 @@ local function attach_for_active_buffer() -- table so we only push once per (buffer, after-load) cycle). -- `tostring(buf)` is stable per BufferId (the metamethod -- formats the wrapped id), so it's a safe table-key - -- replacement for a `:id()` method we don't have to expose. - local key = tostring(buf) + -- replacement for a `:id()` method we don't have to expose (`key` is + -- computed once at the top of this function). if not highlighted_buffers[key] then local ok = pmacs.parse._attach_highlight(buf, lang) if ok then highlighted_buffers[key] = true end diff --git a/tests/m4_acceptance.rs b/tests/m4_acceptance.rs index 9dacd83..c03316e 100644 --- a/tests/m4_acceptance.rs +++ b/tests/m4_acceptance.rs @@ -5878,6 +5878,11 @@ fn m4_shebang_resolver_maps_interpreters() { ("#!/usr/bin/env bash\n", "bash"), ("#!/usr/bin/env python3\n", "python"), ("#!/usr/bin/env -S python3 -u\n", "python"), + // Attached split-string forms carry the interpreter inside the + // option token. + ("#!/usr/bin/env -Spython3 -u\n", "python"), + ("#!/usr/bin/env --split-string=python3 -u\n", "python"), + ("#!/usr/bin/env -vSpython3 -u\n", "python"), // GNU-env options that consume an operand must not have the // operand mistaken for the interpreter. ("#!/usr/bin/env -u FOO python3\n", "python"), @@ -6081,6 +6086,33 @@ fn m4_shebang_edit_keeps_pinned_grammar() { Some("bash"), "editing the shebang must not re-switch the pinned grammar" ); + + // Switch away to another buffer and back: the after-switch reattach + // must reuse the pinned bash grammar rather than re-sniff the (now + // lua) shebang — otherwise grammar and LSP diverge, since the LSP side + // keeps its bash attachment across the switch. `switch_buffer` fires + // `buffer.after-switch` synchronously. + let other = dir.path().join("other.txt"); + std::fs::write(&other, b"plain text\n").expect("write other"); + let other_disp = other.display(); + s.lua_host + .lua() + .load(format!( + "local pinned = pmacs.window.buffer() + pmacs.buffer.find_or_open('{other_disp}') + pmacs.window.switch_buffer(pinned)" + )) + .exec() + .expect("switch away and back"); + for _ in 0..64 { + s.tick_async(); + std::thread::sleep(Duration::from_millis(2)); + } + assert_eq!( + current_tree_language(&s).as_deref(), + Some("bash"), + "switch-away/back must reuse the pinned grammar, not re-sniff the edited shebang" + ); let errs: i64 = s .lua_host .lua() From 7479213c3f0eb34569c75f0e84d305e41bd3814a Mon Sep 17 00:00:00 2001 From: Levi Neuwirth Date: Tue, 14 Jul 2026 16:00:47 +0100 Subject: [PATCH 4/4] fix(highlight): env -S payload is a full arg list, not a bare interpreter The attached split-string payload (`-Spython3`, `--split-string=...`) can itself begin with env options or VAR=value assignments before the interpreter: `-S-i python3`, `-SFOO=bar python3`, `--split-string=-u FOO python3`. Rather than taking the payload's first word as the interpreter, re-inject the attached payload into the token stream so it flows through the same option / operand / assignment state machine as a separated payload. Adds the three cases as resolver tests. --- builtin/runtime/syntax.lua | 26 ++++++++++++++++---------- tests/m4_acceptance.rs | 5 +++++ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/builtin/runtime/syntax.lua b/builtin/runtime/syntax.lua index dd97689..7f4c8df 100644 --- a/builtin/runtime/syntax.lua +++ b/builtin/runtime/syntax.lua @@ -98,13 +98,21 @@ function pmacs.parse.language_from_shebang(buf) base = nil local seen_env = false local skip_next = false + local tokens = {} for tok in rest:gmatch("%S+") do - -- `-S`/`--split-string` introduces a string whose FIRST word is the - -- interpreter. GNU env accepts that value ATTACHED — `-Spython3`, - -- `-vSpython3` (after no-operand short flags i/v/0), or - -- `--split-string=python3` — where the interpreter rides inside the - -- option token. Extract it; the separated forms (`-S python3`) are - -- handled by simply walking on to the next token. + tokens[#tokens + 1] = tok + end + local i = 1 + while i <= #tokens do + local tok = tokens[i] + i = i + 1 + -- `-S`/`--split-string` introduces a complete env argument list, + -- not necessarily an interpreter first: it may begin with more env + -- options or `VAR=value` assignments. GNU env accepts that value + -- ATTACHED — `-Spython3`, `-vSpython3` (after no-operand short flags + -- i/v/0), or `--split-string=python3`. Put an attached payload back + -- into this token stream so it goes through the same option/operand/ + -- assignment state machine as a separated payload. local split_attached = tok:match("^%-[iv0]*S(.+)$") or tok:match("^%-%-split%-string=(.+)$") if not seen_env then @@ -112,12 +120,10 @@ function pmacs.parse.language_from_shebang(buf) elseif skip_next then skip_next = false -- the operand consumed by the previous option elseif split_attached then - local word = split_attached:match("^(%S+)") - base = word and (word:match("([^/]+)$") or word) - break + table.insert(tokens, i, split_attached) elseif tok == "-S" or tok == "--split-string" then -- Separated split-string: the next token starts the string, i.e. - -- the interpreter — keep walking. + -- another env argument — keep walking. elseif tok:find("=", 1, true) then -- `VAR=value` env assignment, or another `--long=value` option: -- self-contained, skip. diff --git a/tests/m4_acceptance.rs b/tests/m4_acceptance.rs index c03316e..e9f1b7f 100644 --- a/tests/m4_acceptance.rs +++ b/tests/m4_acceptance.rs @@ -5883,6 +5883,11 @@ fn m4_shebang_resolver_maps_interpreters() { ("#!/usr/bin/env -Spython3 -u\n", "python"), ("#!/usr/bin/env --split-string=python3 -u\n", "python"), ("#!/usr/bin/env -vSpython3 -u\n", "python"), + // The attached split string is a complete env argument list, so + // options and assignments may precede the interpreter within it. + ("#!/usr/bin/env -S-i python3 -u\n", "python"), + ("#!/usr/bin/env -SFOO=bar python3 -u\n", "python"), + ("#!/usr/bin/env --split-string=-u FOO python3\n", "python"), // GNU-env options that consume an operand must not have the // operand mistaken for the interpreter. ("#!/usr/bin/env -u FOO python3\n", "python"),