diff --git a/builtin/runtime/lsp.lua b/builtin/runtime/lsp.lua index 6607423..4961657 100644 --- a/builtin/runtime/lsp.lua +++ b/builtin/runtime/lsp.lua @@ -374,11 +374,15 @@ local function buffer_language(buf) if not ok or not path then return nil end -- Grammar-backed detection first (keeps rust/.rs etc. exactly as -- before); fall back to the LSP-only filetype map so languages - -- with a server but no tree-sitter grammar (Python) still attach. + -- with a server but no tree-sitter grammar (Python) still attach; + -- finally, for an extensionless file, sniff a `#!interp` shebang so + -- e.g. an extensionless `#!/bin/sh` script still attaches its server. local lang = pmacs.parse.language_for_path(path) if lang then return lang end local ext = path:match("%.([%w_]+)$") - return ext and pmacs.lsp.filetypes[ext] or nil + local by_ext = ext and pmacs.lsp.filetypes[ext] + if by_ext then return by_ext end + return pmacs.parse.language_from_shebang(buf) end -- Public: the per-buffer language chain. Auto-pairing resolves -- relevance against the buffer its typed-edit record names — which a diff --git a/builtin/runtime/syntax.lua b/builtin/runtime/syntax.lua index 5f7fcb8..7f4c8df 100644 --- a/builtin/runtime/syntax.lua +++ b/builtin/runtime/syntax.lua @@ -44,19 +44,153 @@ function pmacs.parse._dispatch(buf, lang) return job_id end +-- Shebang → language detection ------------------------------------------ +-- +-- Extension detection (`language_for_path`) misses extensionless scripts +-- (`scripts/deploy`, git hooks, `configure`), which are the common case +-- for shell. This fills that gap by sniffing the first line: `#!interp` +-- maps the interpreter's basename to a language. It is a *fallback* — +-- every caller tries extension detection first, so a `.py`/`.sh` file is +-- never re-classified by a stray shebang. Deliberately does not cover +-- special filenames (`.bashrc`, `Dockerfile`): those wait on grammars for +-- the languages behind them. +-- +-- The map is user-extensible from init.lua, e.g. +-- `pmacs.parse.shebangs.ruby = "ruby"`. Only interpreters whose language +-- pmacs can act on (grammar and/or LSP config) are seeded; an entry whose +-- language has neither is harmless but inert. +pmacs.parse.shebangs = pmacs.parse.shebangs or { + sh = "bash", bash = "bash", dash = "bash", ash = "bash", + ksh = "bash", mksh = "bash", zsh = "bash", + python = "python", python2 = "python", python3 = "python", + pypy = "python", pypy3 = "python", + node = "javascript", nodejs = "javascript", + lua = "lua", luajit = "lua", +} + +-- First line of `buf` (up to 256 bytes), sans trailing newline, or nil. +-- 256 bytes is well past any real shebang and keeps the slice cheap on +-- the hot path (callers only reach here when extension detection missed). +local function first_line(buf) + local ok_len, n = pcall(function() return buf:len() end) + if not ok_len or not n or n <= 0 then return nil end + if n > 256 then n = 256 end + local ok, s = pcall(function() return buf:slice(0, n) end) + if not ok or type(s) ~= "string" or #s == 0 then return nil end + local nl = s:find("\n", 1, true) + if nl then s = s:sub(1, nl - 1) end + return s +end + +-- Resolve the interpreter basename from a shebang line, resolving the +-- `env` indirection (`#!/usr/bin/env python3` → `python3`, skipping +-- `env`'s own options / `VAR=val` assignments). Returns the language +-- name from `pmacs.parse.shebangs`, or nil. +function pmacs.parse.language_from_shebang(buf) + if not buf then return nil end + local line = first_line(buf) + if not line or line:sub(1, 2) ~= "#!" then return nil end + local rest = line:sub(3):gsub("^%s+", "") + local first = rest:match("^(%S+)") + if not first then return nil end + local base = first:match("([^/]+)$") or first + if base == "env" then + base = nil + local seen_env = false + local skip_next = false + local tokens = {} + for tok in rest:gmatch("%S+") do + tokens[#tokens + 1] = tok + end + local i = 1 + while i <= #tokens do + local tok = tokens[i] + i = i + 1 + -- `-S`/`--split-string` introduces a complete env argument list, + -- not necessarily an interpreter first: it may begin with more env + -- options or `VAR=value` assignments. GNU env accepts that value + -- ATTACHED — `-Spython3`, `-vSpython3` (after no-operand short flags + -- i/v/0), or `--split-string=python3`. Put an attached payload back + -- into this token stream so it goes through the same option/operand/ + -- assignment state machine as a separated payload. + local split_attached = + tok:match("^%-[iv0]*S(.+)$") or tok:match("^%-%-split%-string=(.+)$") + if not seen_env then + seen_env = true -- the `env` path token itself + elseif skip_next then + skip_next = false -- the operand consumed by the previous option + elseif split_attached then + table.insert(tokens, i, split_attached) + elseif tok == "-S" or tok == "--split-string" then + -- Separated split-string: the next token starts the string, i.e. + -- another env argument — keep walking. + elseif tok:find("=", 1, true) then + -- `VAR=value` env assignment, or another `--long=value` option: + -- self-contained, skip. + elseif tok:sub(1, 1) == "-" then + -- An option. A few GNU-env short options and their long forms + -- consume the *next* token as an operand (`-u NAME`, `-C DIR`, + -- `-a NAME`); skip that operand too, or its value is mistaken for + -- the interpreter. An option with an attached operand (`-uNAME`) + -- is one self-contained token and needs no skip. + if tok == "-u" or tok == "-C" or tok == "-a" + or tok == "--unset" or tok == "--chdir" or tok == "--argv0" then + skip_next = true + end + else + base = tok:match("([^/]+)$") or tok + break + end + end + if not base then return nil end + end + return pmacs.parse.shebangs[base] +end + -- Set of buffer ids that already have a highlight overlay -- attached, keyed by raw id (number). A buffer that opens, gets -- highlights, gets killed, and is reopened needs a fresh overlay -- attach; the kill path clears the entry below if/when it lands. local highlighted_buffers = {} +-- Filetype-aware language resolution for the active buffer, in +-- precedence order: grammar extension → LSP filetype map → shebang. The +-- shebang is consulted ONLY when the extension is unrecognized (a known +-- non-grammar extension like `.py` must not fall through to a stray +-- `#!/bin/sh` and be misparsed as bash). Keyed on `buf:name()` for the +-- extension parts (matching the historical behavior — path-less buffers +-- that resolve a grammar by name keep working); the shebang reads buffer +-- content directly. +local function resolve_active_language(buf) + local name = buf:name() + if name then + local grammar = pmacs.parse.language_for_path(name) + if grammar then return grammar end + local ext = name:match("%.([%w_]+)$") + local by_ext = ext and pmacs.lsp and pmacs.lsp.filetypes and pmacs.lsp.filetypes[ext] + -- A recognized (even non-grammar) extension is authoritative; do not + -- consult the shebang for it. + if by_ext then return by_ext end + end + return pmacs.parse.language_from_shebang(buf) +end + local function attach_for_active_buffer() local buf = pmacs.window.buffer() if not buf then return end - local path = buf:name() - if not path then return end - local lang = pmacs.parse.language_for_path(path) - if not lang then return end + local key = tostring(buf) + -- Reuse the language pinned at first attach if this buffer already has + -- a parse view. A switch-away/back re-runs this hook (via after-switch); + -- re-resolving there would re-sniff a shebang the user has since edited + -- and silently swap the grammar — and diverge from the LSP side, which + -- keeps its existing attachment across the switch. A first-seen buffer + -- (no view yet) resolves normally. Gate dispatch on `_has_language`: the + -- resolution chain also yields languages with no grammar (python, + -- javascript), and dispatching one would raise "unknown language" + -- (caught, but noise) and never gives a wrong-grammar tree. + local lang = pmacs.parse._has_view(buf) and parse_lang_by_buffer[key] + or resolve_active_language(buf) + if not lang or not pmacs.parse._has_language(lang) then return end pmacs.parse._dispatch(buf, lang) -- T M4.3: install the syntax-highlight overlay for this buffer. -- Idempotent --- repeated calls for the same buffer are a no-op @@ -66,8 +200,8 @@ local function attach_for_active_buffer() -- table so we only push once per (buffer, after-load) cycle). -- `tostring(buf)` is stable per BufferId (the metamethod -- formats the wrapped id), so it's a safe table-key - -- replacement for a `:id()` method we don't have to expose. - local key = tostring(buf) + -- replacement for a `:id()` method we don't have to expose (`key` is + -- computed once at the top of this function). if not highlighted_buffers[key] then local ok = pmacs.parse._attach_highlight(buf, lang) if ok then highlighted_buffers[key] = true end @@ -108,9 +242,14 @@ local function reparse_active_buffer_after_edit() if not pmacs.parse._has_view(buf) then return end local pending = pmacs.parse._pending_edits(buf) if not pending or pending == 0 then return end - local path = buf:name() - if not path then return end - local lang = pmacs.parse.language_for_path(path) + -- Reparse with the language pinned when the view was first attached --- + -- never re-resolve from the path or (mutable) shebang. The Rust side is + -- "first language wins"; re-sniffing a shebang the user just edited + -- would either raise "unknown language" (sh → python) or swap the parse + -- tree to a new grammar while the highlight overlay still holds the + -- original grammar's query (sh → lua). Language changes need a + -- close/reopen, exactly as they do for a renamed extension. + local lang = parse_lang_by_buffer[tostring(buf)] if not lang then return end pmacs.parse._dispatch(buf, lang) end diff --git a/tests/m4_acceptance.rs b/tests/m4_acceptance.rs index c648bcc..e9f1b7f 100644 --- a/tests/m4_acceptance.rs +++ b/tests/m4_acceptance.rs @@ -5849,6 +5849,284 @@ fn m4_12_default_bundle_wires_bash() { assert_eq!(probe.get::("grammar_bats").unwrap(), "bash"); } +/// Shebang detection: `pmacs.parse.language_from_shebang` maps the +/// interpreter basename (resolving the `#!/usr/bin/env` indirection) to a +/// language, and returns nil for non-shebangs and unmapped interpreters. +/// This is the fallback that lets extensionless scripts (`scripts/deploy`, +/// git hooks, `configure`) resolve a language at all — extension +/// detection misses them. +#[test] +fn m4_shebang_resolver_maps_interpreters() { + use pmacs::editor::EditorState; + let s = EditorState::new(); + let resolve = |first_line: &str| -> Option { + s.lua_host + .lua() + .load(format!( + "local b = pmacs.window.buffer() + if b:len() > 0 then b:delete(0, b:len()) end + b:insert(0, {first_line:?}) + return pmacs.parse.language_from_shebang(b)" + )) + .eval() + .expect("resolve shebang") + }; + for (line, want) in [ + ("#!/bin/sh\n", "bash"), + ("#!/bin/bash -e\n", "bash"), + ("#! /bin/zsh\n", "bash"), + ("#!/usr/bin/env bash\n", "bash"), + ("#!/usr/bin/env python3\n", "python"), + ("#!/usr/bin/env -S python3 -u\n", "python"), + // Attached split-string forms carry the interpreter inside the + // option token. + ("#!/usr/bin/env -Spython3 -u\n", "python"), + ("#!/usr/bin/env --split-string=python3 -u\n", "python"), + ("#!/usr/bin/env -vSpython3 -u\n", "python"), + // The attached split string is a complete env argument list, so + // options and assignments may precede the interpreter within it. + ("#!/usr/bin/env -S-i python3 -u\n", "python"), + ("#!/usr/bin/env -SFOO=bar python3 -u\n", "python"), + ("#!/usr/bin/env --split-string=-u FOO python3\n", "python"), + // GNU-env options that consume an operand must not have the + // operand mistaken for the interpreter. + ("#!/usr/bin/env -u FOO python3\n", "python"), + ("#!/usr/bin/env -C /tmp python3\n", "python"), + ("#!/usr/bin/env -u FOO -C /tmp node\n", "javascript"), + ("#!/usr/bin/node\n", "javascript"), + ("#!/usr/bin/env lua\n", "lua"), + ] { + assert_eq!(resolve(line).as_deref(), Some(want), "{line:?}"); + } + for line in [ + "echo hi\n", + "# just a comment\n", + "#!/usr/bin/env ruby\n", // interpreter not in the seeded map + "\n", + "", + ] { + assert_eq!(resolve(line), None, "{line:?}"); + } +} + +/// End-to-end: opening an extensionless `#!/bin/sh` script resolves to +/// `bash` on both paths — `lsp.lua`'s `buffer_language` chain (so the +/// server would attach) and `syntax.lua`'s grammar attach (so a bash +/// parse tree is produced). `pmacs.lsp.config` is emptied first so the +/// real bash-language-server isn't spawned; grammar detection is +/// independent of the LSP config. +#[test] +fn m4_shebang_extensionless_script_resolves_bash() { + use pmacs::editor::EditorState; + let mut s = EditorState::new(); + let dir = tempfile::tempdir().expect("tempdir"); + let hook = dir.path().join("pre-commit"); // no extension + std::fs::write(&hook, b"#!/bin/sh\nset -e\necho building\n").expect("write"); + let hook_disp = hook.display(); + + s.lua_host + .lua() + .load(format!( + "pmacs.lsp.config = {{}} + pmacs.buffer.find_or_open('{hook_disp}')" + )) + .exec() + .expect("open extensionless shebang script"); + + let lsp_lang: Option = s + .lua_host + .lua() + .load("return pmacs.lsp.active_buffer_language()") + .eval() + .expect("lsp language"); + assert_eq!( + lsp_lang.as_deref(), + Some("bash"), + "extensionless #!/bin/sh resolves to bash for LSP" + ); + + pump_async(&mut s, |st| current_tree_language(st).is_some()); + assert_eq!( + current_tree_language(&s).as_deref(), + Some("bash"), + "extensionless #!/bin/sh gets a bash parse tree" + ); +} + +/// Precedence: a recognized extension always wins over file content, so a +/// `.py` file that happens to open with `#!/bin/sh` still resolves to +/// python — the shebang is consulted only when extension detection misses. +#[test] +fn m4_shebang_does_not_override_extension() { + use pmacs::editor::EditorState; + let s = EditorState::new(); + let dir = tempfile::tempdir().expect("tempdir"); + let f = dir.path().join("tool.py"); + std::fs::write(&f, b"#!/bin/sh\nprint('hi')\n").expect("write"); + let f_disp = f.display(); + s.lua_host + .lua() + .load(format!( + "pmacs.lsp.config = {{}} + pmacs.buffer.find_or_open('{f_disp}')" + )) + .exec() + .expect("open .py with a shell shebang"); + // Both the LSP language *and* the grammar decision must respect the + // extension: python for LSP, and NO grammar parse view (python has no + // grammar) — not a bash tree installed from the `#!/bin/sh` line. + // `_has_view` is set synchronously by `_dispatch`, so no pump is + // needed; without the precedence fix the shebang would have dispatched + // bash and this would be true. + let (lang, has_view): (Option, bool) = s + .lua_host + .lua() + .load( + "return pmacs.lsp.active_buffer_language(), + pmacs.parse._has_view(pmacs.window.buffer())", + ) + .eval() + .expect("language + view"); + assert_eq!( + lang.as_deref(), + Some("python"), + ".py extension wins over a #!/bin/sh shebang (LSP)" + ); + assert!( + !has_view, + ".py file must not get a grammar parse view from a #!/bin/sh line" + ); +} + +/// Finding-1 gate: an extensionless `#!/usr/bin/env python3` script +/// resolves to python for LSP, but python has no grammar — syntax must +/// skip it *silently*. Without the `_has_language` gate, `_dispatch` +/// raises "unknown language: python" (caught by the after-load pcall and +/// reported through `pmacs.error`), which we assert does NOT happen. +#[test] +fn m4_shebang_extensionless_grammarless_language_is_silent() { + use pmacs::editor::EditorState; + let s = EditorState::new(); + let dir = tempfile::tempdir().expect("tempdir"); + let f = dir.path().join("generate"); // no extension + std::fs::write(&f, b"#!/usr/bin/env python3\nprint('hi')\n").expect("write"); + let f_disp = f.display(); + s.lua_host + .lua() + .load(format!( + "pmacs.lsp.config = {{}} + _G.__errs = {{}} + local real = pmacs.error + pmacs.error = function(m) table.insert(_G.__errs, m) end + pmacs.buffer.find_or_open('{f_disp}')" + )) + .exec() + .expect("open extensionless python script"); + let (lang, has_view, errs): (Option, bool, i64) = s + .lua_host + .lua() + .load( + "return pmacs.lsp.active_buffer_language(), + pmacs.parse._has_view(pmacs.window.buffer()), + #_G.__errs", + ) + .eval() + .expect("probe"); + assert_eq!(lang.as_deref(), Some("python"), "python resolves for LSP"); + assert!( + !has_view, + "no grammar parse view for a grammarless language" + ); + assert_eq!(errs, 0, "no 'unknown language' error reported"); +} + +/// Finding-2 pin: editing an open extensionless script's shebang must not +/// re-switch the parse grammar. A `#!/bin/sh` script attaches the bash +/// grammar; rewriting its shebang to lua and firing after-edit must keep +/// the bash tree (the pinned grammar) rather than swap in lua under the +/// stale highlight overlay — and must not error. +#[test] +fn m4_shebang_edit_keeps_pinned_grammar() { + use pmacs::editor::EditorState; + let mut s = EditorState::new(); + let dir = tempfile::tempdir().expect("tempdir"); + let hook = dir.path().join("deploy"); // no extension + std::fs::write(&hook, b"#!/bin/sh\necho one\n").expect("write"); + let hook_disp = hook.display(); + s.lua_host + .lua() + .load(format!( + "pmacs.lsp.config = {{}} + _G.__errs = {{}} + local real = pmacs.error + pmacs.error = function(m) table.insert(_G.__errs, m) end + pmacs.buffer.find_or_open('{hook_disp}')" + )) + .exec() + .expect("open extensionless shell script"); + pump_async(&mut s, |st| { + current_tree_language(st).as_deref() == Some("bash") + }); + + // Rewrite the first line to a lua shebang, then fire after-edit. + s.lua_host + .lua() + .load( + "local b = pmacs.window.buffer() + local text = b:slice(0, b:len()) + local first_len = (text:find('\\n', 1, true) or 1) - 1 + b:replace(0, first_len, '#!/usr/bin/env lua') + pmacs.hook.run('buffer.after-edit')", + ) + .exec() + .expect("rewrite shebang to lua"); + // Let the reparse settle (manual ticks: the tree stays bash with the + // pin, so a `pump_async` for a language *change* would time out). + for _ in 0..64 { + s.tick_async(); + std::thread::sleep(Duration::from_millis(2)); + } + assert_eq!( + current_tree_language(&s).as_deref(), + Some("bash"), + "editing the shebang must not re-switch the pinned grammar" + ); + + // Switch away to another buffer and back: the after-switch reattach + // must reuse the pinned bash grammar rather than re-sniff the (now + // lua) shebang — otherwise grammar and LSP diverge, since the LSP side + // keeps its bash attachment across the switch. `switch_buffer` fires + // `buffer.after-switch` synchronously. + let other = dir.path().join("other.txt"); + std::fs::write(&other, b"plain text\n").expect("write other"); + let other_disp = other.display(); + s.lua_host + .lua() + .load(format!( + "local pinned = pmacs.window.buffer() + pmacs.buffer.find_or_open('{other_disp}') + pmacs.window.switch_buffer(pinned)" + )) + .exec() + .expect("switch away and back"); + for _ in 0..64 { + s.tick_async(); + std::thread::sleep(Duration::from_millis(2)); + } + assert_eq!( + current_tree_language(&s).as_deref(), + Some("bash"), + "switch-away/back must reuse the pinned grammar, not re-sniff the edited shebang" + ); + let errs: i64 = s + .lua_host + .lua() + .load("return #_G.__errs") + .eval() + .expect("errs"); + assert_eq!(errs, 0, "reparse with the pinned grammar reports no error"); +} + /// Typing-perf: the default bundle coalesces full-document /// `didChange` notifications instead of sending one per keystroke /// (each send copies the whole buffer several times and writes