Merge pull request #116 from levineuwirth/shebang-detection

feat(highlight): shebang-based language detection for extensionless scripts
This commit is contained in:
Levi Neuwirth 2026-07-14 15:00:58 +00:00 committed by GitHub
commit 13dbadd0b6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 432 additions and 11 deletions

View File

@ -374,11 +374,15 @@ local function buffer_language(buf)
if not ok or not path then return nil end
-- Grammar-backed detection first (keeps rust/.rs etc. exactly as
-- before); fall back to the LSP-only filetype map so languages
-- with a server but no tree-sitter grammar (Python) still attach.
-- with a server but no tree-sitter grammar (Python) still attach;
-- finally, for an extensionless file, sniff a `#!interp` shebang so
-- e.g. an extensionless `#!/bin/sh` script still attaches its server.
local lang = pmacs.parse.language_for_path(path)
if lang then return lang end
local ext = path:match("%.([%w_]+)$")
return ext and pmacs.lsp.filetypes[ext] or nil
local by_ext = ext and pmacs.lsp.filetypes[ext]
if by_ext then return by_ext end
return pmacs.parse.language_from_shebang(buf)
end
-- Public: the per-buffer language chain. Auto-pairing resolves
-- relevance against the buffer its typed-edit record names — which a

View File

@ -44,19 +44,153 @@ function pmacs.parse._dispatch(buf, lang)
return job_id
end
-- Shebang → language detection ------------------------------------------
--
-- Extension detection (`language_for_path`) misses extensionless scripts
-- (`scripts/deploy`, git hooks, `configure`), which are the common case
-- for shell. This fills that gap by sniffing the first line: `#!interp`
-- maps the interpreter's basename to a language. It is a *fallback* —
-- every caller tries extension detection first, so a `.py`/`.sh` file is
-- never re-classified by a stray shebang. Deliberately does not cover
-- special filenames (`.bashrc`, `Dockerfile`): those wait on grammars for
-- the languages behind them.
--
-- The map is user-extensible from init.lua, e.g.
-- `pmacs.parse.shebangs.ruby = "ruby"`. Only interpreters whose language
-- pmacs can act on (grammar and/or LSP config) are seeded; an entry whose
-- language has neither is harmless but inert.
pmacs.parse.shebangs = pmacs.parse.shebangs or {
sh = "bash", bash = "bash", dash = "bash", ash = "bash",
ksh = "bash", mksh = "bash", zsh = "bash",
python = "python", python2 = "python", python3 = "python",
pypy = "python", pypy3 = "python",
node = "javascript", nodejs = "javascript",
lua = "lua", luajit = "lua",
}
-- First line of `buf` (up to 256 bytes), sans trailing newline, or nil.
-- 256 bytes is well past any real shebang and keeps the slice cheap on
-- the hot path (callers only reach here when extension detection missed).
local function first_line(buf)
local ok_len, n = pcall(function() return buf:len() end)
if not ok_len or not n or n <= 0 then return nil end
if n > 256 then n = 256 end
local ok, s = pcall(function() return buf:slice(0, n) end)
if not ok or type(s) ~= "string" or #s == 0 then return nil end
local nl = s:find("\n", 1, true)
if nl then s = s:sub(1, nl - 1) end
return s
end
-- Resolve the interpreter basename from a shebang line, resolving the
-- `env` indirection (`#!/usr/bin/env python3` → `python3`, skipping
-- `env`'s own options / `VAR=val` assignments). Returns the language
-- name from `pmacs.parse.shebangs`, or nil.
function pmacs.parse.language_from_shebang(buf)
if not buf then return nil end
local line = first_line(buf)
if not line or line:sub(1, 2) ~= "#!" then return nil end
local rest = line:sub(3):gsub("^%s+", "")
local first = rest:match("^(%S+)")
if not first then return nil end
local base = first:match("([^/]+)$") or first
if base == "env" then
base = nil
local seen_env = false
local skip_next = false
local tokens = {}
for tok in rest:gmatch("%S+") do
tokens[#tokens + 1] = tok
end
local i = 1
while i <= #tokens do
local tok = tokens[i]
i = i + 1
-- `-S`/`--split-string` introduces a complete env argument list,
-- not necessarily an interpreter first: it may begin with more env
-- options or `VAR=value` assignments. GNU env accepts that value
-- ATTACHED — `-Spython3`, `-vSpython3` (after no-operand short flags
-- i/v/0), or `--split-string=python3`. Put an attached payload back
-- into this token stream so it goes through the same option/operand/
-- assignment state machine as a separated payload.
local split_attached =
tok:match("^%-[iv0]*S(.+)$") or tok:match("^%-%-split%-string=(.+)$")
if not seen_env then
seen_env = true -- the `env` path token itself
elseif skip_next then
skip_next = false -- the operand consumed by the previous option
elseif split_attached then
table.insert(tokens, i, split_attached)
elseif tok == "-S" or tok == "--split-string" then
-- Separated split-string: the next token starts the string, i.e.
-- another env argument — keep walking.
elseif tok:find("=", 1, true) then
-- `VAR=value` env assignment, or another `--long=value` option:
-- self-contained, skip.
elseif tok:sub(1, 1) == "-" then
-- An option. A few GNU-env short options and their long forms
-- consume the *next* token as an operand (`-u NAME`, `-C DIR`,
-- `-a NAME`); skip that operand too, or its value is mistaken for
-- the interpreter. An option with an attached operand (`-uNAME`)
-- is one self-contained token and needs no skip.
if tok == "-u" or tok == "-C" or tok == "-a"
or tok == "--unset" or tok == "--chdir" or tok == "--argv0" then
skip_next = true
end
else
base = tok:match("([^/]+)$") or tok
break
end
end
if not base then return nil end
end
return pmacs.parse.shebangs[base]
end
-- Set of buffer ids that already have a highlight overlay
-- attached, keyed by raw id (number). A buffer that opens, gets
-- highlights, gets killed, and is reopened needs a fresh overlay
-- attach; the kill path clears the entry below if/when it lands.
local highlighted_buffers = {}
-- Filetype-aware language resolution for the active buffer, in
-- precedence order: grammar extension → LSP filetype map → shebang. The
-- shebang is consulted ONLY when the extension is unrecognized (a known
-- non-grammar extension like `.py` must not fall through to a stray
-- `#!/bin/sh` and be misparsed as bash). Keyed on `buf:name()` for the
-- extension parts (matching the historical behavior — path-less buffers
-- that resolve a grammar by name keep working); the shebang reads buffer
-- content directly.
local function resolve_active_language(buf)
local name = buf:name()
if name then
local grammar = pmacs.parse.language_for_path(name)
if grammar then return grammar end
local ext = name:match("%.([%w_]+)$")
local by_ext = ext and pmacs.lsp and pmacs.lsp.filetypes and pmacs.lsp.filetypes[ext]
-- A recognized (even non-grammar) extension is authoritative; do not
-- consult the shebang for it.
if by_ext then return by_ext end
end
return pmacs.parse.language_from_shebang(buf)
end
local function attach_for_active_buffer()
local buf = pmacs.window.buffer()
if not buf then return end
local path = buf:name()
if not path then return end
local lang = pmacs.parse.language_for_path(path)
if not lang then return end
local key = tostring(buf)
-- Reuse the language pinned at first attach if this buffer already has
-- a parse view. A switch-away/back re-runs this hook (via after-switch);
-- re-resolving there would re-sniff a shebang the user has since edited
-- and silently swap the grammar — and diverge from the LSP side, which
-- keeps its existing attachment across the switch. A first-seen buffer
-- (no view yet) resolves normally. Gate dispatch on `_has_language`: the
-- resolution chain also yields languages with no grammar (python,
-- javascript), and dispatching one would raise "unknown language"
-- (caught, but noise) and never gives a wrong-grammar tree.
local lang = pmacs.parse._has_view(buf) and parse_lang_by_buffer[key]
or resolve_active_language(buf)
if not lang or not pmacs.parse._has_language(lang) then return end
pmacs.parse._dispatch(buf, lang)
-- T M4.3: install the syntax-highlight overlay for this buffer.
-- Idempotent --- repeated calls for the same buffer are a no-op
@ -66,8 +200,8 @@ local function attach_for_active_buffer()
-- table so we only push once per (buffer, after-load) cycle).
-- `tostring(buf)` is stable per BufferId (the metamethod
-- formats the wrapped id), so it's a safe table-key
-- replacement for a `:id()` method we don't have to expose.
local key = tostring(buf)
-- replacement for a `:id()` method we don't have to expose (`key` is
-- computed once at the top of this function).
if not highlighted_buffers[key] then
local ok = pmacs.parse._attach_highlight(buf, lang)
if ok then highlighted_buffers[key] = true end
@ -108,9 +242,14 @@ local function reparse_active_buffer_after_edit()
if not pmacs.parse._has_view(buf) then return end
local pending = pmacs.parse._pending_edits(buf)
if not pending or pending == 0 then return end
local path = buf:name()
if not path then return end
local lang = pmacs.parse.language_for_path(path)
-- Reparse with the language pinned when the view was first attached ---
-- never re-resolve from the path or (mutable) shebang. The Rust side is
-- "first language wins"; re-sniffing a shebang the user just edited
-- would either raise "unknown language" (sh → python) or swap the parse
-- tree to a new grammar while the highlight overlay still holds the
-- original grammar's query (sh → lua). Language changes need a
-- close/reopen, exactly as they do for a renamed extension.
local lang = parse_lang_by_buffer[tostring(buf)]
if not lang then return end
pmacs.parse._dispatch(buf, lang)
end

View File

@ -5849,6 +5849,284 @@ fn m4_12_default_bundle_wires_bash() {
assert_eq!(probe.get::<String>("grammar_bats").unwrap(), "bash");
}
/// Shebang detection: `pmacs.parse.language_from_shebang` maps the
/// interpreter basename (resolving the `#!/usr/bin/env` indirection) to a
/// language, and returns nil for non-shebangs and unmapped interpreters.
/// This is the fallback that lets extensionless scripts (`scripts/deploy`,
/// git hooks, `configure`) resolve a language at all — extension
/// detection misses them.
#[test]
fn m4_shebang_resolver_maps_interpreters() {
use pmacs::editor::EditorState;
let s = EditorState::new();
let resolve = |first_line: &str| -> Option<String> {
s.lua_host
.lua()
.load(format!(
"local b = pmacs.window.buffer()
if b:len() > 0 then b:delete(0, b:len()) end
b:insert(0, {first_line:?})
return pmacs.parse.language_from_shebang(b)"
))
.eval()
.expect("resolve shebang")
};
for (line, want) in [
("#!/bin/sh\n", "bash"),
("#!/bin/bash -e\n", "bash"),
("#! /bin/zsh\n", "bash"),
("#!/usr/bin/env bash\n", "bash"),
("#!/usr/bin/env python3\n", "python"),
("#!/usr/bin/env -S python3 -u\n", "python"),
// Attached split-string forms carry the interpreter inside the
// option token.
("#!/usr/bin/env -Spython3 -u\n", "python"),
("#!/usr/bin/env --split-string=python3 -u\n", "python"),
("#!/usr/bin/env -vSpython3 -u\n", "python"),
// The attached split string is a complete env argument list, so
// options and assignments may precede the interpreter within it.
("#!/usr/bin/env -S-i python3 -u\n", "python"),
("#!/usr/bin/env -SFOO=bar python3 -u\n", "python"),
("#!/usr/bin/env --split-string=-u FOO python3\n", "python"),
// GNU-env options that consume an operand must not have the
// operand mistaken for the interpreter.
("#!/usr/bin/env -u FOO python3\n", "python"),
("#!/usr/bin/env -C /tmp python3\n", "python"),
("#!/usr/bin/env -u FOO -C /tmp node\n", "javascript"),
("#!/usr/bin/node\n", "javascript"),
("#!/usr/bin/env lua\n", "lua"),
] {
assert_eq!(resolve(line).as_deref(), Some(want), "{line:?}");
}
for line in [
"echo hi\n",
"# just a comment\n",
"#!/usr/bin/env ruby\n", // interpreter not in the seeded map
"\n",
"",
] {
assert_eq!(resolve(line), None, "{line:?}");
}
}
/// End-to-end: opening an extensionless `#!/bin/sh` script resolves to
/// `bash` on both paths — `lsp.lua`'s `buffer_language` chain (so the
/// server would attach) and `syntax.lua`'s grammar attach (so a bash
/// parse tree is produced). `pmacs.lsp.config` is emptied first so the
/// real bash-language-server isn't spawned; grammar detection is
/// independent of the LSP config.
#[test]
fn m4_shebang_extensionless_script_resolves_bash() {
use pmacs::editor::EditorState;
let mut s = EditorState::new();
let dir = tempfile::tempdir().expect("tempdir");
let hook = dir.path().join("pre-commit"); // no extension
std::fs::write(&hook, b"#!/bin/sh\nset -e\necho building\n").expect("write");
let hook_disp = hook.display();
s.lua_host
.lua()
.load(format!(
"pmacs.lsp.config = {{}}
pmacs.buffer.find_or_open('{hook_disp}')"
))
.exec()
.expect("open extensionless shebang script");
let lsp_lang: Option<String> = s
.lua_host
.lua()
.load("return pmacs.lsp.active_buffer_language()")
.eval()
.expect("lsp language");
assert_eq!(
lsp_lang.as_deref(),
Some("bash"),
"extensionless #!/bin/sh resolves to bash for LSP"
);
pump_async(&mut s, |st| current_tree_language(st).is_some());
assert_eq!(
current_tree_language(&s).as_deref(),
Some("bash"),
"extensionless #!/bin/sh gets a bash parse tree"
);
}
/// Precedence: a recognized extension always wins over file content, so a
/// `.py` file that happens to open with `#!/bin/sh` still resolves to
/// python — the shebang is consulted only when extension detection misses.
#[test]
fn m4_shebang_does_not_override_extension() {
use pmacs::editor::EditorState;
let s = EditorState::new();
let dir = tempfile::tempdir().expect("tempdir");
let f = dir.path().join("tool.py");
std::fs::write(&f, b"#!/bin/sh\nprint('hi')\n").expect("write");
let f_disp = f.display();
s.lua_host
.lua()
.load(format!(
"pmacs.lsp.config = {{}}
pmacs.buffer.find_or_open('{f_disp}')"
))
.exec()
.expect("open .py with a shell shebang");
// Both the LSP language *and* the grammar decision must respect the
// extension: python for LSP, and NO grammar parse view (python has no
// grammar) — not a bash tree installed from the `#!/bin/sh` line.
// `_has_view` is set synchronously by `_dispatch`, so no pump is
// needed; without the precedence fix the shebang would have dispatched
// bash and this would be true.
let (lang, has_view): (Option<String>, bool) = s
.lua_host
.lua()
.load(
"return pmacs.lsp.active_buffer_language(),
pmacs.parse._has_view(pmacs.window.buffer())",
)
.eval()
.expect("language + view");
assert_eq!(
lang.as_deref(),
Some("python"),
".py extension wins over a #!/bin/sh shebang (LSP)"
);
assert!(
!has_view,
".py file must not get a grammar parse view from a #!/bin/sh line"
);
}
/// Finding-1 gate: an extensionless `#!/usr/bin/env python3` script
/// resolves to python for LSP, but python has no grammar — syntax must
/// skip it *silently*. Without the `_has_language` gate, `_dispatch`
/// raises "unknown language: python" (caught by the after-load pcall and
/// reported through `pmacs.error`), which we assert does NOT happen.
#[test]
fn m4_shebang_extensionless_grammarless_language_is_silent() {
use pmacs::editor::EditorState;
let s = EditorState::new();
let dir = tempfile::tempdir().expect("tempdir");
let f = dir.path().join("generate"); // no extension
std::fs::write(&f, b"#!/usr/bin/env python3\nprint('hi')\n").expect("write");
let f_disp = f.display();
s.lua_host
.lua()
.load(format!(
"pmacs.lsp.config = {{}}
_G.__errs = {{}}
local real = pmacs.error
pmacs.error = function(m) table.insert(_G.__errs, m) end
pmacs.buffer.find_or_open('{f_disp}')"
))
.exec()
.expect("open extensionless python script");
let (lang, has_view, errs): (Option<String>, bool, i64) = s
.lua_host
.lua()
.load(
"return pmacs.lsp.active_buffer_language(),
pmacs.parse._has_view(pmacs.window.buffer()),
#_G.__errs",
)
.eval()
.expect("probe");
assert_eq!(lang.as_deref(), Some("python"), "python resolves for LSP");
assert!(
!has_view,
"no grammar parse view for a grammarless language"
);
assert_eq!(errs, 0, "no 'unknown language' error reported");
}
/// Finding-2 pin: editing an open extensionless script's shebang must not
/// re-switch the parse grammar. A `#!/bin/sh` script attaches the bash
/// grammar; rewriting its shebang to lua and firing after-edit must keep
/// the bash tree (the pinned grammar) rather than swap in lua under the
/// stale highlight overlay — and must not error.
#[test]
fn m4_shebang_edit_keeps_pinned_grammar() {
use pmacs::editor::EditorState;
let mut s = EditorState::new();
let dir = tempfile::tempdir().expect("tempdir");
let hook = dir.path().join("deploy"); // no extension
std::fs::write(&hook, b"#!/bin/sh\necho one\n").expect("write");
let hook_disp = hook.display();
s.lua_host
.lua()
.load(format!(
"pmacs.lsp.config = {{}}
_G.__errs = {{}}
local real = pmacs.error
pmacs.error = function(m) table.insert(_G.__errs, m) end
pmacs.buffer.find_or_open('{hook_disp}')"
))
.exec()
.expect("open extensionless shell script");
pump_async(&mut s, |st| {
current_tree_language(st).as_deref() == Some("bash")
});
// Rewrite the first line to a lua shebang, then fire after-edit.
s.lua_host
.lua()
.load(
"local b = pmacs.window.buffer()
local text = b:slice(0, b:len())
local first_len = (text:find('\\n', 1, true) or 1) - 1
b:replace(0, first_len, '#!/usr/bin/env lua')
pmacs.hook.run('buffer.after-edit')",
)
.exec()
.expect("rewrite shebang to lua");
// Let the reparse settle (manual ticks: the tree stays bash with the
// pin, so a `pump_async` for a language *change* would time out).
for _ in 0..64 {
s.tick_async();
std::thread::sleep(Duration::from_millis(2));
}
assert_eq!(
current_tree_language(&s).as_deref(),
Some("bash"),
"editing the shebang must not re-switch the pinned grammar"
);
// Switch away to another buffer and back: the after-switch reattach
// must reuse the pinned bash grammar rather than re-sniff the (now
// lua) shebang — otherwise grammar and LSP diverge, since the LSP side
// keeps its bash attachment across the switch. `switch_buffer` fires
// `buffer.after-switch` synchronously.
let other = dir.path().join("other.txt");
std::fs::write(&other, b"plain text\n").expect("write other");
let other_disp = other.display();
s.lua_host
.lua()
.load(format!(
"local pinned = pmacs.window.buffer()
pmacs.buffer.find_or_open('{other_disp}')
pmacs.window.switch_buffer(pinned)"
))
.exec()
.expect("switch away and back");
for _ in 0..64 {
s.tick_async();
std::thread::sleep(Duration::from_millis(2));
}
assert_eq!(
current_tree_language(&s).as_deref(),
Some("bash"),
"switch-away/back must reuse the pinned grammar, not re-sniff the edited shebang"
);
let errs: i64 = s
.lua_host
.lua()
.load("return #_G.__errs")
.eval()
.expect("errs");
assert_eq!(errs, 0, "reparse with the pinned grammar reports no error");
}
/// Typing-perf: the default bundle coalesces full-document
/// `didChange` notifications instead of sending one per keystroke
/// (each send copies the whole buffer several times and writes