Merge pull request #116 from levineuwirth/shebang-detection

feat(highlight): shebang-based language detection for extensionless scripts

builtin/runtime/lsp.lua

@@ -374,11 +374,15 @@ local function buffer_language(buf)
   if not ok or not path then return nil end
   -- Grammar-backed detection first (keeps rust/.rs etc. exactly as
   -- before); fall back to the LSP-only filetype map so languages
-  -- with a server but no tree-sitter grammar (Python) still attach.
+  -- with a server but no tree-sitter grammar (Python) still attach;
+  -- finally, for an extensionless file, sniff a `#!interp` shebang so
+  -- e.g. an extensionless `#!/bin/sh` script still attaches its server.
   local lang = pmacs.parse.language_for_path(path)
   if lang then return lang end
   local ext = path:match("%.([%w_]+)$")
-  return ext and pmacs.lsp.filetypes[ext] or nil
+  local by_ext = ext and pmacs.lsp.filetypes[ext]
+  if by_ext then return by_ext end
+  return pmacs.parse.language_from_shebang(buf)
 end
 -- Public: the per-buffer language chain. Auto-pairing resolves
 -- relevance against the buffer its typed-edit record names — which a

builtin/runtime/syntax.lua

@@ -44,19 +44,153 @@ function pmacs.parse._dispatch(buf, lang)
   return job_id
 end
 
+-- Shebang → language detection ------------------------------------------
+--
+-- Extension detection (`language_for_path`) misses extensionless scripts
+-- (`scripts/deploy`, git hooks, `configure`), which are the common case
+-- for shell. This fills that gap by sniffing the first line: `#!interp`
+-- maps the interpreter's basename to a language. It is a *fallback* —
+-- every caller tries extension detection first, so a `.py`/`.sh` file is
+-- never re-classified by a stray shebang. Deliberately does not cover
+-- special filenames (`.bashrc`, `Dockerfile`): those wait on grammars for
+-- the languages behind them.
+--
+-- The map is user-extensible from init.lua, e.g.
+-- `pmacs.parse.shebangs.ruby = "ruby"`. Only interpreters whose language
+-- pmacs can act on (grammar and/or LSP config) are seeded; an entry whose
+-- language has neither is harmless but inert.
+pmacs.parse.shebangs = pmacs.parse.shebangs or {
+  sh = "bash", bash = "bash", dash = "bash", ash = "bash",
+  ksh = "bash", mksh = "bash", zsh = "bash",
+  python = "python", python2 = "python", python3 = "python",
+  pypy = "python", pypy3 = "python",
+  node = "javascript", nodejs = "javascript",
+  lua = "lua", luajit = "lua",
+}
+
+-- First line of `buf` (up to 256 bytes), sans trailing newline, or nil.
+-- 256 bytes is well past any real shebang and keeps the slice cheap on
+-- the hot path (callers only reach here when extension detection missed).
+local function first_line(buf)
+  local ok_len, n = pcall(function() return buf:len() end)
+  if not ok_len or not n or n <= 0 then return nil end
+  if n > 256 then n = 256 end
+  local ok, s = pcall(function() return buf:slice(0, n) end)
+  if not ok or type(s) ~= "string" or #s == 0 then return nil end
+  local nl = s:find("\n", 1, true)
+  if nl then s = s:sub(1, nl - 1) end
+  return s
+end
+
+-- Resolve the interpreter basename from a shebang line, resolving the
+-- `env` indirection (`#!/usr/bin/env python3` → `python3`, skipping
+-- `env`'s own options / `VAR=val` assignments). Returns the language
+-- name from `pmacs.parse.shebangs`, or nil.
+function pmacs.parse.language_from_shebang(buf)
+  if not buf then return nil end
+  local line = first_line(buf)
+  if not line or line:sub(1, 2) ~= "#!" then return nil end
+  local rest = line:sub(3):gsub("^%s+", "")
+  local first = rest:match("^(%S+)")
+  if not first then return nil end
+  local base = first:match("([^/]+)$") or first
+  if base == "env" then
+    base = nil
+    local seen_env = false
+    local skip_next = false
+    local tokens = {}
+    for tok in rest:gmatch("%S+") do
+      tokens[#tokens + 1] = tok
+    end
+    local i = 1
+    while i <= #tokens do
+      local tok = tokens[i]
+      i = i + 1
+      -- `-S`/`--split-string` introduces a complete env argument list,
+      -- not necessarily an interpreter first: it may begin with more env
+      -- options or `VAR=value` assignments. GNU env accepts that value
+      -- ATTACHED — `-Spython3`, `-vSpython3` (after no-operand short flags
+      -- i/v/0), or `--split-string=python3`. Put an attached payload back
+      -- into this token stream so it goes through the same option/operand/
+      -- assignment state machine as a separated payload.
+      local split_attached =
+        tok:match("^%-[iv0]*S(.+)$") or tok:match("^%-%-split%-string=(.+)$")
+      if not seen_env then
+        seen_env = true -- the `env` path token itself
+      elseif skip_next then
+        skip_next = false -- the operand consumed by the previous option
+      elseif split_attached then
+        table.insert(tokens, i, split_attached)
+      elseif tok == "-S" or tok == "--split-string" then
+        -- Separated split-string: the next token starts the string, i.e.
+        -- another env argument — keep walking.
+      elseif tok:find("=", 1, true) then
+        -- `VAR=value` env assignment, or another `--long=value` option:
+        -- self-contained, skip.
+      elseif tok:sub(1, 1) == "-" then
+        -- An option. A few GNU-env short options and their long forms
+        -- consume the *next* token as an operand (`-u NAME`, `-C DIR`,
+        -- `-a NAME`); skip that operand too, or its value is mistaken for
+        -- the interpreter. An option with an attached operand (`-uNAME`)
+        -- is one self-contained token and needs no skip.
+        if tok == "-u" or tok == "-C" or tok == "-a"
+            or tok == "--unset" or tok == "--chdir" or tok == "--argv0" then
+          skip_next = true
+        end
+      else
+        base = tok:match("([^/]+)$") or tok
+        break
+      end
+    end
+    if not base then return nil end
+  end
+  return pmacs.parse.shebangs[base]
+end
+
 -- Set of buffer ids that already have a highlight overlay
 -- attached, keyed by raw id (number). A buffer that opens, gets
 -- highlights, gets killed, and is reopened needs a fresh overlay
 -- attach; the kill path clears the entry below if/when it lands.
 local highlighted_buffers = {}
 
+-- Filetype-aware language resolution for the active buffer, in
+-- precedence order: grammar extension → LSP filetype map → shebang. The
+-- shebang is consulted ONLY when the extension is unrecognized (a known
+-- non-grammar extension like `.py` must not fall through to a stray
+-- `#!/bin/sh` and be misparsed as bash). Keyed on `buf:name()` for the
+-- extension parts (matching the historical behavior — path-less buffers
+-- that resolve a grammar by name keep working); the shebang reads buffer
+-- content directly.
+local function resolve_active_language(buf)
+  local name = buf:name()
+  if name then
+    local grammar = pmacs.parse.language_for_path(name)
+    if grammar then return grammar end
+    local ext = name:match("%.([%w_]+)$")
+    local by_ext = ext and pmacs.lsp and pmacs.lsp.filetypes and pmacs.lsp.filetypes[ext]
+    -- A recognized (even non-grammar) extension is authoritative; do not
+    -- consult the shebang for it.
+    if by_ext then return by_ext end
+  end
+  return pmacs.parse.language_from_shebang(buf)
+end
+
 local function attach_for_active_buffer()
   local buf = pmacs.window.buffer()
   if not buf then return end
-  local path = buf:name()
+  local key = tostring(buf)
-  if not path then return end
+  -- Reuse the language pinned at first attach if this buffer already has
-  local lang = pmacs.parse.language_for_path(path)
+  -- a parse view. A switch-away/back re-runs this hook (via after-switch);
-  if not lang then return end
+  -- re-resolving there would re-sniff a shebang the user has since edited
+  -- and silently swap the grammar — and diverge from the LSP side, which
+  -- keeps its existing attachment across the switch. A first-seen buffer
+  -- (no view yet) resolves normally. Gate dispatch on `_has_language`: the
+  -- resolution chain also yields languages with no grammar (python,
+  -- javascript), and dispatching one would raise "unknown language"
+  -- (caught, but noise) and never gives a wrong-grammar tree.
+  local lang = pmacs.parse._has_view(buf) and parse_lang_by_buffer[key]
+    or resolve_active_language(buf)
+  if not lang or not pmacs.parse._has_language(lang) then return end
   pmacs.parse._dispatch(buf, lang)
   -- T M4.3: install the syntax-highlight overlay for this buffer.
   -- Idempotent --- repeated calls for the same buffer are a no-op
@@ -66,8 +200,8 @@ local function attach_for_active_buffer()
   -- table so we only push once per (buffer, after-load) cycle).
   -- `tostring(buf)` is stable per BufferId (the metamethod
   -- formats the wrapped id), so it's a safe table-key
-  -- replacement for a `:id()` method we don't have to expose.
+  -- replacement for a `:id()` method we don't have to expose (`key` is
-  local key = tostring(buf)
+  -- computed once at the top of this function).
   if not highlighted_buffers[key] then
     local ok = pmacs.parse._attach_highlight(buf, lang)
     if ok then highlighted_buffers[key] = true end
@@ -108,9 +242,14 @@ local function reparse_active_buffer_after_edit()
   if not pmacs.parse._has_view(buf) then return end
   local pending = pmacs.parse._pending_edits(buf)
   if not pending or pending == 0 then return end
-  local path = buf:name()
+  -- Reparse with the language pinned when the view was first attached ---
-  if not path then return end
+  -- never re-resolve from the path or (mutable) shebang. The Rust side is
-  local lang = pmacs.parse.language_for_path(path)
+  -- "first language wins"; re-sniffing a shebang the user just edited
+  -- would either raise "unknown language" (sh → python) or swap the parse
+  -- tree to a new grammar while the highlight overlay still holds the
+  -- original grammar's query (sh → lua). Language changes need a
+  -- close/reopen, exactly as they do for a renamed extension.
+  local lang = parse_lang_by_buffer[tostring(buf)]
   if not lang then return end
   pmacs.parse._dispatch(buf, lang)
 end

tests/m4_acceptance.rs

@@ -5849,6 +5849,284 @@ fn m4_12_default_bundle_wires_bash() {
     assert_eq!(probe.get::<String>("grammar_bats").unwrap(), "bash");
 }
 
+/// Shebang detection: `pmacs.parse.language_from_shebang` maps the
+/// interpreter basename (resolving the `#!/usr/bin/env` indirection) to a
+/// language, and returns nil for non-shebangs and unmapped interpreters.
+/// This is the fallback that lets extensionless scripts (`scripts/deploy`,
+/// git hooks, `configure`) resolve a language at all — extension
+/// detection misses them.
+#[test]
+fn m4_shebang_resolver_maps_interpreters() {
+    use pmacs::editor::EditorState;
+    let s = EditorState::new();
+    let resolve = |first_line: &str| -> Option<String> {
+        s.lua_host
+            .lua()
+            .load(format!(
+                "local b = pmacs.window.buffer()
+                 if b:len() > 0 then b:delete(0, b:len()) end
+                 b:insert(0, {first_line:?})
+                 return pmacs.parse.language_from_shebang(b)"
+            ))
+            .eval()
+            .expect("resolve shebang")
+    };
+    for (line, want) in [
+        ("#!/bin/sh\n", "bash"),
+        ("#!/bin/bash -e\n", "bash"),
+        ("#! /bin/zsh\n", "bash"),
+        ("#!/usr/bin/env bash\n", "bash"),
+        ("#!/usr/bin/env python3\n", "python"),
+        ("#!/usr/bin/env -S python3 -u\n", "python"),
+        // Attached split-string forms carry the interpreter inside the
+        // option token.
+        ("#!/usr/bin/env -Spython3 -u\n", "python"),
+        ("#!/usr/bin/env --split-string=python3 -u\n", "python"),
+        ("#!/usr/bin/env -vSpython3 -u\n", "python"),
+        // The attached split string is a complete env argument list, so
+        // options and assignments may precede the interpreter within it.
+        ("#!/usr/bin/env -S-i python3 -u\n", "python"),
+        ("#!/usr/bin/env -SFOO=bar python3 -u\n", "python"),
+        ("#!/usr/bin/env --split-string=-u FOO python3\n", "python"),
+        // GNU-env options that consume an operand must not have the
+        // operand mistaken for the interpreter.
+        ("#!/usr/bin/env -u FOO python3\n", "python"),
+        ("#!/usr/bin/env -C /tmp python3\n", "python"),
+        ("#!/usr/bin/env -u FOO -C /tmp node\n", "javascript"),
+        ("#!/usr/bin/node\n", "javascript"),
+        ("#!/usr/bin/env lua\n", "lua"),
+    ] {
+        assert_eq!(resolve(line).as_deref(), Some(want), "{line:?}");
+    }
+    for line in [
+        "echo hi\n",
+        "# just a comment\n",
+        "#!/usr/bin/env ruby\n", // interpreter not in the seeded map
+        "\n",
+        "",
+    ] {
+        assert_eq!(resolve(line), None, "{line:?}");
+    }
+}
+
+/// End-to-end: opening an extensionless `#!/bin/sh` script resolves to
+/// `bash` on both paths — `lsp.lua`'s `buffer_language` chain (so the
+/// server would attach) and `syntax.lua`'s grammar attach (so a bash
+/// parse tree is produced). `pmacs.lsp.config` is emptied first so the
+/// real bash-language-server isn't spawned; grammar detection is
+/// independent of the LSP config.
+#[test]
+fn m4_shebang_extensionless_script_resolves_bash() {
+    use pmacs::editor::EditorState;
+    let mut s = EditorState::new();
+    let dir = tempfile::tempdir().expect("tempdir");
+    let hook = dir.path().join("pre-commit"); // no extension
+    std::fs::write(&hook, b"#!/bin/sh\nset -e\necho building\n").expect("write");
+    let hook_disp = hook.display();
+
+    s.lua_host
+        .lua()
+        .load(format!(
+            "pmacs.lsp.config = {{}}
+             pmacs.buffer.find_or_open('{hook_disp}')"
+        ))
+        .exec()
+        .expect("open extensionless shebang script");
+
+    let lsp_lang: Option<String> = s
+        .lua_host
+        .lua()
+        .load("return pmacs.lsp.active_buffer_language()")
+        .eval()
+        .expect("lsp language");
+    assert_eq!(
+        lsp_lang.as_deref(),
+        Some("bash"),
+        "extensionless #!/bin/sh resolves to bash for LSP"
+    );
+
+    pump_async(&mut s, |st| current_tree_language(st).is_some());
+    assert_eq!(
+        current_tree_language(&s).as_deref(),
+        Some("bash"),
+        "extensionless #!/bin/sh gets a bash parse tree"
+    );
+}
+
+/// Precedence: a recognized extension always wins over file content, so a
+/// `.py` file that happens to open with `#!/bin/sh` still resolves to
+/// python — the shebang is consulted only when extension detection misses.
+#[test]
+fn m4_shebang_does_not_override_extension() {
+    use pmacs::editor::EditorState;
+    let s = EditorState::new();
+    let dir = tempfile::tempdir().expect("tempdir");
+    let f = dir.path().join("tool.py");
+    std::fs::write(&f, b"#!/bin/sh\nprint('hi')\n").expect("write");
+    let f_disp = f.display();
+    s.lua_host
+        .lua()
+        .load(format!(
+            "pmacs.lsp.config = {{}}
+             pmacs.buffer.find_or_open('{f_disp}')"
+        ))
+        .exec()
+        .expect("open .py with a shell shebang");
+    // Both the LSP language *and* the grammar decision must respect the
+    // extension: python for LSP, and NO grammar parse view (python has no
+    // grammar) — not a bash tree installed from the `#!/bin/sh` line.
+    // `_has_view` is set synchronously by `_dispatch`, so no pump is
+    // needed; without the precedence fix the shebang would have dispatched
+    // bash and this would be true.
+    let (lang, has_view): (Option<String>, bool) = s
+        .lua_host
+        .lua()
+        .load(
+            "return pmacs.lsp.active_buffer_language(),
+                    pmacs.parse._has_view(pmacs.window.buffer())",
+        )
+        .eval()
+        .expect("language + view");
+    assert_eq!(
+        lang.as_deref(),
+        Some("python"),
+        ".py extension wins over a #!/bin/sh shebang (LSP)"
+    );
+    assert!(
+        !has_view,
+        ".py file must not get a grammar parse view from a #!/bin/sh line"
+    );
+}
+
+/// Finding-1 gate: an extensionless `#!/usr/bin/env python3` script
+/// resolves to python for LSP, but python has no grammar — syntax must
+/// skip it *silently*. Without the `_has_language` gate, `_dispatch`
+/// raises "unknown language: python" (caught by the after-load pcall and
+/// reported through `pmacs.error`), which we assert does NOT happen.
+#[test]
+fn m4_shebang_extensionless_grammarless_language_is_silent() {
+    use pmacs::editor::EditorState;
+    let s = EditorState::new();
+    let dir = tempfile::tempdir().expect("tempdir");
+    let f = dir.path().join("generate"); // no extension
+    std::fs::write(&f, b"#!/usr/bin/env python3\nprint('hi')\n").expect("write");
+    let f_disp = f.display();
+    s.lua_host
+        .lua()
+        .load(format!(
+            "pmacs.lsp.config = {{}}
+             _G.__errs = {{}}
+             local real = pmacs.error
+             pmacs.error = function(m) table.insert(_G.__errs, m) end
+             pmacs.buffer.find_or_open('{f_disp}')"
+        ))
+        .exec()
+        .expect("open extensionless python script");
+    let (lang, has_view, errs): (Option<String>, bool, i64) = s
+        .lua_host
+        .lua()
+        .load(
+            "return pmacs.lsp.active_buffer_language(),
+                    pmacs.parse._has_view(pmacs.window.buffer()),
+                    #_G.__errs",
+        )
+        .eval()
+        .expect("probe");
+    assert_eq!(lang.as_deref(), Some("python"), "python resolves for LSP");
+    assert!(
+        !has_view,
+        "no grammar parse view for a grammarless language"
+    );
+    assert_eq!(errs, 0, "no 'unknown language' error reported");
+}
+
+/// Finding-2 pin: editing an open extensionless script's shebang must not
+/// re-switch the parse grammar. A `#!/bin/sh` script attaches the bash
+/// grammar; rewriting its shebang to lua and firing after-edit must keep
+/// the bash tree (the pinned grammar) rather than swap in lua under the
+/// stale highlight overlay — and must not error.
+#[test]
+fn m4_shebang_edit_keeps_pinned_grammar() {
+    use pmacs::editor::EditorState;
+    let mut s = EditorState::new();
+    let dir = tempfile::tempdir().expect("tempdir");
+    let hook = dir.path().join("deploy"); // no extension
+    std::fs::write(&hook, b"#!/bin/sh\necho one\n").expect("write");
+    let hook_disp = hook.display();
+    s.lua_host
+        .lua()
+        .load(format!(
+            "pmacs.lsp.config = {{}}
+             _G.__errs = {{}}
+             local real = pmacs.error
+             pmacs.error = function(m) table.insert(_G.__errs, m) end
+             pmacs.buffer.find_or_open('{hook_disp}')"
+        ))
+        .exec()
+        .expect("open extensionless shell script");
+    pump_async(&mut s, |st| {
+        current_tree_language(st).as_deref() == Some("bash")
+    });
+
+    // Rewrite the first line to a lua shebang, then fire after-edit.
+    s.lua_host
+        .lua()
+        .load(
+            "local b = pmacs.window.buffer()
+             local text = b:slice(0, b:len())
+             local first_len = (text:find('\\n', 1, true) or 1) - 1
+             b:replace(0, first_len, '#!/usr/bin/env lua')
+             pmacs.hook.run('buffer.after-edit')",
+        )
+        .exec()
+        .expect("rewrite shebang to lua");
+    // Let the reparse settle (manual ticks: the tree stays bash with the
+    // pin, so a `pump_async` for a language *change* would time out).
+    for _ in 0..64 {
+        s.tick_async();
+        std::thread::sleep(Duration::from_millis(2));
+    }
+    assert_eq!(
+        current_tree_language(&s).as_deref(),
+        Some("bash"),
+        "editing the shebang must not re-switch the pinned grammar"
+    );
+
+    // Switch away to another buffer and back: the after-switch reattach
+    // must reuse the pinned bash grammar rather than re-sniff the (now
+    // lua) shebang — otherwise grammar and LSP diverge, since the LSP side
+    // keeps its bash attachment across the switch. `switch_buffer` fires
+    // `buffer.after-switch` synchronously.
+    let other = dir.path().join("other.txt");
+    std::fs::write(&other, b"plain text\n").expect("write other");
+    let other_disp = other.display();
+    s.lua_host
+        .lua()
+        .load(format!(
+            "local pinned = pmacs.window.buffer()
+             pmacs.buffer.find_or_open('{other_disp}')
+             pmacs.window.switch_buffer(pinned)"
+        ))
+        .exec()
+        .expect("switch away and back");
+    for _ in 0..64 {
+        s.tick_async();
+        std::thread::sleep(Duration::from_millis(2));
+    }
+    assert_eq!(
+        current_tree_language(&s).as_deref(),
+        Some("bash"),
+        "switch-away/back must reuse the pinned grammar, not re-sniff the edited shebang"
+    );
+    let errs: i64 = s
+        .lua_host
+        .lua()
+        .load("return #_G.__errs")
+        .eval()
+        .expect("errs");
+    assert_eq!(errs, 0, "reparse with the pinned grammar reports no error");
+}
+
 /// Typing-perf: the default bundle coalesces full-document
 /// `didChange` notifications instead of sending one per keystroke
 /// (each send copies the whole buffer several times and writes