From 4a60e4c85805e23e819afa2bd617eac7f73bc2ee Mon Sep 17 00:00:00 2001 From: Levi Neuwirth Date: Tue, 14 Jul 2026 14:33:06 +0100 Subject: [PATCH] feat(highlight): shebang-based language detection for extensionless scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extension detection missed extensionless scripts — `scripts/deploy`, git hooks, `configure`, and `scripts/bite` itself — so they got neither highlighting nor an LSP server. Add a first-line shebang fallback. - New `pmacs.parse.language_from_shebang(buf)` (builtin/runtime/syntax.lua): sniffs the first line (capped at 256 bytes), maps the interpreter's basename to a language, and resolves the `#!/usr/bin/env python3` indirection (skipping env's own `-S`/flags and `VAR=val` assignments). Backed by `pmacs.parse.shebangs`, a user-extensible map seeded with the interpreters pmacs can act on: sh-family -> bash, python* -> python, node -> javascript, lua* -> lua. - Wired as a strict *fallback* on both resolution paths: syntax.lua's grammar attach (`language_for_path or language_from_shebang`) and lsp.lua's `buffer_language` (grammar -> filetypes -> shebang). A recognized extension always wins, so a `.py`/`.sh` file is never re-classified by a stray shebang. - Cross-language, not shell-only: `#!/usr/bin/env python` /`node` /`lua` resolve too. Special filenames (`.bashrc`, `Dockerfile`, `Makefile`) are intentionally deferred until there are grammars behind them. Bite-verified acceptance (tests/m4_acceptance.rs): - m4_shebang_resolver_maps_interpreters — the mapping incl. env indirection and `env -S`; non-shebangs and unmapped interpreters (ruby) resolve to nil. - m4_shebang_extensionless_script_resolves_bash — opening an extensionless `#!/bin/sh` script resolves to bash on BOTH paths: lsp.lua's `active_buffer_language()` and a settled bash parse tree (grammar attach). Reachable only via the shebang, since the file has no extension. - m4_shebang_does_not_override_extension — a `.py` file opening with `#!/bin/sh` still resolves to python (extension precedence). Gates green: fmt; clippy -D warnings; test --lib; --features crdt; m4_acceptance --skip basedpyright; GPU; full workspace sweep; git diff --check. Change is Lua-only plus the acceptance tests. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01YJ9FQ832QwftJXCD9LeFan --- builtin/runtime/lsp.lua | 8 ++- builtin/runtime/syntax.lua | 68 +++++++++++++++++++++ tests/m4_acceptance.rs | 121 +++++++++++++++++++++++++++++++++++++ 3 files changed, 195 insertions(+), 2 deletions(-) diff --git a/builtin/runtime/lsp.lua b/builtin/runtime/lsp.lua index 6607423..4961657 100644 --- a/builtin/runtime/lsp.lua +++ b/builtin/runtime/lsp.lua @@ -374,11 +374,15 @@ local function buffer_language(buf) if not ok or not path then return nil end -- Grammar-backed detection first (keeps rust/.rs etc. exactly as -- before); fall back to the LSP-only filetype map so languages - -- with a server but no tree-sitter grammar (Python) still attach. + -- with a server but no tree-sitter grammar (Python) still attach; + -- finally, for an extensionless file, sniff a `#!interp` shebang so + -- e.g. an extensionless `#!/bin/sh` script still attaches its server. local lang = pmacs.parse.language_for_path(path) if lang then return lang end local ext = path:match("%.([%w_]+)$") - return ext and pmacs.lsp.filetypes[ext] or nil + local by_ext = ext and pmacs.lsp.filetypes[ext] + if by_ext then return by_ext end + return pmacs.parse.language_from_shebang(buf) end -- Public: the per-buffer language chain. Auto-pairing resolves -- relevance against the buffer its typed-edit record names — which a diff --git a/builtin/runtime/syntax.lua b/builtin/runtime/syntax.lua index 5f7fcb8..ae73b76 100644 --- a/builtin/runtime/syntax.lua +++ b/builtin/runtime/syntax.lua @@ -44,6 +44,72 @@ function pmacs.parse._dispatch(buf, lang) return job_id end +-- Shebang → language detection ------------------------------------------ +-- +-- Extension detection (`language_for_path`) misses extensionless scripts +-- (`scripts/deploy`, git hooks, `configure`), which are the common case +-- for shell. This fills that gap by sniffing the first line: `#!interp` +-- maps the interpreter's basename to a language. It is a *fallback* — +-- every caller tries extension detection first, so a `.py`/`.sh` file is +-- never re-classified by a stray shebang. Deliberately does not cover +-- special filenames (`.bashrc`, `Dockerfile`): those wait on grammars for +-- the languages behind them. +-- +-- The map is user-extensible from init.lua, e.g. +-- `pmacs.parse.shebangs.ruby = "ruby"`. Only interpreters whose language +-- pmacs can act on (grammar and/or LSP config) are seeded; an entry whose +-- language has neither is harmless but inert. +pmacs.parse.shebangs = pmacs.parse.shebangs or { + sh = "bash", bash = "bash", dash = "bash", ash = "bash", + ksh = "bash", mksh = "bash", zsh = "bash", + python = "python", python2 = "python", python3 = "python", + pypy = "python", pypy3 = "python", + node = "javascript", nodejs = "javascript", + lua = "lua", luajit = "lua", +} + +-- First line of `buf` (up to 256 bytes), sans trailing newline, or nil. +-- 256 bytes is well past any real shebang and keeps the slice cheap on +-- the hot path (callers only reach here when extension detection missed). +local function first_line(buf) + local ok_len, n = pcall(function() return buf:len() end) + if not ok_len or not n or n <= 0 then return nil end + if n > 256 then n = 256 end + local ok, s = pcall(function() return buf:slice(0, n) end) + if not ok or type(s) ~= "string" or #s == 0 then return nil end + local nl = s:find("\n", 1, true) + if nl then s = s:sub(1, nl - 1) end + return s +end + +-- Resolve the interpreter basename from a shebang line, resolving the +-- `env` indirection (`#!/usr/bin/env python3` → `python3`, skipping +-- `env`'s own options / `VAR=val` assignments). Returns the language +-- name from `pmacs.parse.shebangs`, or nil. +function pmacs.parse.language_from_shebang(buf) + if not buf then return nil end + local line = first_line(buf) + if not line or line:sub(1, 2) ~= "#!" then return nil end + local rest = line:sub(3):gsub("^%s+", "") + local first = rest:match("^(%S+)") + if not first then return nil end + local base = first:match("([^/]+)$") or first + if base == "env" then + base = nil + for tok in rest:gmatch("%S+") do + -- Skip the `env` path itself, its flags (`-S`, `--split-string`), + -- and inline `VAR=value` assignments; the first bare word left is + -- the real interpreter. + if tok ~= first and tok:sub(1, 1) ~= "-" and not tok:find("=", 1, true) then + base = tok:match("([^/]+)$") or tok + break + end + end + if not base then return nil end + end + return pmacs.parse.shebangs[base] +end + -- Set of buffer ids that already have a highlight overlay -- attached, keyed by raw id (number). A buffer that opens, gets -- highlights, gets killed, and is reopened needs a fresh overlay @@ -56,6 +122,7 @@ local function attach_for_active_buffer() local path = buf:name() if not path then return end local lang = pmacs.parse.language_for_path(path) + or pmacs.parse.language_from_shebang(buf) if not lang then return end pmacs.parse._dispatch(buf, lang) -- T M4.3: install the syntax-highlight overlay for this buffer. @@ -111,6 +178,7 @@ local function reparse_active_buffer_after_edit() local path = buf:name() if not path then return end local lang = pmacs.parse.language_for_path(path) + or pmacs.parse.language_from_shebang(buf) if not lang then return end pmacs.parse._dispatch(buf, lang) end diff --git a/tests/m4_acceptance.rs b/tests/m4_acceptance.rs index c648bcc..2d84aa1 100644 --- a/tests/m4_acceptance.rs +++ b/tests/m4_acceptance.rs @@ -5849,6 +5849,127 @@ fn m4_12_default_bundle_wires_bash() { assert_eq!(probe.get::("grammar_bats").unwrap(), "bash"); } +/// Shebang detection: `pmacs.parse.language_from_shebang` maps the +/// interpreter basename (resolving the `#!/usr/bin/env` indirection) to a +/// language, and returns nil for non-shebangs and unmapped interpreters. +/// This is the fallback that lets extensionless scripts (`scripts/deploy`, +/// git hooks, `configure`) resolve a language at all — extension +/// detection misses them. +#[test] +fn m4_shebang_resolver_maps_interpreters() { + use pmacs::editor::EditorState; + let s = EditorState::new(); + let resolve = |first_line: &str| -> Option { + s.lua_host + .lua() + .load(format!( + "local b = pmacs.window.buffer() + if b:len() > 0 then b:delete(0, b:len()) end + b:insert(0, {first_line:?}) + return pmacs.parse.language_from_shebang(b)" + )) + .eval() + .expect("resolve shebang") + }; + for (line, want) in [ + ("#!/bin/sh\n", "bash"), + ("#!/bin/bash -e\n", "bash"), + ("#! /bin/zsh\n", "bash"), + ("#!/usr/bin/env bash\n", "bash"), + ("#!/usr/bin/env python3\n", "python"), + ("#!/usr/bin/env -S python3 -u\n", "python"), + ("#!/usr/bin/node\n", "javascript"), + ("#!/usr/bin/env lua\n", "lua"), + ] { + assert_eq!(resolve(line).as_deref(), Some(want), "{line:?}"); + } + for line in [ + "echo hi\n", + "# just a comment\n", + "#!/usr/bin/env ruby\n", // interpreter not in the seeded map + "\n", + "", + ] { + assert_eq!(resolve(line), None, "{line:?}"); + } +} + +/// End-to-end: opening an extensionless `#!/bin/sh` script resolves to +/// `bash` on both paths — `lsp.lua`'s `buffer_language` chain (so the +/// server would attach) and `syntax.lua`'s grammar attach (so a bash +/// parse tree is produced). `pmacs.lsp.config` is emptied first so the +/// real bash-language-server isn't spawned; grammar detection is +/// independent of the LSP config. +#[test] +fn m4_shebang_extensionless_script_resolves_bash() { + use pmacs::editor::EditorState; + let mut s = EditorState::new(); + let dir = tempfile::tempdir().expect("tempdir"); + let hook = dir.path().join("pre-commit"); // no extension + std::fs::write(&hook, b"#!/bin/sh\nset -e\necho building\n").expect("write"); + let hook_disp = hook.display(); + + s.lua_host + .lua() + .load(format!( + "pmacs.lsp.config = {{}} + pmacs.buffer.find_or_open('{hook_disp}')" + )) + .exec() + .expect("open extensionless shebang script"); + + let lsp_lang: Option = s + .lua_host + .lua() + .load("return pmacs.lsp.active_buffer_language()") + .eval() + .expect("lsp language"); + assert_eq!( + lsp_lang.as_deref(), + Some("bash"), + "extensionless #!/bin/sh resolves to bash for LSP" + ); + + pump_async(&mut s, |st| current_tree_language(st).is_some()); + assert_eq!( + current_tree_language(&s).as_deref(), + Some("bash"), + "extensionless #!/bin/sh gets a bash parse tree" + ); +} + +/// Precedence: a recognized extension always wins over file content, so a +/// `.py` file that happens to open with `#!/bin/sh` still resolves to +/// python — the shebang is consulted only when extension detection misses. +#[test] +fn m4_shebang_does_not_override_extension() { + use pmacs::editor::EditorState; + let s = EditorState::new(); + let dir = tempfile::tempdir().expect("tempdir"); + let f = dir.path().join("tool.py"); + std::fs::write(&f, b"#!/bin/sh\nprint('hi')\n").expect("write"); + let f_disp = f.display(); + s.lua_host + .lua() + .load(format!( + "pmacs.lsp.config = {{}} + pmacs.buffer.find_or_open('{f_disp}')" + )) + .exec() + .expect("open .py with a shell shebang"); + let lang: Option = s + .lua_host + .lua() + .load("return pmacs.lsp.active_buffer_language()") + .eval() + .expect("language"); + assert_eq!( + lang.as_deref(), + Some("python"), + ".py extension wins over a #!/bin/sh shebang" + ); +} + /// Typing-perf: the default bundle coalesces full-document /// `didChange` notifications instead of sending one per keystroke /// (each send copies the whole buffer several times and writes