neuwirth
/
pmacs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #132 from levineuwirth/modeline-detection 

Detect language from modelines
This commit is contained in:
Levi Neuwirth 2026-07-22 15:24:25 +00:00 committed by GitHub
parent 86fc1bccae b45e5ee5ec
commit 1dd47fcad1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 1018 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
builtin/runtime/lsp.lua
View File

@ -471,33 +471,21 @@ end
local function buffer_language(buf)
local ok, path = pcall(function() return buf and buf:path() end)
if not ok or not path then return nil end
-- Grammar-backed detection first (keeps rust/.rs etc. exactly as
-- before); fall back to the LSP-only filetype map so languages with a
-- server but no tree-sitter grammar (Python) still attach; then the
-- basename map for filename-identified files (`Dockerfile`,
-- `CMakeLists.txt`); finally, for an extensionless file, sniff a
-- `#!interp` shebang so e.g. `#!/bin/sh` still attaches its server.
local lang = pmacs.parse.language_for_path(path)
if lang then return lang end
local ext = path:match("%.([%w_]+)$")
local by_ext = ext and pmacs.lsp.filetypes[ext]
if by_ext then return by_ext end
local by_name = pmacs.parse.language_from_filename(path)
if by_name then return by_name end
return pmacs.parse.language_from_shebang(buf)
-- Syntax owns the fresh-load inference and pin. LSP retains only its
-- path-eligibility rule: without a backing path it cannot construct a URI or
-- project root, even when syntax can infer a grammar from a buffer name.
return pmacs.parse.buffer_language(buf)
end
-- Public: the per-buffer language chain. Auto-pairing resolves
-- relevance against the buffer its typed-edit record names — which a
-- context-switching command may have left inactive by callback time —
-- so the parameterized form is the primitive and the active-buffer
-- form delegates.
-- Public: the pinned per-buffer language. Auto-pairing resolves relevance
-- against the buffer its typed-edit record names — which a context-switching
-- command may have left inactive by callback time — so the parameterized form
-- is the primitive and the active-buffer form delegates.
pmacs.lsp.buffer_language = buffer_language
local function active_buffer_language()
return buffer_language(pmacs.window.buffer())
end
-- Public: the comment-toggle module (and future language-aware Lua)
-- reuses this grammar+filetypes chain instead of replicating it.
-- Public: comment-toggle and other language-aware Lua reuse the same pin.
pmacs.lsp.active_buffer_language = active_buffer_language
-- Directory component of a path, or nil if it has none.
@ -714,7 +702,7 @@ local function attach_buffer(buf)
-- carries the full current text, superseding them.
pending_did_change[key] = nil
end
local language = active_buffer_language()
local language = buffer_language(buf)
if not language then return nil end
-- Path resolved before spawn so the server's `rootUri` can be
-- derived from the file's project (see `project_root_for`).

300
builtin/runtime/syntax.lua
View File

@ -22,6 +22,10 @@ local inflight_parse_by_buffer = {}
local parse_buffer_by_key = {}
local parse_lang_by_buffer = {}
local reparse_requested_by_buffer = {}
-- Fresh-load language decision, including an explicit false sentinel for
-- "resolved none". Syntax, LSP, pairing, comments, and initial major mode all
-- consume this pin rather than re-sniffing mutable file content independently.
local detected_language_by_buffer = {}
-- Buffers already warned about hitting the injection layer cap (Q#IJ3);
-- keyed like the others so we warn once, and re-arm if the file stops
-- capping (an edit removed the excess regions).
@ -200,21 +204,254 @@ function pmacs.parse.language_from_filename(name)
return pmacs.parse.filenames[base]
end
-- Modeline → language detection ---------------------------------------------
local MODELINE_WINDOW_BYTES = 8 * 1024
local VIM_MODELINE_LINES = 5
local MODELINE_NAME_BYTES = 128
pmacs.parse.modeline_aliases = pmacs.parse.modeline_aliases or {}
local default_modeline_aliases = {
["c++"] = "cpp",
cxx = "cpp",
sh = "bash",
shell = "bash",
["shell-script"] = "bash",
zsh = "bash",
py = "python",
js = "javascript",
js2 = "javascript",
jsx = "javascriptreact",
ts = "typescript",
tsx = "typescriptreact",
yml = "yaml",
makefile = "make",
docker = "dockerfile",
}
for name, language in pairs(default_modeline_aliases) do
if pmacs.parse.modeline_aliases[name] == nil then
pmacs.parse.modeline_aliases[name] = language
end
end
local function normalize_modeline_name(name)
if type(name) ~= "string" then return nil end
name = name:gsub("^[ \t]+", ""):gsub("[ \t]+$", ""):lower()
if #name == 0 or #name > MODELINE_NAME_BYTES then return nil end
if not name:match("^[a-z0-9][a-z0-9+_-]*$") then return nil end
local alias = pmacs.parse.modeline_aliases[name]
if alias == nil then return name end
if type(alias) ~= "string" or #alias == 0 or #alias > MODELINE_NAME_BYTES then
return nil
end
if not alias:match("^[a-z0-9][a-z0-9+_-]*$") then return nil end
return alias
end
local function without_trailing_cr(line)
if line:sub(-1) == "\r" then return line:sub(1, -2) end
return line
end
-- Return the first five and last five complete logical lines. Each entry is
-- `{ text, offset }`, where offset is the zero-based buffer byte position.
-- The suffix's leading partial line is discarded and does not consume a slot.
local function modeline_edge_lines(buf)
local ok_len, length = pcall(function() return buf:len() end)
if not ok_len or not length or length <= 0 then return {}, {} end
local prefix_end = math.min(length, MODELINE_WINDOW_BYTES)
local ok_prefix, prefix = pcall(function() return buf:slice(0, prefix_end) end)
if not ok_prefix or type(prefix) ~= "string" then return {}, {} end
local front = {}
local pos = 1
while #front < VIM_MODELINE_LINES and pos <= #prefix do
local newline = prefix:find("\n", pos, true)
if not newline then
if prefix_end < length then break end
newline = #prefix + 1
end
front[#front + 1] = {
text = without_trailing_cr(prefix:sub(pos, newline - 1)),
offset = pos - 1,
}
if newline > #prefix then break end
pos = newline + 1
end
local tail_start = 0
local scan_start = 1
if length > MODELINE_WINDOW_BYTES then
-- Keep the one-byte line-boundary probe plus suffix content within the
-- same 8 KiB read budget.
tail_start = length - (MODELINE_WINDOW_BYTES - 1)
local ok_probe, probe =
pcall(function() return buf:slice(tail_start - 1, tail_start) end)
if not ok_probe or probe ~= "\n" then
scan_start = nil
end
end
local tail = prefix
if tail_start > 0 then
local ok_tail
ok_tail, tail = pcall(function() return buf:slice(tail_start, length) end)
if not ok_tail or type(tail) ~= "string" then return front, front end
end
if scan_start == nil then
local first_newline = tail:find("\n", 1, true)
if not first_newline then return front, front end
scan_start = first_newline + 1
end
local reverse_tail = {}
local line_end = #tail
if line_end >= scan_start and tail:byte(line_end) == 10 then
line_end = line_end - 1
end
while line_end >= scan_start and #reverse_tail < VIM_MODELINE_LINES do
local i = line_end
while i >= scan_start and tail:byte(i) ~= 10 do
i = i - 1
end
local line_start = i + 1
reverse_tail[#reverse_tail + 1] = {
text = without_trailing_cr(tail:sub(line_start, line_end)),
offset = tail_start + line_start - 1,
}
line_end = i - 1
end
local edges = {}
local seen = {}
for _, entry in ipairs(front) do
edges[#edges + 1] = entry
seen[entry.offset] = true
end
for i = #reverse_tail, 1, -1 do
local entry = reverse_tail[i]
if not seen[entry.offset] then
edges[#edges + 1] = entry
seen[entry.offset] = true
end
end
return front, edges
end
local function emacs_mode_on_line(entry, consider)
local line = entry.text
local search_from = 1
while true do
local open = line:find("-*-", search_from, true)
if not open then return end
local close = line:find("-*-", open + 3, true)
if not close then return end
local payload = line:sub(open + 3, close - 1)
if payload:find(":", 1, true) then
for part_at, part in payload:gmatch("()([^;]+)") do
local key, value =
part:match("^[ \t]*([%w_-]+)[ \t]*:[ \t]*(.-)[ \t]*$")
if key and key:lower() == "mode" then
local mode = normalize_modeline_name(value)
if mode then consider(mode, entry.offset + open + part_at) end
end
end
else
local mode = normalize_modeline_name(payload)
if mode then consider(mode, entry.offset + open) end
end
search_from = close + 3
end
end
local function vim_assignment(token)
return token:match("^ft=(.+)$") or token:match("^filetype=(.+)$")
end
local function vim_mode_at(entry, marker_at, marker, consider)
local line = entry.text
local rest_at = marker_at + #marker
local rest = line:sub(rest_at)
local full_set_at = rest:match("^[ \t]*set[ \t]+()")
local option_at = full_set_at
if not option_at and marker ~= "Vim:" then
option_at = rest:match("^[ \t]*se[ \t]+()")
end
if marker == "Vim:" and not full_set_at then return end
if option_at then
local terminator = rest:find(":", option_at, true)
if not terminator then return end
local options = rest:sub(option_at, terminator - 1)
for token_at, token in options:gmatch("()([^ \t]+)") do
local value = vim_assignment(token)
local mode = value and normalize_modeline_name(value)
if mode then
consider(mode, entry.offset + rest_at + option_at + token_at - 3)
end
end
return
end
for token_at, token in rest:gmatch("()([^ \t:]+)") do
local value = vim_assignment(token)
local mode = value and normalize_modeline_name(value)
if mode then consider(mode, entry.offset + rest_at + token_at - 2) end
end
end
local function vim_modes_on_line(entry, consider)
local line = entry.text
for _, marker in ipairs({ "vim:", "vi:", "Vim:" }) do
local search_from = 1
while true do
local marker_at = line:find(marker, search_from, true)
if not marker_at then break end
local previous = marker_at > 1 and line:sub(marker_at - 1, marker_at - 1)
if marker_at == 1 or previous == " " or previous == "\t" then
vim_mode_at(entry, marker_at, marker, consider)
end
search_from = marker_at + 1
end
end
end
function pmacs.parse.language_from_modeline(buf)
if not buf then return nil end
local front, edges = modeline_edge_lines(buf)
local mode
local mode_at = -1
local function consider(candidate, candidate_at)
if candidate_at >= mode_at then
mode = candidate
mode_at = candidate_at
end
end
if front[1] then emacs_mode_on_line(front[1], consider) end
if front[1] and front[1].text:sub(1, 2) == "#!" and front[2] then
emacs_mode_on_line(front[2], consider)
end
for _, entry in ipairs(edges) do
vim_modes_on_line(entry, consider)
end
return mode
end
-- Set of buffer ids that already have a highlight overlay
-- attached, keyed by raw id (number). A buffer that opens, gets
-- highlights, gets killed, and is reopened needs a fresh overlay
-- attach; the kill path clears the entry below if/when it lands.
local highlighted_buffers = {}
-- Filetype-aware language resolution for the active buffer, in precedence
-- order: grammar extension → LSP filetype map → filename → shebang. A
-- recognized extension is authoritative (a `.py` must not fall through to
-- a stray `#!/bin/sh` and be misparsed as bash); the basename map handles
-- extensionless `Dockerfile`/`Makefile`/rc-dotfiles, and only then does
-- the shebang (buffer content) get a look. Keyed on `buf:name()` for the
-- path parts (matching the historical behavior — path-less buffers that
-- resolve a grammar by name keep working).
local function resolve_active_language(buf)
-- Fresh language inference for a buffer, in precedence order: explicit
-- modeline → grammar extension → LSP filetype map → filename → shebang.
-- Path components intentionally come from `buf:name()` to preserve syntax's
-- historical grammar-by-name behavior for pathless buffers.
local function detect_buffer_language(buf)
local modeline = pmacs.parse.language_from_modeline(buf)
if modeline then return modeline end
local name = buf:name()
if name then
local grammar = pmacs.parse.language_for_path(name)
@ -228,31 +465,34 @@ local function resolve_active_language(buf)
return pmacs.parse.language_from_shebang(buf)
end
local function refresh_buffer_language(buf)
local language = detect_buffer_language(buf)
detected_language_by_buffer[tostring(buf)] = language or false
return language
end
function pmacs.parse.buffer_language(buf)
if not buf then return nil end
local key = tostring(buf)
local language = detected_language_by_buffer[key]
if language ~= nil then return language or nil end
return refresh_buffer_language(buf)
end
local function attach_for_active_buffer(initialize_mode)
local buf = pmacs.window.buffer()
if not buf then return end
local key = tostring(buf)
-- Reuse the language pinned at first attach if this buffer already has
-- a parse view. A switch-away/back re-runs this hook (via after-switch);
-- re-resolving there would re-sniff a shebang the user has since edited
-- and silently swap the grammar — and diverge from the LSP side, which
-- keeps its existing attachment across the switch. A first-seen buffer
-- (no view yet) resolves normally. Gate dispatch on `_has_language`: the
-- resolution chain can still yield a language with no grammar — a
-- shebang or filetype mapping to a server-only or unsupported language
-- (e.g. an init.lua `pmacs.parse.shebangs.ruby = "ruby"`) — and
-- dispatching one would raise "unknown language" (caught, but noise) and
-- never gives a wrong-grammar tree.
local lang = pmacs.parse._has_view(buf) and parse_lang_by_buffer[key]
or resolve_active_language(buf)
-- The detected language is also the initial major-mode name. Do this
-- before grammar gating: a language supplied only by an LSP filetype or
-- shebang mapping is still a valid mode even when no parser is bundled.
-- Only after-load initializes it; after-switch must preserve explicit
-- overrides and explicit nil clears.
if initialize_mode and lang and pmacs.buffer.major_mode(buf) == nil then
pmacs.buffer.set_major_mode(buf, lang)
end
-- A genuine load refreshes every detection signal and replaces the initial
-- major mode, including with nil. Switches consume the pin: editing a
-- shebang/modeline cannot silently swap parser/LSP language, while a
-- registry-only hidden buffer still resolves when first visited.
local lang = initialize_mode and refresh_buffer_language(buf)
or pmacs.parse.buffer_language(buf)
if initialize_mode then pmacs.buffer.set_major_mode(buf, lang) end
-- The resolution chain can yield a valid mode with no grammar. Gate dispatch
-- so custom/server-only modes stay quiet rather than raising "unknown
-- language" from the parse worker.
if not lang or not pmacs.parse._has_language(lang) then return end
pmacs.parse._dispatch(buf, lang)
-- T M4.3: install the syntax-highlight overlay for this buffer.

72
docs/active-work.md
View File

@ -14,8 +14,7 @@ backlog.
machine-local: `origin` may name this canonical URL, a release mirror,
or something else, and therefore has no authority by name alone.
- Canonical base at this snapshot:
`githubsucks/main` @ `d5d9b9c` (mode system handoff #131 merged;
protocol v18).
`githubsucks/main` @ `86fc1bc` (Vterm Stage 2 #130 merged; protocol v18).
- On the transfer source, `origin/main` named a release mirror at
`d3fa632` and lagged badly. On the current destination, `origin` names
the canonical URL. This difference is why all recovery begins by
@ -49,55 +48,44 @@ git worktree list
git status --short --branch
```
The first command must expose `d5d9b9c` or a newer intentional main.
The first command must expose `86fc1bc` or a newer intentional main.
If it does not, stop and repair the remote/fetch configuration.
## Vterm Stage 2 implementation lane
## Modeline detection implementation lane
- Portable branch: `githubsucks/vterm-tui`
- Integrated feature head: `3f0252f` (review-fix head `b9a7e40`)
- Base: originally canonical `main` @ `f1a2f75`; current canonical `main`
@ `d5d9b9c` (mode system wiring #129 and handoff #131) is integrated before
merge.
- PR: #130, <https://github.com/levineuwirth/pmacs/pull/130>, open against
- Portable branch: `githubsucks/modeline-detection`
- Approved framing head: `6b4f3c3`
- Implementation head: `f8d05d2`
- Base: originally canonical `main` @ `d5d9b9c`; current canonical `main`
@ `86fc1bc` (Vterm Stage 2 #130) is integrated before merge.
- PR: #132, <https://github.com/levineuwirth/pmacs/pull/132>, open against
canonical `main` and explicitly authorized for merge.
- State: `docs/vterm-framing.md` Revision 7 criteria 15–27 are implemented.
The lane composes per-frontend/window terminal views in the TUI, installs
the strict `pmacs.terminal` API and terminal-local bindings, drains
clipboard/BEL through the authenticated frontend, and routes daemon
terminal input by connection source. Protocol remains v18; Stage 2 changes
neither the wire schema nor the GPU renderer.
- Review round 1: addressed. Dispatch now requires `C-c` before terminal-local
editor bindings; non-terminal context operations error; controller
replacement is atomic per frontend; zero-area layouts retain view anchors;
view projection borrows retained rows instead of deep-cloning scrollback.
- Review round 2: addressed. Partial eviction clamps anchors to the first
surviving wrapped-line cell; `invoke_interactive` now inherits only an
authenticated dispatch origin; explicit context failures are named; terminal
mouse routing reads geometry without cloning cells; the framing records the
v18 semantic-controller boundary and bracketed-paste injection deferral.
- Implementation commits: `39e07cb`, `7c39535`, `0a846d9`, `0dacac7`,
`dc92257`, merge `0ddff24`, integration hardening `da8f6ae`, first-review
fixes `8702791`, second-review fixes `b9a7e40`, and current-main integration
`3f0252f`.
- Post-integration verification: `cargo fmt --check`; strict workspace
Clippy; 1,753 default + 1,929 CRDT library tests (3 ignored each);
mode-system acceptance 1 default + 1 CRDT; Stage 1 acceptance 9 default +
10 CRDT; Stage 2 acceptance 4 default + 4 CRDT; statusline acceptance
7 default + 8 CRDT; M4 114 passed (3 ignored, 1 filtered); required GPU
109; workspace 2,882 passed across 82 suites (19 ignored, 1 filtered);
`git diff --check` clean. The first parallel M4 attempt timed out after
partial progress; a serial isolation pass and the immediate exact parallel
rerun both passed, and the workspace sweep also passed.
- Next: push the integrated head and merge PR #130 as authorized.
- State: Revision 2's bounded Emacs/Vim parser, explicit-over-inferred
precedence, alias normalization, shared fresh-load language pin, LSP path
guard, and all thirteen acceptance criteria are implemented. Protocol
remains v18.
- Verification:
- focused modeline + shebang-pin acceptance before integration: 7 passed on
default LuaJIT and 7 passed on non-default Lua 5.4;
- post-integration `cargo fmt --check`;
- post-integration `cargo clippy --workspace --all-targets -- -D warnings`;
- post-integration `cargo test --lib`: 1,753 passed;
- post-integration `cargo test --lib --features crdt`: 1,929 passed;
- post-integration `cargo test --test m4_acceptance -- --skip basedpyright`:
120 passed, 3 ignored, 1 filtered;
- post-integration `PMACS_REQUIRE_GPU=1 cargo test -p pmacs-gpu`: 109 passed;
- post-integration `cargo test --workspace -- --skip basedpyright`: 2,888
passed across 82 suites, 19 ignored, 1 filtered;
- `git diff --check`.
- Next: push the current-main integration and merge PR #132 as authorized.
Recovery worktree on a machine that does not already own the branch:
```sh
git worktree add --track \
-b vterm-tui \
../pmacs-vterm-tui \
githubsucks/vterm-tui
-b modeline-detection \
../pmacs-modeline-detection \
githubsucks/modeline-detection
```
## Parked lane: kill-ring browser + persistence

381
docs/modeline-detection-framing.md Normal file
View File

@ -0,0 +1,381 @@
# Modeline language detection — side quest
**Status:** Revision 2, approved for implementation by the user on 2026-07-22.
No implementation was present at approval.
**Base:** `githubsucks/main` at `d5d9b9c`; protocol v18.
## Problem
Language inference currently uses four signals, in order:
1. bundled grammar extension;
2. `pmacs.lsp.filetypes` extension;
3. exact basename;
4. shebang.
That chain cannot classify a deliberately misleading or extensionless file when
its author supplied editor metadata such as `-*- mode: python -*-` or
`vim: set ft=python:`. Mode-system wiring #129 gives the result a persistent
per-buffer home and makes it observable through key dispatch, help, and the
statusline, but no modeline parser exists.
The goal is one bounded, non-executing modeline detector whose result drives the
same initial language used by syntax, LSP, and `Buffer.major_mode`. This is a
language-detection feature, not a general file-local settings system.
## Ground truth
### Detection is duplicated today
`builtin/runtime/syntax.lua::resolve_active_language` and
`builtin/runtime/lsp.lua::buffer_language` independently implement the same
extension → filetype → filename → shebang chain. They already have small but
important differences:
- syntax uses `buf:name()`, preserving historical grammar-by-name behavior for
pathless buffers;
- LSP requires `buf:path()`, because it cannot construct a URI or project root
for a pathless buffer;
- syntax pins the grammar in `parse_lang_by_buffer` after dispatch, while the
public LSP language query re-sniffs mutable shebang text on every call.
Adding the fifth signal to both copies would create a third convention and let
syntax, LSP, mode initialization, auto-pairing, and comment commands disagree.
The implementation must consolidate the actual inference in `syntax.lua` and
leave only the LSP path-eligibility guard in `lsp.lua`.
### Hook order is already useful
`src/editor.rs` loads `syntax.lua` before `lsp.lua`. Their `buffer.after-load`
callbacks therefore run in that registration order:
1. syntax detects the language, initializes the major mode, and dispatches a
grammar when one exists;
2. LSP resolves the same buffer language and attaches a server when configured.
The modeline result must be computed and pinned by step 1 so step 2 cannot
independently reinterpret the file.
### Major mode and parser language have different later lifecycles
At first load, the detected language is the initial major-mode name. Afterward:
- explicit `pmacs.buffer.set_major_mode` changes dispatch/statusline only;
- syntax and LSP remain attached to their initially selected language;
- `buffer.after-switch` restores views without re-detecting;
- edits do not change the selected grammar.
Modelines follow that same load-time contract. Editing a cookie is not a live
mode or parser switch. Close/reopen re-evaluates it. A future true reload that
fires `buffer.after-load` re-evaluates it as a fresh load; explicit mode
overrides and clears are not reload-persistent, matching the #129 framing.
### Existing Lua reads are sufficient
`BufferIdLua` exposes byte-length and byte-slice operations. Detection can read
bounded prefix/suffix windows without copying the whole buffer or adding a Rust
binding. The rope is byte-addressed, so a bounded slice does not need UTF-8
boundary repair before Lua pattern matching.
### Compatibility references
The supported subset follows the documented, non-evaluating pieces of:
- GNU Emacs, “Specifying File Variables”:
<https://www.gnu.org/software/emacs/manual/html_node/emacs/Specifying-File-Variables.html>
- Vim, `:help modeline` and `:help 'modelines'`:
<https://vimhelp.org/options.txt.html#modeline>
Compatibility is deliberately bounded below. pmacs does not become an Emacs
file-variable evaluator or a Vim `:set` interpreter.
## Scope
In scope:
- Emacs `-*- mode: NAME -*-` and mode-only `-*- NAME -*-` cookies;
- Vim/Vi `ft=NAME` and `filetype=NAME` modelines;
- first/last-line scanning with fixed byte and line limits;
- canonical aliases for common external filetype names;
- modeline precedence over inferred path/shebang language;
- one shared, pinned per-buffer language decision;
- initial major mode, grammar, LSP, and language-aware Lua consumers agreeing;
- focused parser and end-to-end acceptance on both Lua backends.
Out of scope:
- Emacs `Local Variables:` tail blocks;
- variables other than Emacs `mode` or Vim `ft`/`filetype`;
- `eval`, Vim commands, option mutation, directory-local variables, or project
trust prompts;
- Vim `ex:` markers, version predicates, escaped option values, and combined
dotted filetypes;
- live re-detection after edits, saves, renames, or buffer switches;
- `buffer.after-mode-change`, minor modes, mode-scoped settings, `describe-mode`,
or session persistence of explicit mode overrides;
- a protocol change or frontend-specific work.
## Decisions
### Q#MD1 — Scan only bounded edge lines
Read at most 8 KiB from each end of the buffer and inspect:
- Emacs: line 1, or line 2 only when line 1 begins with `#!`;
- Vim/Vi: the first five and last five logical lines, matching Vim's default
`'modelines'=5` behavior.
When the prefix and suffix overlap, deduplicate lines before parsing. Strip one
trailing `\r` so CRLF and LF behave identically. Discard the suffix window's
leading fragment when its line begins before the 8 KiB boundary; that fragment
does not count toward the five complete logical lines counted backward from
buffer end. No truncated candidate is parsed. Detection therefore allocates at
most 16 KiB per fresh load, independent of file size, and an adversarial giant
edge line cannot force a whole-buffer copy.
The Emacs 3000-character tail `Local Variables:` mechanism is a separate parser
with comment-prefix/suffix rules and is excluded.
### Q#MD2 — Recognize a conservative syntax subset
Emacs:
- require a complete pair of `-*-` delimiters on the eligible line;
- accept `mode: NAME` in a semicolon-separated property list;
- accept a mode-only payload such as `-*- Lisp -*-`;
- ignore every property except `mode`;
- when a cookie contains multiple valid `mode` properties, the last wins.
Vim/Vi:
- accept `vim:`, `vi:`, and `Vim:` at line start or preceded by ASCII space or
tab; uppercase `Vim:` requires literal `set` rather than abbreviated `se`,
matching Vim;
- accept only exact `ft=NAME` and `filetype=NAME` assignments; `ft:NAME` and
`filetype:NAME` are not modeline assignment forms and are rejected;
- in the direct form, split option tokens on ASCII whitespace and `:`, so the
common `vim:ft=python:sw=4:` form yields `ft=python`;
- in the `set` / `se` form (`se` is lowercase-marker-only), end the option
section at the first `:` and split only the preceding text on ASCII
whitespace; `vim: set sw=4: ft=python` therefore contains no live filetype
assignment;
- ignore all other live option tokens rather than interpreting them;
- require that terminating colon for the `set` / `se` form, so a comment suffix
is never consumed as an option value;
- reject `ex:`, Vim version predicates, and marker substrings embedded in a
word.
Across all eligible lines, the last valid mode assignment in document order
wins. This matches Emacs's “final defined mode” behavior and ordinary sequential
option assignment. A footer Vim modeline can intentionally override a header
Emacs cookie; conflicting metadata does not depend on Lua table iteration.
### Q#MD3 — Modeline names are data, never code
Trim ASCII edge whitespace, ASCII-lowercase the name, require
`[a-z0-9][a-z0-9+_-]*`, and cap it at 128 bytes. Empty, non-ASCII, control,
whitespace-containing, or overlong values are ignored silently.
The restriction is intentionally narrower than `pmacs.buffer.set_major_mode`,
which continues accepting arbitrary Lua strings for trusted configuration.
Untrusted file content receives no path to control characters, huge statusline
values, Lua evaluation, or option mutation.
### Q#MD4 — Normalize common external names through one alias table
Expose a user-extensible Lua table:
```lua
pmacs.parse.modeline_aliases = {
["c++"] = "cpp",
cxx = "cpp",
sh = "bash",
shell = "bash",
["shell-script"] = "bash",
zsh = "bash",
py = "python",
js = "javascript",
js2 = "javascript",
jsx = "javascriptreact",
ts = "typescript",
tsx = "typescriptreact",
yml = "yaml",
makefile = "make",
docker = "dockerfile",
}
```
A normalized name absent from the table passes through unchanged. Users may
add, replace, or remove aliases in `init.lua`; defaults use `or`-style seeding
so preconfigured entries are not overwritten. Alias outputs must satisfy the
same 128-byte token rule before use.
This keeps canonical pmacs names stable without pretending that extension maps
and editor-mode names are the same namespace. Do not strip a trailing `-mode`:
GNU Emacs explicitly specifies the value without that suffix, and silent
stripping would make custom names ambiguous.
### Q#MD5 — Explicit modelines override inference
The final fresh-load order is:
1. modeline;
2. bundled grammar extension;
3. `pmacs.lsp.filetypes` extension;
4. exact basename;
5. shebang.
A modeline is explicit file metadata; the remaining signals are inference. Thus
a `template.txt` containing `vim: set ft=python:` selects `python`, and a
misnamed `.py` file containing `-*- mode: lua -*-` selects `lua` consistently
for mode, parser, and LSP.
This interprets the backlog's “fifth layer after extension → filetype → filename
→ shebang” as a layer added after that work, not as a lowest-priority fallback.
Making explicit metadata lose to a suffix would defeat the feature's primary
use case.
### Q#MD6 — One resolver owns the effective language
`syntax.lua` owns:
- `pmacs.parse.language_from_modeline(buf)`: parse current content and return
the normalized modeline language or nil;
- the private fresh inference chain;
- `pmacs.parse.buffer_language(buf)`: return the language pinned for this
buffer's current load, resolving once only for an unseen buffer.
The syntax `buffer.after-load` path forces a fresh inference, records either the
language or an explicit “resolved none” sentinel, then uses that same value for
mode initialization and grammar dispatch. `buffer.after-switch` consumes the
pin and resolves only the pre-existing hidden-buffer case that never received
an after-load event.
`pmacs.lsp.buffer_language(buf)` keeps its current path requirement, then
delegates to `pmacs.parse.buffer_language(buf)`. The active-buffer wrapper stays
unchanged. This preserves pathless LSP behavior while deleting the duplicate
extension/filetype/filename/shebang chain.
The pin also closes an existing shebang inconsistency: editing `#!/bin/sh` to
`#!/usr/bin/env lua` no longer leaves a bash parse tree while making later
auto-pair/comment queries report Lua. Raw parser tests may call
`language_from_modeline`; behavior-driving consumers use the pin.
### Q#MD7 — Initial mode, syntax, and LSP share one value
On `buffer.after-load`, set the buffer's major mode to the freshly resolved
language, including nil when no signal resolves. This replaces the current
“only if nil” guard and makes the already-documented reload contract exact:
a fresh after-load decision replaces an earlier explicit override or clear.
Then:
- dispatch a grammar only if `pmacs.parse._has_language(lang)`;
- let LSP attach only if the same language has a configured server and a real
file path;
- retain a valid unknown language as the major mode, while silently skipping
grammar/LSP attachment.
After load, explicit `set_major_mode` remains independent: it immediately
changes mode key dispatch and statusline display but does not rewrite the
pinned parser/LSP language. Switches preserve both values.
### Q#MD8 — Malformed or unsupported metadata is fail-closed and quiet
A malformed marker, invalid name, unsupported form, or truncated candidate
never raises from `buffer.after-load` and never emits an `*errors*` entry. The
detector returns nil and the existing inference chain continues.
A syntactically valid unknown name is different: it is a legitimate major mode
and is pinned, but `_has_language` and LSP config gates prevent a bogus parser
or server launch. This preserves #129's custom-mode capability without
executing file content.
A file can already select a configured language—and therefore which LSP server
pmacs starts—through its extension or a content-sniffed shebang. Modelines add
another bounded language selector within that existing capability class; they
do not introduce automatic execution beyond what current language detection
already permits.
No enable/disable setting is added. The supported input can only select a
bounded string already consumed as passive mode/language identity; it cannot
run hooks or set options. If a future `buffer.after-mode-change` hook makes mode
selection executable, modeline trust must be revisited in that feature's
framing.
### Q#MD9 — No Rust or protocol surface is required
Expected implementation touch set:
- `builtin/runtime/syntax.lua` — bounded parser, aliases, shared resolver, pin,
and after-load initialization;
- `builtin/runtime/lsp.lua` — delegate language inference while retaining the
path guard;
- `tests/m4_acceptance.rs` — parser, precedence, lifecycle, and end-to-end
regression coverage;
- this framing and the side-quest/handoff state documents when the feature
lands.
No changes are expected in `src/buffer.rs`, Lua bindings, frontends,
`pmacs-protocol`, or protocol version 18.
## Bets
1. **Sixteen KiB of edge text is sufficient.** Real modelines are short; files
with an 8 KiB first/last candidate line are better treated as malformed than
copied wholesale during load.
2. **One canonical language should drive mode, parser, and LSP initially.** A
future distinction between editor mode and parser language needs a real
consumer and an explicit mapping contract, not accidental divergence.
3. **ASCII-lowercased identifiers cover interoperable modelines.** Trusted Lua
remains available for arbitrary UTF-8 custom mode names.
4. **Load-time detection is enough.** Live cookie edits would require orderly
parser teardown, LSP `didClose`/`didOpen`, overlay replacement, mode-change
notification, and failure rollback; that is not a one-shot detector.
5. **Pathless LSP buffers stay ineligible.** Syntax may still infer from a
buffer name, but spawning a server without a URI/project root remains wrong.
## Acceptance
All end-to-end fixtures clear `pmacs.lsp.config` unless the case intentionally
observes server selection, so opening a test file never starts a machine-local
language server.
1. **Emacs forms:** first-line property and shorthand cookies resolve; a cookie
on line 2 resolves only after a shebang; unrelated properties are ignored;
the last `mode` property wins.
2. **Vim forms:** `vim:`/`vi:` direct and `set` forms resolve `ft` and
`filetype` in the first/last five lines; CRLF works; `Vim:` requires `set`.
The direct `vim:ft=sh:et:sw=2:` form resolves `sh`, while
`vim: set sw=4: ft=python` ignores the assignment after the terminating
colon. `ft:python` and `filetype:python` resolve nothing in either form.
3. **Boundary rejection:** sixth-line, sixth-from-end, middle-of-file,
word-embedded, unterminated, truncated, invalid-character, and overlong
candidates do not resolve and do not log errors. A partial line at the start
of the suffix byte window is discarded without consuming one of the five
complete tail-line slots.
4. **Conflict order:** overlapping edge windows are deduplicated and the last
valid assignment in document order wins deterministically.
5. **Alias behavior:** seeded aliases map `sh`/`zsh → bash`, `c++ → cpp`,
`js2 → javascript`, `tsx → typescriptreact`, and `docker → dockerfile`; a
user override wins; an invalid alias output is ignored.
6. **Explicit precedence:** a `.py` file with a Lua modeline yields `lua` from
`pmacs.parse.buffer_language`, `pmacs.lsp.active_buffer_language`, the parse
tree, and `pmacs.buffer.major_mode`.
7. **Unknown valid mode:** a `.txt` file with `mode: prose` receives major mode
`prose`, creates no parse view, starts no LSP server, and produces no error.
8. **No-modeline regression:** extension, filetype, filename, and shebang cases
retain their present precedence and outputs.
9. **Shebang pin regression:** open an extensionless `#!/bin/sh` file, replace
its shebang with `#!/usr/bin/env lua`, and assert the parse tree and
`pmacs.lsp.buffer_language` both remain `bash`.
10. **Pinned modeline lifecycle:** changing a loaded modeline does not change
the pinned language, parser, or major mode; switch-away/back remains stable;
close and reopen re-evaluates the changed on-disk cookie.
11. **Explicit override independence:** `set_major_mode` after load changes
dispatch/statusline but not `pmacs.parse.buffer_language`; switches preserve
the override.
12. **Pathless preservation:** syntax-by-buffer-name behavior remains, while
`pmacs.lsp.buffer_language` still returns nil without a backing path.
13. **Backend parity:** focused acceptance passes with default LuaJIT and
`--no-default-features --features lua54`; protocol remains v18.

327
tests/m4_acceptance.rs
View File

@ -6101,6 +6101,17 @@ fn m4_shebang_edit_keeps_pinned_grammar() {
Some("bash"),
"editing the shebang must not re-switch the pinned grammar"
);
let lsp_language: Option<String> = s
.lua_host
.lua()
.load("return pmacs.lsp.buffer_language(pmacs.window.buffer())")
.eval()
.expect("pinned LSP language");
assert_eq!(
lsp_language.as_deref(),
Some("bash"),
"language-aware consumers must share the shebang pin"
);
// Switch away to another buffer and back: the after-switch reattach
// must reuse the pinned bash grammar rather than re-sniff the (now
@ -6137,6 +6148,322 @@ fn m4_shebang_edit_keeps_pinned_grammar() {
assert_eq!(errs, 0, "reparse with the pinned grammar reports no error");
}
/// Modeline smoke: explicit file metadata overrides a misleading extension,
/// and syntax, LSP language introspection, and initial major mode agree.
#[test]
fn m4_modeline_overrides_extension_end_to_end() {
use pmacs::editor::EditorState;
let mut s = EditorState::new();
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("misleading.py");
std::fs::write(&file, b"-- -*- mode: lua -*-\nprint('ok')\n").expect("write");
let file_disp = file.display();
s.lua_host
.lua()
.load(format!(
"pmacs.lsp.config = {{}}
pmacs.buffer.find_or_open('{file_disp}')"
))
.exec()
.expect("open modeline fixture");
let (parsed, lsp, mode): (Option<String>, Option<String>, Option<String>) = s
.lua_host
.lua()
.load(
"local b = pmacs.window.buffer()
return pmacs.parse.buffer_language(b),
pmacs.lsp.active_buffer_language(),
pmacs.buffer.major_mode(b)",
)
.eval()
.expect("modeline language surfaces");
assert_eq!(parsed.as_deref(), Some("lua"));
assert_eq!(lsp.as_deref(), Some("lua"));
assert_eq!(mode.as_deref(), Some("lua"));
pump_async(&mut s, |st| {
current_tree_language(st).as_deref() == Some("lua")
});
}
#[test]
fn m4_modeline_parser_matches_supported_emacs_and_vim_forms() {
use pmacs::editor::EditorState;
let s = EditorState::new();
let resolve = |text: &str| -> Option<String> {
s.lua_host
.lua()
.load(format!(
"local b = pmacs.window.buffer()
if b:len() > 0 then b:delete(0, b:len()) end
b:insert(0, {text:?})
return pmacs.parse.language_from_modeline(b)"
))
.eval()
.expect("resolve modeline")
};
for (text, want) in [
("# -*- mode: Python; coding: utf-8 -*-\n", Some("python")),
("-- -*- Lua -*-\n", Some("lua")),
("#!/usr/bin/env python\n# -*- mode: Lua -*-\n", Some("lua")),
("# -*- mode: python; mode: lua -*-\n", Some("lua")),
("vim:ft=python:sw=4:\n", Some("python")),
("# vim: set ft=lua sw=2:\n", Some("lua")),
("# vi:filetype=yaml:et:\n", Some("yaml")),
("# Vim: set filetype=toml:\n", Some("toml")),
("one\ntwo\nthree\nfour\nfive\n# vim:ft=lua:\n", Some("lua")),
("# vim: set ft=python:\r\n", Some("python")),
] {
assert_eq!(resolve(text).as_deref(), want, "{text:?}");
}
for text in [
"plain\n# -*- mode: lua -*-\n", // line 2 needs a shebang
"# Vim:ft=lua:\n", // uppercase marker requires `set`
"# Vim: se ft=lua:\n", // uppercase marker requires literal `set`
"# vim: set sw=4: ft=python\n", // assignment follows the terminator
"# vim: set ft=python\n", // set form needs a terminator
"# vim:ft:python:\n", // colon is an option separator
"# vim: set ft:python :\n", // colon terminates the option section
] {
assert_eq!(resolve(text), None, "{text:?}");
}
}
#[test]
fn m4_modeline_parser_enforces_boundaries_aliases_and_conflicts() {
use pmacs::editor::EditorState;
let s = EditorState::new();
let resolve = |text: &str| -> Option<String> {
s.lua_host
.lua()
.load(format!(
"local b = pmacs.window.buffer()
if b:len() > 0 then b:delete(0, b:len()) end
b:insert(0, {text:?})
return pmacs.parse.language_from_modeline(b)"
))
.eval()
.expect("resolve modeline")
};
for (text, want) in [
("# vim:ft=zsh:\n", "bash"),
("# -*- mode: C++ -*-\n", "cpp"),
("# -*- mode: js2 -*-\n", "javascript"),
("# vim:ft=tsx:\n", "typescriptreact"),
("# vim:ft=docker:\n", "dockerfile"),
] {
assert_eq!(resolve(text).as_deref(), Some(want), "{text:?}");
}
let conflict = "# -*- mode: python; mode: yaml -*-\n2\n3\n4\n5\n# vim:ft=lua:\n";
assert_eq!(
resolve(conflict).as_deref(),
Some("lua"),
"last valid assignment in document order wins across overlapping edges"
);
let middle = "1\n2\n3\n4\n5\n# vim:ft=lua:\n7\n8\n9\n10\n11\n";
assert_eq!(
resolve(middle),
None,
"sixth line from both edges is outside the scan"
);
let partial_with_live_tail =
format!("{}\n# vim:ft=lua:\n1\n2\n3\n4", "x".repeat(8 * 1024 + 64));
assert_eq!(
resolve(&partial_with_live_tail).as_deref(),
Some("lua"),
"discarded suffix fragment does not consume a tail-line slot"
);
let marker_in_partial = format!("{} vim:ft=lua:\n1\n2\n3\n4\n5", "x".repeat(8 * 1024 + 64));
assert_eq!(
resolve(&marker_in_partial),
None,
"modeline in a truncated edge line is ignored"
);
let overlong = format!("# vim:ft={}:\n", "a".repeat(129));
for text in [
"prefixvim:ft=lua:\n".to_owned(),
"# vim:ft=lua!:\n".to_owned(),
overlong,
] {
assert_eq!(resolve(&text), None, "{text:?}");
}
s.lua_host
.lua()
.load("pmacs.parse.modeline_aliases.sh = 'lua'")
.exec()
.expect("override modeline alias");
assert_eq!(resolve("# vim:ft=sh:\n").as_deref(), Some("lua"));
s.lua_host
.lua()
.load("pmacs.parse.modeline_aliases.sh = 'BAD VALUE'")
.exec()
.expect("install invalid modeline alias");
assert_eq!(resolve("# vim:ft=sh:\n"), None);
s.lua_host
.lua()
.load("pmacs.parse.modeline_aliases.sh = 'bash'")
.exec()
.expect("restore modeline alias");
}
#[test]
fn m4_modeline_unknown_mode_is_quiet_and_parser_free() {
use pmacs::editor::EditorState;
let s = EditorState::new();
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("notes.txt");
std::fs::write(&file, b"# vim:ft=prose:\nhello\n").expect("write");
let file_disp = file.display();
let (mode, language, has_view, errors): (Option<String>, Option<String>, bool, i64) = s
.lua_host
.lua()
.load(format!(
"pmacs.lsp.config = {{}}
_G.__modeline_errors = {{}}
local real_error = pmacs.error
pmacs.error = function(message)
table.insert(_G.__modeline_errors, message)
end
local b = pmacs.buffer.find_or_open('{file_disp}')
local mode = pmacs.buffer.major_mode(b)
local language = pmacs.parse.buffer_language(b)
local has_view = pmacs.parse._has_view(b)
local errors = #_G.__modeline_errors
pmacs.error = real_error
return mode, language, has_view, errors"
))
.eval()
.expect("open unknown modeline mode");
assert_eq!(mode.as_deref(), Some("prose"));
assert_eq!(language.as_deref(), Some("prose"));
assert!(!has_view, "unknown modeline must not dispatch a parser");
assert_eq!(errors, 0, "unknown modeline must not report an error");
}
#[test]
fn m4_modeline_language_is_pinned_until_reopen() {
use pmacs::editor::EditorState;
let mut s = EditorState::new();
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("mutable.txt");
let other = dir.path().join("other.txt");
std::fs::write(&file, b"-- -*- mode: lua -*-\nprint('one')\n").expect("write");
std::fs::write(&other, b"other\n").expect("write other");
let file_disp = file.display();
let other_disp = other.display();
s.lua_host
.lua()
.load(format!(
"pmacs.lsp.config = {{}}
_G.MODELINE_BUFFER = pmacs.buffer.find_or_open('{file_disp}')"
))
.exec()
.expect("open mutable modeline fixture");
pump_async(&mut s, |st| {
current_tree_language(st).as_deref() == Some("lua")
});
let (language, mode): (Option<String>, Option<String>) = s
.lua_host
.lua()
.load(
"local b = MODELINE_BUFFER
local text = b:slice(0, b:len())
local start = assert(text:find('lua', 1, true)) - 1
b:replace(start, start + 3, 'python')
pmacs.hook.run('buffer.after-edit')
return pmacs.lsp.buffer_language(b), pmacs.buffer.major_mode(b)",
)
.eval()
.expect("edit loaded modeline");
assert_eq!(language.as_deref(), Some("lua"));
assert_eq!(mode.as_deref(), Some("lua"));
for _ in 0..64 {
s.tick_async();
std::thread::sleep(Duration::from_millis(2));
}
assert_eq!(current_tree_language(&s).as_deref(), Some("lua"));
let (language, mode): (Option<String>, Option<String>) = s
.lua_host
.lua()
.load(format!(
"pmacs.buffer.set_major_mode(MODELINE_BUFFER, 'markdown')
pmacs.buffer.find_or_open('{other_disp}')
pmacs.window.switch_buffer(MODELINE_BUFFER)
return pmacs.lsp.buffer_language(MODELINE_BUFFER),
pmacs.buffer.major_mode(MODELINE_BUFFER)"
))
.eval()
.expect("switch with explicit major-mode override");
assert_eq!(language.as_deref(), Some("lua"));
assert_eq!(mode.as_deref(), Some("markdown"));
s.lua_host
.lua()
.load(format!("pmacs.buffer.find_or_open('{other_disp}')"))
.exec()
.expect("switch away before removing old buffer");
std::fs::write(&file, b"# -*- mode: python -*-\nprint('two')\n").expect("rewrite");
let (new_id, language, mode): (String, Option<String>, Option<String>) = s
.lua_host
.lua()
.load(format!(
"local old = MODELINE_BUFFER
pmacs.buffer.remove(old)
local reopened = pmacs.buffer.find_or_open('{file_disp}')
return tostring(reopened), pmacs.lsp.buffer_language(reopened),
pmacs.buffer.major_mode(reopened)"
))
.eval()
.expect("reopen changed modeline fixture");
assert_ne!(
new_id,
s.lua_host
.lua()
.load("return tostring(MODELINE_BUFFER)")
.eval::<String>()
.unwrap(),
"reopen must allocate a fresh buffer id"
);
assert_eq!(language.as_deref(), Some("python"));
assert_eq!(mode.as_deref(), Some("python"));
pump_async(&mut s, |st| {
current_tree_language(st).as_deref() == Some("python")
});
}
#[test]
fn m4_modeline_shared_resolver_preserves_pathless_lsp_guard() {
use pmacs::editor::EditorState;
let s = EditorState::new();
let (syntax, lsp): (Option<String>, Option<String>) = s
.lua_host
.lua()
.load(
"local b = pmacs.buffer.create('scratch.lua')
return pmacs.parse.buffer_language(b), pmacs.lsp.buffer_language(b)",
)
.eval()
.expect("resolve pathless language");
assert_eq!(syntax.as_deref(), Some("lua"));
assert_eq!(lsp, None, "LSP requires a backing path");
}
/// Filename detection: `pmacs.parse.language_from_filename` maps a
/// basename (Dockerfile / Makefile / CMakeLists.txt / rc dotfiles) to a
/// language, resolving a full path too, and returns nil for a plain file.