diff --git a/builtin/runtime/lsp.lua b/builtin/runtime/lsp.lua index 251f4b7..d7ab9d9 100644 --- a/builtin/runtime/lsp.lua +++ b/builtin/runtime/lsp.lua @@ -471,33 +471,21 @@ end local function buffer_language(buf) local ok, path = pcall(function() return buf and buf:path() end) if not ok or not path then return nil end - -- Grammar-backed detection first (keeps rust/.rs etc. exactly as - -- before); fall back to the LSP-only filetype map so languages with a - -- server but no tree-sitter grammar (Python) still attach; then the - -- basename map for filename-identified files (`Dockerfile`, - -- `CMakeLists.txt`); finally, for an extensionless file, sniff a - -- `#!interp` shebang so e.g. `#!/bin/sh` still attaches its server. - local lang = pmacs.parse.language_for_path(path) - if lang then return lang end - local ext = path:match("%.([%w_]+)$") - local by_ext = ext and pmacs.lsp.filetypes[ext] - if by_ext then return by_ext end - local by_name = pmacs.parse.language_from_filename(path) - if by_name then return by_name end - return pmacs.parse.language_from_shebang(buf) + -- Syntax owns the fresh-load inference and pin. LSP retains only its + -- path-eligibility rule: without a backing path it cannot construct a URI or + -- project root, even when syntax can infer a grammar from a buffer name. + return pmacs.parse.buffer_language(buf) end --- Public: the per-buffer language chain. Auto-pairing resolves --- relevance against the buffer its typed-edit record names — which a --- context-switching command may have left inactive by callback time — --- so the parameterized form is the primitive and the active-buffer --- form delegates. +-- Public: the pinned per-buffer language. Auto-pairing resolves relevance +-- against the buffer its typed-edit record names — which a context-switching +-- command may have left inactive by callback time — so the parameterized form +-- is the primitive and the active-buffer form delegates. pmacs.lsp.buffer_language = buffer_language local function active_buffer_language() return buffer_language(pmacs.window.buffer()) end --- Public: the comment-toggle module (and future language-aware Lua) --- reuses this grammar+filetypes chain instead of replicating it. +-- Public: comment-toggle and other language-aware Lua reuse the same pin. pmacs.lsp.active_buffer_language = active_buffer_language -- Directory component of a path, or nil if it has none. @@ -714,7 +702,7 @@ local function attach_buffer(buf) -- carries the full current text, superseding them. pending_did_change[key] = nil end - local language = active_buffer_language() + local language = buffer_language(buf) if not language then return nil end -- Path resolved before spawn so the server's `rootUri` can be -- derived from the file's project (see `project_root_for`). diff --git a/builtin/runtime/syntax.lua b/builtin/runtime/syntax.lua index 2d33368..812e621 100644 --- a/builtin/runtime/syntax.lua +++ b/builtin/runtime/syntax.lua @@ -22,6 +22,10 @@ local inflight_parse_by_buffer = {} local parse_buffer_by_key = {} local parse_lang_by_buffer = {} local reparse_requested_by_buffer = {} +-- Fresh-load language decision, including an explicit false sentinel for +-- "resolved none". Syntax, LSP, pairing, comments, and initial major mode all +-- consume this pin rather than re-sniffing mutable file content independently. +local detected_language_by_buffer = {} -- Buffers already warned about hitting the injection layer cap (Q#IJ3); -- keyed like the others so we warn once, and re-arm if the file stops -- capping (an edit removed the excess regions). @@ -200,21 +204,254 @@ function pmacs.parse.language_from_filename(name) return pmacs.parse.filenames[base] end +-- Modeline → language detection --------------------------------------------- + +local MODELINE_WINDOW_BYTES = 8 * 1024 +local VIM_MODELINE_LINES = 5 +local MODELINE_NAME_BYTES = 128 + +pmacs.parse.modeline_aliases = pmacs.parse.modeline_aliases or {} +local default_modeline_aliases = { + ["c++"] = "cpp", + cxx = "cpp", + sh = "bash", + shell = "bash", + ["shell-script"] = "bash", + zsh = "bash", + py = "python", + js = "javascript", + js2 = "javascript", + jsx = "javascriptreact", + ts = "typescript", + tsx = "typescriptreact", + yml = "yaml", + makefile = "make", + docker = "dockerfile", +} +for name, language in pairs(default_modeline_aliases) do + if pmacs.parse.modeline_aliases[name] == nil then + pmacs.parse.modeline_aliases[name] = language + end +end + +local function normalize_modeline_name(name) + if type(name) ~= "string" then return nil end + name = name:gsub("^[ \t]+", ""):gsub("[ \t]+$", ""):lower() + if #name == 0 or #name > MODELINE_NAME_BYTES then return nil end + if not name:match("^[a-z0-9][a-z0-9+_-]*$") then return nil end + local alias = pmacs.parse.modeline_aliases[name] + if alias == nil then return name end + if type(alias) ~= "string" or #alias == 0 or #alias > MODELINE_NAME_BYTES then + return nil + end + if not alias:match("^[a-z0-9][a-z0-9+_-]*$") then return nil end + return alias +end + +local function without_trailing_cr(line) + if line:sub(-1) == "\r" then return line:sub(1, -2) end + return line +end + +-- Return the first five and last five complete logical lines. Each entry is +-- `{ text, offset }`, where offset is the zero-based buffer byte position. +-- The suffix's leading partial line is discarded and does not consume a slot. +local function modeline_edge_lines(buf) + local ok_len, length = pcall(function() return buf:len() end) + if not ok_len or not length or length <= 0 then return {}, {} end + + local prefix_end = math.min(length, MODELINE_WINDOW_BYTES) + local ok_prefix, prefix = pcall(function() return buf:slice(0, prefix_end) end) + if not ok_prefix or type(prefix) ~= "string" then return {}, {} end + + local front = {} + local pos = 1 + while #front < VIM_MODELINE_LINES and pos <= #prefix do + local newline = prefix:find("\n", pos, true) + if not newline then + if prefix_end < length then break end + newline = #prefix + 1 + end + front[#front + 1] = { + text = without_trailing_cr(prefix:sub(pos, newline - 1)), + offset = pos - 1, + } + if newline > #prefix then break end + pos = newline + 1 + end + + local tail_start = 0 + local scan_start = 1 + if length > MODELINE_WINDOW_BYTES then + -- Keep the one-byte line-boundary probe plus suffix content within the + -- same 8 KiB read budget. + tail_start = length - (MODELINE_WINDOW_BYTES - 1) + local ok_probe, probe = + pcall(function() return buf:slice(tail_start - 1, tail_start) end) + if not ok_probe or probe ~= "\n" then + scan_start = nil + end + end + + local tail = prefix + if tail_start > 0 then + local ok_tail + ok_tail, tail = pcall(function() return buf:slice(tail_start, length) end) + if not ok_tail or type(tail) ~= "string" then return front, front end + end + if scan_start == nil then + local first_newline = tail:find("\n", 1, true) + if not first_newline then return front, front end + scan_start = first_newline + 1 + end + + local reverse_tail = {} + local line_end = #tail + if line_end >= scan_start and tail:byte(line_end) == 10 then + line_end = line_end - 1 + end + while line_end >= scan_start and #reverse_tail < VIM_MODELINE_LINES do + local i = line_end + while i >= scan_start and tail:byte(i) ~= 10 do + i = i - 1 + end + local line_start = i + 1 + reverse_tail[#reverse_tail + 1] = { + text = without_trailing_cr(tail:sub(line_start, line_end)), + offset = tail_start + line_start - 1, + } + line_end = i - 1 + end + + local edges = {} + local seen = {} + for _, entry in ipairs(front) do + edges[#edges + 1] = entry + seen[entry.offset] = true + end + for i = #reverse_tail, 1, -1 do + local entry = reverse_tail[i] + if not seen[entry.offset] then + edges[#edges + 1] = entry + seen[entry.offset] = true + end + end + return front, edges +end + +local function emacs_mode_on_line(entry, consider) + local line = entry.text + local search_from = 1 + while true do + local open = line:find("-*-", search_from, true) + if not open then return end + local close = line:find("-*-", open + 3, true) + if not close then return end + local payload = line:sub(open + 3, close - 1) + if payload:find(":", 1, true) then + for part_at, part in payload:gmatch("()([^;]+)") do + local key, value = + part:match("^[ \t]*([%w_-]+)[ \t]*:[ \t]*(.-)[ \t]*$") + if key and key:lower() == "mode" then + local mode = normalize_modeline_name(value) + if mode then consider(mode, entry.offset + open + part_at) end + end + end + else + local mode = normalize_modeline_name(payload) + if mode then consider(mode, entry.offset + open) end + end + search_from = close + 3 + end +end + +local function vim_assignment(token) + return token:match("^ft=(.+)$") or token:match("^filetype=(.+)$") +end + +local function vim_mode_at(entry, marker_at, marker, consider) + local line = entry.text + local rest_at = marker_at + #marker + local rest = line:sub(rest_at) + local full_set_at = rest:match("^[ \t]*set[ \t]+()") + local option_at = full_set_at + if not option_at and marker ~= "Vim:" then + option_at = rest:match("^[ \t]*se[ \t]+()") + end + if marker == "Vim:" and not full_set_at then return end + + if option_at then + local terminator = rest:find(":", option_at, true) + if not terminator then return end + local options = rest:sub(option_at, terminator - 1) + for token_at, token in options:gmatch("()([^ \t]+)") do + local value = vim_assignment(token) + local mode = value and normalize_modeline_name(value) + if mode then + consider(mode, entry.offset + rest_at + option_at + token_at - 3) + end + end + return + end + + for token_at, token in rest:gmatch("()([^ \t:]+)") do + local value = vim_assignment(token) + local mode = value and normalize_modeline_name(value) + if mode then consider(mode, entry.offset + rest_at + token_at - 2) end + end +end + +local function vim_modes_on_line(entry, consider) + local line = entry.text + for _, marker in ipairs({ "vim:", "vi:", "Vim:" }) do + local search_from = 1 + while true do + local marker_at = line:find(marker, search_from, true) + if not marker_at then break end + local previous = marker_at > 1 and line:sub(marker_at - 1, marker_at - 1) + if marker_at == 1 or previous == " " or previous == "\t" then + vim_mode_at(entry, marker_at, marker, consider) + end + search_from = marker_at + 1 + end + end +end + +function pmacs.parse.language_from_modeline(buf) + if not buf then return nil end + local front, edges = modeline_edge_lines(buf) + local mode + local mode_at = -1 + local function consider(candidate, candidate_at) + if candidate_at >= mode_at then + mode = candidate + mode_at = candidate_at + end + end + + if front[1] then emacs_mode_on_line(front[1], consider) end + if front[1] and front[1].text:sub(1, 2) == "#!" and front[2] then + emacs_mode_on_line(front[2], consider) + end + for _, entry in ipairs(edges) do + vim_modes_on_line(entry, consider) + end + return mode +end + -- Set of buffer ids that already have a highlight overlay -- attached, keyed by raw id (number). A buffer that opens, gets -- highlights, gets killed, and is reopened needs a fresh overlay -- attach; the kill path clears the entry below if/when it lands. local highlighted_buffers = {} --- Filetype-aware language resolution for the active buffer, in precedence --- order: grammar extension → LSP filetype map → filename → shebang. A --- recognized extension is authoritative (a `.py` must not fall through to --- a stray `#!/bin/sh` and be misparsed as bash); the basename map handles --- extensionless `Dockerfile`/`Makefile`/rc-dotfiles, and only then does --- the shebang (buffer content) get a look. Keyed on `buf:name()` for the --- path parts (matching the historical behavior — path-less buffers that --- resolve a grammar by name keep working). -local function resolve_active_language(buf) +-- Fresh language inference for a buffer, in precedence order: explicit +-- modeline → grammar extension → LSP filetype map → filename → shebang. +-- Path components intentionally come from `buf:name()` to preserve syntax's +-- historical grammar-by-name behavior for pathless buffers. +local function detect_buffer_language(buf) + local modeline = pmacs.parse.language_from_modeline(buf) + if modeline then return modeline end local name = buf:name() if name then local grammar = pmacs.parse.language_for_path(name) @@ -228,31 +465,34 @@ local function resolve_active_language(buf) return pmacs.parse.language_from_shebang(buf) end +local function refresh_buffer_language(buf) + local language = detect_buffer_language(buf) + detected_language_by_buffer[tostring(buf)] = language or false + return language +end + +function pmacs.parse.buffer_language(buf) + if not buf then return nil end + local key = tostring(buf) + local language = detected_language_by_buffer[key] + if language ~= nil then return language or nil end + return refresh_buffer_language(buf) +end + local function attach_for_active_buffer(initialize_mode) local buf = pmacs.window.buffer() if not buf then return end local key = tostring(buf) - -- Reuse the language pinned at first attach if this buffer already has - -- a parse view. A switch-away/back re-runs this hook (via after-switch); - -- re-resolving there would re-sniff a shebang the user has since edited - -- and silently swap the grammar — and diverge from the LSP side, which - -- keeps its existing attachment across the switch. A first-seen buffer - -- (no view yet) resolves normally. Gate dispatch on `_has_language`: the - -- resolution chain can still yield a language with no grammar — a - -- shebang or filetype mapping to a server-only or unsupported language - -- (e.g. an init.lua `pmacs.parse.shebangs.ruby = "ruby"`) — and - -- dispatching one would raise "unknown language" (caught, but noise) and - -- never gives a wrong-grammar tree. - local lang = pmacs.parse._has_view(buf) and parse_lang_by_buffer[key] - or resolve_active_language(buf) - -- The detected language is also the initial major-mode name. Do this - -- before grammar gating: a language supplied only by an LSP filetype or - -- shebang mapping is still a valid mode even when no parser is bundled. - -- Only after-load initializes it; after-switch must preserve explicit - -- overrides and explicit nil clears. - if initialize_mode and lang and pmacs.buffer.major_mode(buf) == nil then - pmacs.buffer.set_major_mode(buf, lang) - end + -- A genuine load refreshes every detection signal and replaces the initial + -- major mode, including with nil. Switches consume the pin: editing a + -- shebang/modeline cannot silently swap parser/LSP language, while a + -- registry-only hidden buffer still resolves when first visited. + local lang = initialize_mode and refresh_buffer_language(buf) + or pmacs.parse.buffer_language(buf) + if initialize_mode then pmacs.buffer.set_major_mode(buf, lang) end + -- The resolution chain can yield a valid mode with no grammar. Gate dispatch + -- so custom/server-only modes stay quiet rather than raising "unknown + -- language" from the parse worker. if not lang or not pmacs.parse._has_language(lang) then return end pmacs.parse._dispatch(buf, lang) -- T M4.3: install the syntax-highlight overlay for this buffer. diff --git a/docs/active-work.md b/docs/active-work.md index 909bf85..49e000f 100644 --- a/docs/active-work.md +++ b/docs/active-work.md @@ -14,8 +14,7 @@ backlog. machine-local: `origin` may name this canonical URL, a release mirror, or something else, and therefore has no authority by name alone. - Canonical base at this snapshot: - `githubsucks/main` @ `d5d9b9c` (mode system handoff #131 merged; - protocol v18). + `githubsucks/main` @ `86fc1bc` (Vterm Stage 2 #130 merged; protocol v18). - On the transfer source, `origin/main` named a release mirror at `d3fa632` and lagged badly. On the current destination, `origin` names the canonical URL. This difference is why all recovery begins by @@ -49,55 +48,44 @@ git worktree list git status --short --branch ``` -The first command must expose `d5d9b9c` or a newer intentional main. +The first command must expose `86fc1bc` or a newer intentional main. If it does not, stop and repair the remote/fetch configuration. -## Vterm Stage 2 implementation lane +## Modeline detection implementation lane -- Portable branch: `githubsucks/vterm-tui` -- Integrated feature head: `3f0252f` (review-fix head `b9a7e40`) -- Base: originally canonical `main` @ `f1a2f75`; current canonical `main` - @ `d5d9b9c` (mode system wiring #129 and handoff #131) is integrated before - merge. -- PR: #130, , open against +- Portable branch: `githubsucks/modeline-detection` +- Approved framing head: `6b4f3c3` +- Implementation head: `f8d05d2` +- Base: originally canonical `main` @ `d5d9b9c`; current canonical `main` + @ `86fc1bc` (Vterm Stage 2 #130) is integrated before merge. +- PR: #132, , open against canonical `main` and explicitly authorized for merge. -- State: `docs/vterm-framing.md` Revision 7 criteria 15–27 are implemented. - The lane composes per-frontend/window terminal views in the TUI, installs - the strict `pmacs.terminal` API and terminal-local bindings, drains - clipboard/BEL through the authenticated frontend, and routes daemon - terminal input by connection source. Protocol remains v18; Stage 2 changes - neither the wire schema nor the GPU renderer. -- Review round 1: addressed. Dispatch now requires `C-c` before terminal-local - editor bindings; non-terminal context operations error; controller - replacement is atomic per frontend; zero-area layouts retain view anchors; - view projection borrows retained rows instead of deep-cloning scrollback. -- Review round 2: addressed. Partial eviction clamps anchors to the first - surviving wrapped-line cell; `invoke_interactive` now inherits only an - authenticated dispatch origin; explicit context failures are named; terminal - mouse routing reads geometry without cloning cells; the framing records the - v18 semantic-controller boundary and bracketed-paste injection deferral. -- Implementation commits: `39e07cb`, `7c39535`, `0a846d9`, `0dacac7`, - `dc92257`, merge `0ddff24`, integration hardening `da8f6ae`, first-review - fixes `8702791`, second-review fixes `b9a7e40`, and current-main integration - `3f0252f`. -- Post-integration verification: `cargo fmt --check`; strict workspace - Clippy; 1,753 default + 1,929 CRDT library tests (3 ignored each); - mode-system acceptance 1 default + 1 CRDT; Stage 1 acceptance 9 default + - 10 CRDT; Stage 2 acceptance 4 default + 4 CRDT; statusline acceptance - 7 default + 8 CRDT; M4 114 passed (3 ignored, 1 filtered); required GPU - 109; workspace 2,882 passed across 82 suites (19 ignored, 1 filtered); - `git diff --check` clean. The first parallel M4 attempt timed out after - partial progress; a serial isolation pass and the immediate exact parallel - rerun both passed, and the workspace sweep also passed. -- Next: push the integrated head and merge PR #130 as authorized. +- State: Revision 2's bounded Emacs/Vim parser, explicit-over-inferred + precedence, alias normalization, shared fresh-load language pin, LSP path + guard, and all thirteen acceptance criteria are implemented. Protocol + remains v18. +- Verification: + - focused modeline + shebang-pin acceptance before integration: 7 passed on + default LuaJIT and 7 passed on non-default Lua 5.4; + - post-integration `cargo fmt --check`; + - post-integration `cargo clippy --workspace --all-targets -- -D warnings`; + - post-integration `cargo test --lib`: 1,753 passed; + - post-integration `cargo test --lib --features crdt`: 1,929 passed; + - post-integration `cargo test --test m4_acceptance -- --skip basedpyright`: + 120 passed, 3 ignored, 1 filtered; + - post-integration `PMACS_REQUIRE_GPU=1 cargo test -p pmacs-gpu`: 109 passed; + - post-integration `cargo test --workspace -- --skip basedpyright`: 2,888 + passed across 82 suites, 19 ignored, 1 filtered; + - `git diff --check`. +- Next: push the current-main integration and merge PR #132 as authorized. Recovery worktree on a machine that does not already own the branch: ```sh git worktree add --track \ - -b vterm-tui \ - ../pmacs-vterm-tui \ - githubsucks/vterm-tui + -b modeline-detection \ + ../pmacs-modeline-detection \ + githubsucks/modeline-detection ``` ## Parked lane: kill-ring browser + persistence diff --git a/docs/modeline-detection-framing.md b/docs/modeline-detection-framing.md new file mode 100644 index 0000000..e93e2d1 --- /dev/null +++ b/docs/modeline-detection-framing.md @@ -0,0 +1,381 @@ +# Modeline language detection — side quest + +**Status:** Revision 2, approved for implementation by the user on 2026-07-22. +No implementation was present at approval. +**Base:** `githubsucks/main` at `d5d9b9c`; protocol v18. + +## Problem + +Language inference currently uses four signals, in order: + +1. bundled grammar extension; +2. `pmacs.lsp.filetypes` extension; +3. exact basename; +4. shebang. + +That chain cannot classify a deliberately misleading or extensionless file when +its author supplied editor metadata such as `-*- mode: python -*-` or +`vim: set ft=python:`. Mode-system wiring #129 gives the result a persistent +per-buffer home and makes it observable through key dispatch, help, and the +statusline, but no modeline parser exists. + +The goal is one bounded, non-executing modeline detector whose result drives the +same initial language used by syntax, LSP, and `Buffer.major_mode`. This is a +language-detection feature, not a general file-local settings system. + +## Ground truth + +### Detection is duplicated today + +`builtin/runtime/syntax.lua::resolve_active_language` and +`builtin/runtime/lsp.lua::buffer_language` independently implement the same +extension → filetype → filename → shebang chain. They already have small but +important differences: + +- syntax uses `buf:name()`, preserving historical grammar-by-name behavior for + pathless buffers; +- LSP requires `buf:path()`, because it cannot construct a URI or project root + for a pathless buffer; +- syntax pins the grammar in `parse_lang_by_buffer` after dispatch, while the + public LSP language query re-sniffs mutable shebang text on every call. + +Adding the fifth signal to both copies would create a third convention and let +syntax, LSP, mode initialization, auto-pairing, and comment commands disagree. +The implementation must consolidate the actual inference in `syntax.lua` and +leave only the LSP path-eligibility guard in `lsp.lua`. + +### Hook order is already useful + +`src/editor.rs` loads `syntax.lua` before `lsp.lua`. Their `buffer.after-load` +callbacks therefore run in that registration order: + +1. syntax detects the language, initializes the major mode, and dispatches a + grammar when one exists; +2. LSP resolves the same buffer language and attaches a server when configured. + +The modeline result must be computed and pinned by step 1 so step 2 cannot +independently reinterpret the file. + +### Major mode and parser language have different later lifecycles + +At first load, the detected language is the initial major-mode name. Afterward: + +- explicit `pmacs.buffer.set_major_mode` changes dispatch/statusline only; +- syntax and LSP remain attached to their initially selected language; +- `buffer.after-switch` restores views without re-detecting; +- edits do not change the selected grammar. + +Modelines follow that same load-time contract. Editing a cookie is not a live +mode or parser switch. Close/reopen re-evaluates it. A future true reload that +fires `buffer.after-load` re-evaluates it as a fresh load; explicit mode +overrides and clears are not reload-persistent, matching the #129 framing. + +### Existing Lua reads are sufficient + +`BufferIdLua` exposes byte-length and byte-slice operations. Detection can read +bounded prefix/suffix windows without copying the whole buffer or adding a Rust +binding. The rope is byte-addressed, so a bounded slice does not need UTF-8 +boundary repair before Lua pattern matching. + +### Compatibility references + +The supported subset follows the documented, non-evaluating pieces of: + +- GNU Emacs, “Specifying File Variables”: + +- Vim, `:help modeline` and `:help 'modelines'`: + + +Compatibility is deliberately bounded below. pmacs does not become an Emacs +file-variable evaluator or a Vim `:set` interpreter. + +## Scope + +In scope: + +- Emacs `-*- mode: NAME -*-` and mode-only `-*- NAME -*-` cookies; +- Vim/Vi `ft=NAME` and `filetype=NAME` modelines; +- first/last-line scanning with fixed byte and line limits; +- canonical aliases for common external filetype names; +- modeline precedence over inferred path/shebang language; +- one shared, pinned per-buffer language decision; +- initial major mode, grammar, LSP, and language-aware Lua consumers agreeing; +- focused parser and end-to-end acceptance on both Lua backends. + +Out of scope: + +- Emacs `Local Variables:` tail blocks; +- variables other than Emacs `mode` or Vim `ft`/`filetype`; +- `eval`, Vim commands, option mutation, directory-local variables, or project + trust prompts; +- Vim `ex:` markers, version predicates, escaped option values, and combined + dotted filetypes; +- live re-detection after edits, saves, renames, or buffer switches; +- `buffer.after-mode-change`, minor modes, mode-scoped settings, `describe-mode`, + or session persistence of explicit mode overrides; +- a protocol change or frontend-specific work. + +## Decisions + +### Q#MD1 — Scan only bounded edge lines + +Read at most 8 KiB from each end of the buffer and inspect: + +- Emacs: line 1, or line 2 only when line 1 begins with `#!`; +- Vim/Vi: the first five and last five logical lines, matching Vim's default + `'modelines'=5` behavior. + +When the prefix and suffix overlap, deduplicate lines before parsing. Strip one +trailing `\r` so CRLF and LF behave identically. Discard the suffix window's +leading fragment when its line begins before the 8 KiB boundary; that fragment +does not count toward the five complete logical lines counted backward from +buffer end. No truncated candidate is parsed. Detection therefore allocates at +most 16 KiB per fresh load, independent of file size, and an adversarial giant +edge line cannot force a whole-buffer copy. + +The Emacs 3000-character tail `Local Variables:` mechanism is a separate parser +with comment-prefix/suffix rules and is excluded. + +### Q#MD2 — Recognize a conservative syntax subset + +Emacs: + +- require a complete pair of `-*-` delimiters on the eligible line; +- accept `mode: NAME` in a semicolon-separated property list; +- accept a mode-only payload such as `-*- Lisp -*-`; +- ignore every property except `mode`; +- when a cookie contains multiple valid `mode` properties, the last wins. + +Vim/Vi: + +- accept `vim:`, `vi:`, and `Vim:` at line start or preceded by ASCII space or + tab; uppercase `Vim:` requires literal `set` rather than abbreviated `se`, + matching Vim; +- accept only exact `ft=NAME` and `filetype=NAME` assignments; `ft:NAME` and + `filetype:NAME` are not modeline assignment forms and are rejected; +- in the direct form, split option tokens on ASCII whitespace and `:`, so the + common `vim:ft=python:sw=4:` form yields `ft=python`; +- in the `set` / `se` form (`se` is lowercase-marker-only), end the option + section at the first `:` and split only the preceding text on ASCII + whitespace; `vim: set sw=4: ft=python` therefore contains no live filetype + assignment; +- ignore all other live option tokens rather than interpreting them; +- require that terminating colon for the `set` / `se` form, so a comment suffix + is never consumed as an option value; +- reject `ex:`, Vim version predicates, and marker substrings embedded in a + word. + +Across all eligible lines, the last valid mode assignment in document order +wins. This matches Emacs's “final defined mode” behavior and ordinary sequential +option assignment. A footer Vim modeline can intentionally override a header +Emacs cookie; conflicting metadata does not depend on Lua table iteration. + +### Q#MD3 — Modeline names are data, never code + +Trim ASCII edge whitespace, ASCII-lowercase the name, require +`[a-z0-9][a-z0-9+_-]*`, and cap it at 128 bytes. Empty, non-ASCII, control, +whitespace-containing, or overlong values are ignored silently. + +The restriction is intentionally narrower than `pmacs.buffer.set_major_mode`, +which continues accepting arbitrary Lua strings for trusted configuration. +Untrusted file content receives no path to control characters, huge statusline +values, Lua evaluation, or option mutation. + +### Q#MD4 — Normalize common external names through one alias table + +Expose a user-extensible Lua table: + +```lua +pmacs.parse.modeline_aliases = { + ["c++"] = "cpp", + cxx = "cpp", + sh = "bash", + shell = "bash", + ["shell-script"] = "bash", + zsh = "bash", + py = "python", + js = "javascript", + js2 = "javascript", + jsx = "javascriptreact", + ts = "typescript", + tsx = "typescriptreact", + yml = "yaml", + makefile = "make", + docker = "dockerfile", +} +``` + +A normalized name absent from the table passes through unchanged. Users may +add, replace, or remove aliases in `init.lua`; defaults use `or`-style seeding +so preconfigured entries are not overwritten. Alias outputs must satisfy the +same 128-byte token rule before use. + +This keeps canonical pmacs names stable without pretending that extension maps +and editor-mode names are the same namespace. Do not strip a trailing `-mode`: +GNU Emacs explicitly specifies the value without that suffix, and silent +stripping would make custom names ambiguous. + +### Q#MD5 — Explicit modelines override inference + +The final fresh-load order is: + +1. modeline; +2. bundled grammar extension; +3. `pmacs.lsp.filetypes` extension; +4. exact basename; +5. shebang. + +A modeline is explicit file metadata; the remaining signals are inference. Thus +a `template.txt` containing `vim: set ft=python:` selects `python`, and a +misnamed `.py` file containing `-*- mode: lua -*-` selects `lua` consistently +for mode, parser, and LSP. + +This interprets the backlog's “fifth layer after extension → filetype → filename +→ shebang” as a layer added after that work, not as a lowest-priority fallback. +Making explicit metadata lose to a suffix would defeat the feature's primary +use case. + +### Q#MD6 — One resolver owns the effective language + +`syntax.lua` owns: + +- `pmacs.parse.language_from_modeline(buf)`: parse current content and return + the normalized modeline language or nil; +- the private fresh inference chain; +- `pmacs.parse.buffer_language(buf)`: return the language pinned for this + buffer's current load, resolving once only for an unseen buffer. + +The syntax `buffer.after-load` path forces a fresh inference, records either the +language or an explicit “resolved none” sentinel, then uses that same value for +mode initialization and grammar dispatch. `buffer.after-switch` consumes the +pin and resolves only the pre-existing hidden-buffer case that never received +an after-load event. + +`pmacs.lsp.buffer_language(buf)` keeps its current path requirement, then +delegates to `pmacs.parse.buffer_language(buf)`. The active-buffer wrapper stays +unchanged. This preserves pathless LSP behavior while deleting the duplicate +extension/filetype/filename/shebang chain. + +The pin also closes an existing shebang inconsistency: editing `#!/bin/sh` to +`#!/usr/bin/env lua` no longer leaves a bash parse tree while making later +auto-pair/comment queries report Lua. Raw parser tests may call +`language_from_modeline`; behavior-driving consumers use the pin. + +### Q#MD7 — Initial mode, syntax, and LSP share one value + +On `buffer.after-load`, set the buffer's major mode to the freshly resolved +language, including nil when no signal resolves. This replaces the current +“only if nil” guard and makes the already-documented reload contract exact: +a fresh after-load decision replaces an earlier explicit override or clear. + +Then: + +- dispatch a grammar only if `pmacs.parse._has_language(lang)`; +- let LSP attach only if the same language has a configured server and a real + file path; +- retain a valid unknown language as the major mode, while silently skipping + grammar/LSP attachment. + +After load, explicit `set_major_mode` remains independent: it immediately +changes mode key dispatch and statusline display but does not rewrite the +pinned parser/LSP language. Switches preserve both values. + +### Q#MD8 — Malformed or unsupported metadata is fail-closed and quiet + +A malformed marker, invalid name, unsupported form, or truncated candidate +never raises from `buffer.after-load` and never emits an `*errors*` entry. The +detector returns nil and the existing inference chain continues. + +A syntactically valid unknown name is different: it is a legitimate major mode +and is pinned, but `_has_language` and LSP config gates prevent a bogus parser +or server launch. This preserves #129's custom-mode capability without +executing file content. + +A file can already select a configured language—and therefore which LSP server +pmacs starts—through its extension or a content-sniffed shebang. Modelines add +another bounded language selector within that existing capability class; they +do not introduce automatic execution beyond what current language detection +already permits. + +No enable/disable setting is added. The supported input can only select a +bounded string already consumed as passive mode/language identity; it cannot +run hooks or set options. If a future `buffer.after-mode-change` hook makes mode +selection executable, modeline trust must be revisited in that feature's +framing. + +### Q#MD9 — No Rust or protocol surface is required + +Expected implementation touch set: + +- `builtin/runtime/syntax.lua` — bounded parser, aliases, shared resolver, pin, + and after-load initialization; +- `builtin/runtime/lsp.lua` — delegate language inference while retaining the + path guard; +- `tests/m4_acceptance.rs` — parser, precedence, lifecycle, and end-to-end + regression coverage; +- this framing and the side-quest/handoff state documents when the feature + lands. + +No changes are expected in `src/buffer.rs`, Lua bindings, frontends, +`pmacs-protocol`, or protocol version 18. + +## Bets + +1. **Sixteen KiB of edge text is sufficient.** Real modelines are short; files + with an 8 KiB first/last candidate line are better treated as malformed than + copied wholesale during load. +2. **One canonical language should drive mode, parser, and LSP initially.** A + future distinction between editor mode and parser language needs a real + consumer and an explicit mapping contract, not accidental divergence. +3. **ASCII-lowercased identifiers cover interoperable modelines.** Trusted Lua + remains available for arbitrary UTF-8 custom mode names. +4. **Load-time detection is enough.** Live cookie edits would require orderly + parser teardown, LSP `didClose`/`didOpen`, overlay replacement, mode-change + notification, and failure rollback; that is not a one-shot detector. +5. **Pathless LSP buffers stay ineligible.** Syntax may still infer from a + buffer name, but spawning a server without a URI/project root remains wrong. + +## Acceptance + +All end-to-end fixtures clear `pmacs.lsp.config` unless the case intentionally +observes server selection, so opening a test file never starts a machine-local +language server. + +1. **Emacs forms:** first-line property and shorthand cookies resolve; a cookie + on line 2 resolves only after a shebang; unrelated properties are ignored; + the last `mode` property wins. +2. **Vim forms:** `vim:`/`vi:` direct and `set` forms resolve `ft` and + `filetype` in the first/last five lines; CRLF works; `Vim:` requires `set`. + The direct `vim:ft=sh:et:sw=2:` form resolves `sh`, while + `vim: set sw=4: ft=python` ignores the assignment after the terminating + colon. `ft:python` and `filetype:python` resolve nothing in either form. +3. **Boundary rejection:** sixth-line, sixth-from-end, middle-of-file, + word-embedded, unterminated, truncated, invalid-character, and overlong + candidates do not resolve and do not log errors. A partial line at the start + of the suffix byte window is discarded without consuming one of the five + complete tail-line slots. +4. **Conflict order:** overlapping edge windows are deduplicated and the last + valid assignment in document order wins deterministically. +5. **Alias behavior:** seeded aliases map `sh`/`zsh → bash`, `c++ → cpp`, + `js2 → javascript`, `tsx → typescriptreact`, and `docker → dockerfile`; a + user override wins; an invalid alias output is ignored. +6. **Explicit precedence:** a `.py` file with a Lua modeline yields `lua` from + `pmacs.parse.buffer_language`, `pmacs.lsp.active_buffer_language`, the parse + tree, and `pmacs.buffer.major_mode`. +7. **Unknown valid mode:** a `.txt` file with `mode: prose` receives major mode + `prose`, creates no parse view, starts no LSP server, and produces no error. +8. **No-modeline regression:** extension, filetype, filename, and shebang cases + retain their present precedence and outputs. +9. **Shebang pin regression:** open an extensionless `#!/bin/sh` file, replace + its shebang with `#!/usr/bin/env lua`, and assert the parse tree and + `pmacs.lsp.buffer_language` both remain `bash`. +10. **Pinned modeline lifecycle:** changing a loaded modeline does not change + the pinned language, parser, or major mode; switch-away/back remains stable; + close and reopen re-evaluates the changed on-disk cookie. +11. **Explicit override independence:** `set_major_mode` after load changes + dispatch/statusline but not `pmacs.parse.buffer_language`; switches preserve + the override. +12. **Pathless preservation:** syntax-by-buffer-name behavior remains, while + `pmacs.lsp.buffer_language` still returns nil without a backing path. +13. **Backend parity:** focused acceptance passes with default LuaJIT and + `--no-default-features --features lua54`; protocol remains v18. diff --git a/tests/m4_acceptance.rs b/tests/m4_acceptance.rs index 2a62e68..753aeec 100644 --- a/tests/m4_acceptance.rs +++ b/tests/m4_acceptance.rs @@ -6101,6 +6101,17 @@ fn m4_shebang_edit_keeps_pinned_grammar() { Some("bash"), "editing the shebang must not re-switch the pinned grammar" ); + let lsp_language: Option = s + .lua_host + .lua() + .load("return pmacs.lsp.buffer_language(pmacs.window.buffer())") + .eval() + .expect("pinned LSP language"); + assert_eq!( + lsp_language.as_deref(), + Some("bash"), + "language-aware consumers must share the shebang pin" + ); // Switch away to another buffer and back: the after-switch reattach // must reuse the pinned bash grammar rather than re-sniff the (now @@ -6137,6 +6148,322 @@ fn m4_shebang_edit_keeps_pinned_grammar() { assert_eq!(errs, 0, "reparse with the pinned grammar reports no error"); } +/// Modeline smoke: explicit file metadata overrides a misleading extension, +/// and syntax, LSP language introspection, and initial major mode agree. +#[test] +fn m4_modeline_overrides_extension_end_to_end() { + use pmacs::editor::EditorState; + let mut s = EditorState::new(); + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("misleading.py"); + std::fs::write(&file, b"-- -*- mode: lua -*-\nprint('ok')\n").expect("write"); + let file_disp = file.display(); + + s.lua_host + .lua() + .load(format!( + "pmacs.lsp.config = {{}} + pmacs.buffer.find_or_open('{file_disp}')" + )) + .exec() + .expect("open modeline fixture"); + + let (parsed, lsp, mode): (Option, Option, Option) = s + .lua_host + .lua() + .load( + "local b = pmacs.window.buffer() + return pmacs.parse.buffer_language(b), + pmacs.lsp.active_buffer_language(), + pmacs.buffer.major_mode(b)", + ) + .eval() + .expect("modeline language surfaces"); + assert_eq!(parsed.as_deref(), Some("lua")); + assert_eq!(lsp.as_deref(), Some("lua")); + assert_eq!(mode.as_deref(), Some("lua")); + + pump_async(&mut s, |st| { + current_tree_language(st).as_deref() == Some("lua") + }); +} + +#[test] +fn m4_modeline_parser_matches_supported_emacs_and_vim_forms() { + use pmacs::editor::EditorState; + let s = EditorState::new(); + let resolve = |text: &str| -> Option { + s.lua_host + .lua() + .load(format!( + "local b = pmacs.window.buffer() + if b:len() > 0 then b:delete(0, b:len()) end + b:insert(0, {text:?}) + return pmacs.parse.language_from_modeline(b)" + )) + .eval() + .expect("resolve modeline") + }; + + for (text, want) in [ + ("# -*- mode: Python; coding: utf-8 -*-\n", Some("python")), + ("-- -*- Lua -*-\n", Some("lua")), + ("#!/usr/bin/env python\n# -*- mode: Lua -*-\n", Some("lua")), + ("# -*- mode: python; mode: lua -*-\n", Some("lua")), + ("vim:ft=python:sw=4:\n", Some("python")), + ("# vim: set ft=lua sw=2:\n", Some("lua")), + ("# vi:filetype=yaml:et:\n", Some("yaml")), + ("# Vim: set filetype=toml:\n", Some("toml")), + ("one\ntwo\nthree\nfour\nfive\n# vim:ft=lua:\n", Some("lua")), + ("# vim: set ft=python:\r\n", Some("python")), + ] { + assert_eq!(resolve(text).as_deref(), want, "{text:?}"); + } + + for text in [ + "plain\n# -*- mode: lua -*-\n", // line 2 needs a shebang + "# Vim:ft=lua:\n", // uppercase marker requires `set` + "# Vim: se ft=lua:\n", // uppercase marker requires literal `set` + "# vim: set sw=4: ft=python\n", // assignment follows the terminator + "# vim: set ft=python\n", // set form needs a terminator + "# vim:ft:python:\n", // colon is an option separator + "# vim: set ft:python :\n", // colon terminates the option section + ] { + assert_eq!(resolve(text), None, "{text:?}"); + } +} + +#[test] +fn m4_modeline_parser_enforces_boundaries_aliases_and_conflicts() { + use pmacs::editor::EditorState; + let s = EditorState::new(); + let resolve = |text: &str| -> Option { + s.lua_host + .lua() + .load(format!( + "local b = pmacs.window.buffer() + if b:len() > 0 then b:delete(0, b:len()) end + b:insert(0, {text:?}) + return pmacs.parse.language_from_modeline(b)" + )) + .eval() + .expect("resolve modeline") + }; + + for (text, want) in [ + ("# vim:ft=zsh:\n", "bash"), + ("# -*- mode: C++ -*-\n", "cpp"), + ("# -*- mode: js2 -*-\n", "javascript"), + ("# vim:ft=tsx:\n", "typescriptreact"), + ("# vim:ft=docker:\n", "dockerfile"), + ] { + assert_eq!(resolve(text).as_deref(), Some(want), "{text:?}"); + } + + let conflict = "# -*- mode: python; mode: yaml -*-\n2\n3\n4\n5\n# vim:ft=lua:\n"; + assert_eq!( + resolve(conflict).as_deref(), + Some("lua"), + "last valid assignment in document order wins across overlapping edges" + ); + + let middle = "1\n2\n3\n4\n5\n# vim:ft=lua:\n7\n8\n9\n10\n11\n"; + assert_eq!( + resolve(middle), + None, + "sixth line from both edges is outside the scan" + ); + + let partial_with_live_tail = + format!("{}\n# vim:ft=lua:\n1\n2\n3\n4", "x".repeat(8 * 1024 + 64)); + assert_eq!( + resolve(&partial_with_live_tail).as_deref(), + Some("lua"), + "discarded suffix fragment does not consume a tail-line slot" + ); + + let marker_in_partial = format!("{} vim:ft=lua:\n1\n2\n3\n4\n5", "x".repeat(8 * 1024 + 64)); + assert_eq!( + resolve(&marker_in_partial), + None, + "modeline in a truncated edge line is ignored" + ); + + let overlong = format!("# vim:ft={}:\n", "a".repeat(129)); + for text in [ + "prefixvim:ft=lua:\n".to_owned(), + "# vim:ft=lua!:\n".to_owned(), + overlong, + ] { + assert_eq!(resolve(&text), None, "{text:?}"); + } + + s.lua_host + .lua() + .load("pmacs.parse.modeline_aliases.sh = 'lua'") + .exec() + .expect("override modeline alias"); + assert_eq!(resolve("# vim:ft=sh:\n").as_deref(), Some("lua")); + s.lua_host + .lua() + .load("pmacs.parse.modeline_aliases.sh = 'BAD VALUE'") + .exec() + .expect("install invalid modeline alias"); + assert_eq!(resolve("# vim:ft=sh:\n"), None); + s.lua_host + .lua() + .load("pmacs.parse.modeline_aliases.sh = 'bash'") + .exec() + .expect("restore modeline alias"); +} + +#[test] +fn m4_modeline_unknown_mode_is_quiet_and_parser_free() { + use pmacs::editor::EditorState; + let s = EditorState::new(); + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("notes.txt"); + std::fs::write(&file, b"# vim:ft=prose:\nhello\n").expect("write"); + let file_disp = file.display(); + + let (mode, language, has_view, errors): (Option, Option, bool, i64) = s + .lua_host + .lua() + .load(format!( + "pmacs.lsp.config = {{}} + _G.__modeline_errors = {{}} + local real_error = pmacs.error + pmacs.error = function(message) + table.insert(_G.__modeline_errors, message) + end + local b = pmacs.buffer.find_or_open('{file_disp}') + local mode = pmacs.buffer.major_mode(b) + local language = pmacs.parse.buffer_language(b) + local has_view = pmacs.parse._has_view(b) + local errors = #_G.__modeline_errors + pmacs.error = real_error + return mode, language, has_view, errors" + )) + .eval() + .expect("open unknown modeline mode"); + + assert_eq!(mode.as_deref(), Some("prose")); + assert_eq!(language.as_deref(), Some("prose")); + assert!(!has_view, "unknown modeline must not dispatch a parser"); + assert_eq!(errors, 0, "unknown modeline must not report an error"); +} + +#[test] +fn m4_modeline_language_is_pinned_until_reopen() { + use pmacs::editor::EditorState; + let mut s = EditorState::new(); + let dir = tempfile::tempdir().expect("tempdir"); + let file = dir.path().join("mutable.txt"); + let other = dir.path().join("other.txt"); + std::fs::write(&file, b"-- -*- mode: lua -*-\nprint('one')\n").expect("write"); + std::fs::write(&other, b"other\n").expect("write other"); + let file_disp = file.display(); + let other_disp = other.display(); + + s.lua_host + .lua() + .load(format!( + "pmacs.lsp.config = {{}} + _G.MODELINE_BUFFER = pmacs.buffer.find_or_open('{file_disp}')" + )) + .exec() + .expect("open mutable modeline fixture"); + pump_async(&mut s, |st| { + current_tree_language(st).as_deref() == Some("lua") + }); + + let (language, mode): (Option, Option) = s + .lua_host + .lua() + .load( + "local b = MODELINE_BUFFER + local text = b:slice(0, b:len()) + local start = assert(text:find('lua', 1, true)) - 1 + b:replace(start, start + 3, 'python') + pmacs.hook.run('buffer.after-edit') + return pmacs.lsp.buffer_language(b), pmacs.buffer.major_mode(b)", + ) + .eval() + .expect("edit loaded modeline"); + assert_eq!(language.as_deref(), Some("lua")); + assert_eq!(mode.as_deref(), Some("lua")); + for _ in 0..64 { + s.tick_async(); + std::thread::sleep(Duration::from_millis(2)); + } + assert_eq!(current_tree_language(&s).as_deref(), Some("lua")); + + let (language, mode): (Option, Option) = s + .lua_host + .lua() + .load(format!( + "pmacs.buffer.set_major_mode(MODELINE_BUFFER, 'markdown') + pmacs.buffer.find_or_open('{other_disp}') + pmacs.window.switch_buffer(MODELINE_BUFFER) + return pmacs.lsp.buffer_language(MODELINE_BUFFER), + pmacs.buffer.major_mode(MODELINE_BUFFER)" + )) + .eval() + .expect("switch with explicit major-mode override"); + assert_eq!(language.as_deref(), Some("lua")); + assert_eq!(mode.as_deref(), Some("markdown")); + + s.lua_host + .lua() + .load(format!("pmacs.buffer.find_or_open('{other_disp}')")) + .exec() + .expect("switch away before removing old buffer"); + std::fs::write(&file, b"# -*- mode: python -*-\nprint('two')\n").expect("rewrite"); + let (new_id, language, mode): (String, Option, Option) = s + .lua_host + .lua() + .load(format!( + "local old = MODELINE_BUFFER + pmacs.buffer.remove(old) + local reopened = pmacs.buffer.find_or_open('{file_disp}') + return tostring(reopened), pmacs.lsp.buffer_language(reopened), + pmacs.buffer.major_mode(reopened)" + )) + .eval() + .expect("reopen changed modeline fixture"); + assert_ne!( + new_id, + s.lua_host + .lua() + .load("return tostring(MODELINE_BUFFER)") + .eval::() + .unwrap(), + "reopen must allocate a fresh buffer id" + ); + assert_eq!(language.as_deref(), Some("python")); + assert_eq!(mode.as_deref(), Some("python")); + pump_async(&mut s, |st| { + current_tree_language(st).as_deref() == Some("python") + }); +} + +#[test] +fn m4_modeline_shared_resolver_preserves_pathless_lsp_guard() { + use pmacs::editor::EditorState; + let s = EditorState::new(); + let (syntax, lsp): (Option, Option) = s + .lua_host + .lua() + .load( + "local b = pmacs.buffer.create('scratch.lua') + return pmacs.parse.buffer_language(b), pmacs.lsp.buffer_language(b)", + ) + .eval() + .expect("resolve pathless language"); + assert_eq!(syntax.as_deref(), Some("lua")); + assert_eq!(lsp, None, "LSP requires a backing path"); +} + /// Filename detection: `pmacs.parse.language_from_filename` maps a /// basename (Dockerfile / Makefile / CMakeLists.txt / rc dotfiles) to a /// language, resolving a full path too, and returns nil for a plain file.