diff --git a/Cargo.lock b/Cargo.lock index ef12b8c..ba0dc78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2556,10 +2556,16 @@ dependencies = [ "tree-sitter-containerfile", "tree-sitter-cpp", "tree-sitter-cuda", + "tree-sitter-go", + "tree-sitter-javascript", "tree-sitter-lua", "tree-sitter-make", "tree-sitter-md", + "tree-sitter-python", "tree-sitter-rust", + "tree-sitter-toml-ng", + "tree-sitter-typescript", + "tree-sitter-zig", "unicode-width", ] @@ -3780,6 +3786,26 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-go" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8560a4d2f835cc0d4d2c2e03cbd0dde2f6114b43bc491164238d333e28b16ea" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-javascript" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68204f2abc0627a90bdf06e605f5c470aa26fdcb2081ea553a04bdad756693f5" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-language" version = "0.1.7" @@ -3816,6 +3842,16 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-python" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf85fd39652e740bf60f46f4cda9492c3a9ad75880575bf14960f775cb74a1c" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-rust" version = "0.24.2" @@ -3826,6 +3862,36 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-toml-ng" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9adc2c898ae49730e857d75be403da3f92bb81d8e37a2f918a08dd10de5ebb1" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-typescript" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-zig" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab11fc124851b0db4dd5e55983bbd9631192e93238389dcd44521715e5d53e28" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree_magic_mini" version = "3.2.2" diff --git a/Cargo.toml b/Cargo.toml index 52b2a77..98fec84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -181,6 +181,21 @@ tree-sitter-bash = "0.25" tree-sitter-containerfile = "0.9" tree-sitter-make = "1.1" tree-sitter-cmake = "0.7" +# Grammar-gap languages: these already have LSP configs (basedpyright / +# gopls / tsserver / taplo / zls) but shipped no tree-sitter grammar, so +# they rendered with no lexical color. Fill the gap. All ride +# `tree-sitter-language 0.1` (tree-sitter dev-only) and export +# `LANGUAGE`/`HIGHLIGHTS_QUERY`. TypeScript is special: one crate ships +# TWO grammars (`LANGUAGE_TYPESCRIPT`/`LANGUAGE_TSX`) and its highlights +# inherit JavaScript, so the `typescript*` entries compose the JS query +# ahead of the TS one (see `crate::syntax::BUILTIN_LANGUAGES`). JavaScript +# parses both `.js` and `.jsx`. +tree-sitter-python = "0.25" +tree-sitter-go = "0.25" +tree-sitter-javascript = "0.25" +tree-sitter-typescript = "0.23" +tree-sitter-toml-ng = "0.7" +tree-sitter-zig = "1.1" # T M9.7: markdown grammar so prompt result buffers with # `_meta.format = "markdown"` get structured highlighting through the # same M4 path as rust/lua — no special-case painter in Lua. diff --git a/builtin/runtime/lsp.lua b/builtin/runtime/lsp.lua index befc3ad..eb4f1e4 100644 --- a/builtin/runtime/lsp.lua +++ b/builtin/runtime/lsp.lua @@ -201,11 +201,13 @@ pmacs.lsp.config.zig = pmacs.lsp.config.zig or { } -- LSP-side extension → language map, deliberately independent of the --- tree-sitter detection in `pmacs.parse` (which is grammar-gated: --- Python has an LSP server but no bundled grammar). Consulted only --- when `pmacs.parse.language_for_path` finds nothing, so grammar- --- backed languages keep their existing detection. Extensible from --- init.lua: `pmacs.lsp.filetypes.foo = "bar"`. +-- tree-sitter detection in `pmacs.parse`. Consulted only when +-- `pmacs.parse.language_for_path` finds nothing (an extension with a +-- server but no bundled grammar), so grammar-backed languages keep their +-- existing detection. Every language with an LSP config now also ships a +-- grammar, so this is mainly the LSP-only fallback that keeps a language +-- id stable if a grammar is ever dropped, plus the seam for user-added +-- mappings. Extensible from init.lua: `pmacs.lsp.filetypes.foo = "bar"`. pmacs.lsp.filetypes = pmacs.lsp.filetypes or {} pmacs.lsp.filetypes.py = pmacs.lsp.filetypes.py or "python" pmacs.lsp.filetypes.pyi = pmacs.lsp.filetypes.pyi or "python" diff --git a/builtin/runtime/syntax.lua b/builtin/runtime/syntax.lua index f988d24..b639377 100644 --- a/builtin/runtime/syntax.lua +++ b/builtin/runtime/syntax.lua @@ -220,9 +220,11 @@ local function attach_for_active_buffer() -- and silently swap the grammar — and diverge from the LSP side, which -- keeps its existing attachment across the switch. A first-seen buffer -- (no view yet) resolves normally. Gate dispatch on `_has_language`: the - -- resolution chain also yields languages with no grammar (python, - -- javascript), and dispatching one would raise "unknown language" - -- (caught, but noise) and never gives a wrong-grammar tree. + -- resolution chain can still yield a language with no grammar — a + -- shebang or filetype mapping to a server-only or unsupported language + -- (e.g. an init.lua `pmacs.parse.shebangs.ruby = "ruby"`) — and + -- dispatching one would raise "unknown language" (caught, but noise) and + -- never gives a wrong-grammar tree. local lang = pmacs.parse._has_view(buf) and parse_lang_by_buffer[key] or resolve_active_language(buf) if not lang or not pmacs.parse._has_language(lang) then return end diff --git a/src/syntax.rs b/src/syntax.rs index c33decb..33577e0 100644 --- a/src/syntax.rs +++ b/src/syntax.rs @@ -487,6 +487,78 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ loader: || tree_sitter_cmake::LANGUAGE.into(), highlights_query: &[tree_sitter_cmake::HIGHLIGHTS_QUERY], }, + // Grammar-gap languages — these already had LSP configs but no + // grammar, so they rendered without lexical color. Each language name + // matches its existing `pmacs.lsp.config.` key, so grammar + // detection (which wins over the filetype map) resolves the same id + // the server keys off. Root kinds: python `module`, go/zig + // `source_file`, js/ts family `program`, toml `document`. + LanguageEntry { + name: "python", + extensions: &["py", "pyi"], + loader: || tree_sitter_python::LANGUAGE.into(), + highlights_query: &[tree_sitter_python::HIGHLIGHTS_QUERY], + }, + LanguageEntry { + name: "go", + extensions: &["go"], + loader: || tree_sitter_go::LANGUAGE.into(), + highlights_query: &[tree_sitter_go::HIGHLIGHTS_QUERY], + }, + // JavaScript / TypeScript. One `tree-sitter-javascript` grammar parses + // both `.js` and `.jsx`; `tree-sitter-typescript` ships two grammars + // (`LANGUAGE_TYPESCRIPT`, `LANGUAGE_TSX`). Highlights inherit: the TS + // query is a ~5-capture delta over JavaScript, and JSX is a further + // `JSX_HIGHLIGHT_QUERY` delta — so the `*react` and `typescript*` + // entries compose base-first (js → jsx → ts), the same pattern as + // `cuda` over C/C++. The four names mirror the LSP filetype map + // (typescriptreact/javascriptreact) so tsserver enables the JSX parser. + LanguageEntry { + name: "javascript", + extensions: &["js", "mjs", "cjs"], + loader: || tree_sitter_javascript::LANGUAGE.into(), + highlights_query: &[tree_sitter_javascript::HIGHLIGHT_QUERY], + }, + LanguageEntry { + name: "javascriptreact", + extensions: &["jsx"], + loader: || tree_sitter_javascript::LANGUAGE.into(), + highlights_query: &[ + tree_sitter_javascript::HIGHLIGHT_QUERY, + tree_sitter_javascript::JSX_HIGHLIGHT_QUERY, + ], + }, + LanguageEntry { + name: "typescript", + extensions: &["ts", "mts", "cts"], + loader: || tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(), + highlights_query: &[ + tree_sitter_javascript::HIGHLIGHT_QUERY, + tree_sitter_typescript::HIGHLIGHTS_QUERY, + ], + }, + LanguageEntry { + name: "typescriptreact", + extensions: &["tsx"], + loader: || tree_sitter_typescript::LANGUAGE_TSX.into(), + highlights_query: &[ + tree_sitter_javascript::HIGHLIGHT_QUERY, + tree_sitter_javascript::JSX_HIGHLIGHT_QUERY, + tree_sitter_typescript::HIGHLIGHTS_QUERY, + ], + }, + LanguageEntry { + name: "toml", + extensions: &["toml"], + loader: || tree_sitter_toml_ng::LANGUAGE.into(), + highlights_query: &[tree_sitter_toml_ng::HIGHLIGHTS_QUERY], + }, + LanguageEntry { + name: "zig", + extensions: &["zig", "zon"], + loader: || tree_sitter_zig::LANGUAGE.into(), + highlights_query: &[tree_sitter_zig::HIGHLIGHTS_QUERY], + }, ]; /// Registry that the Lua surface ([`crate::lua_bindings::install_parse`]) @@ -814,6 +886,24 @@ pub fn compute_highlight_spans_in_range( let root = bundle.tree.root_node(); let mut iter = cursor.captures(query, root, source); while let Some((qmatch, capture_idx)) = iter.next() { + // Fail-closed on the locals property predicate. The capture + // iterator already applies text predicates (`#eq?`/`#match?`/ + // `#any-of?`), but `#is? local` / `#is-not? local` are *property* + // predicates (`Query::property_predicates`) that need a scope map + // built from the grammar's LOCALS_QUERY, which pmacs does not run. + // Applying such a capture regardless mis-styles shadowed locals — + // e.g. a local `console`/`require` in JS/TS would still capture as + // `@variable.builtin`/`@function.builtin`. Until locals processing + // exists, drop captures whose pattern carries one; the identifier + // falls back to its non-builtin capture. `#set!` (property + // *settings*) is a different API and is not consulted here. + if query + .property_predicates(qmatch.pattern_index) + .iter() + .any(|(prop, _)| &*prop.key == "local") + { + continue; + } let cap = qmatch.captures[*capture_idx]; spans.push(HighlightSpan { start_byte: cap.node.start_byte() as u32, @@ -1171,6 +1261,172 @@ mod tests { } } + #[test] + fn builtin_languages_include_gap_grammars() { + for (name, exts) in [ + ("python", &["py", "pyi"][..]), + ("go", &["go"][..]), + ("javascript", &["js", "mjs", "cjs"][..]), + ("javascriptreact", &["jsx"][..]), + ("typescript", &["ts", "mts", "cts"][..]), + ("typescriptreact", &["tsx"][..]), + ("toml", &["toml"][..]), + ("zig", &["zig", "zon"][..]), + ] { + let entry = BUILTIN_LANGUAGES + .iter() + .find(|l| l.name == name) + .unwrap_or_else(|| panic!("`{name}` language entry must be present")); + for ext in exts { + assert!(entry.extensions.contains(ext), "`{name}` claims `.{ext}`"); + } + assert!( + entry.highlights_query.iter().any(|q| !q.is_empty()), + "`{name}` ships a highlights query" + ); + } + } + + #[test] + fn gap_grammars_load_and_parse() { + // ABI acceptance for each new grammar (set_language succeeds at + // runtime) + a snippet that parses without error at the expected + // root. Covers both `tree-sitter-typescript` grammars. + let reg = SyntaxRegistry::new(); + let cases: &[(&str, &str, &[u8])] = &[ + ("python", "module", b"def f(x):\n return x + 1\n"), + ("go", "source_file", b"package main\nfunc main() {}\n"), + ("javascript", "program", b"const x = 1;\nlet y = [x];\n"), + ( + "javascriptreact", + "program", + b"const e =
;\n", + ), + ("typescript", "program", b"const x: number = 1;\n"), + ("typescriptreact", "program", b"const e =
;\n"), + ("toml", "document", b"[pkg]\nname = \"x\"\n"), + ("zig", "source_file", b"const std = @import(\"std\");\n"), + ]; + for (lang, root_kind, src) in cases { + let language = reg + .language(lang) + .unwrap_or_else(|| panic!("`{lang}` loads from BUILTIN_LANGUAGES")); + let mut buf = fresh_buffer(&format!("probe_{lang}")); + buf.apply_edit(EditOp::Insert { pos: 0, bytes: src }) + .unwrap(); + let view = ParseView::new(&buf, language, (*lang).to_owned()); + let handle = view.handle(); + let _vid = buf.attach_view(Box::new(view)); + let bundle = parse_synchronously(&handle); + assert_eq!( + bundle.tree.root_node().kind(), + *root_kind, + "`{lang}` roots at `{root_kind}`" + ); + assert!( + !bundle.tree.root_node().has_error(), + "`{lang}` parses its snippet without error" + ); + } + } + + #[test] + fn typescript_highlights_compose_the_javascript_base() { + // The bundled TypeScript highlights are a ~5-capture delta over + // JavaScript; the entries prepend the JS query (and JSX for tsx). + // Assert the COMPILED query resolves far more than the delta — the + // JS base is really there, not just the ts-specific captures. + let reg = SyntaxRegistry::new(); + for lang in ["typescript", "typescriptreact"] { + let query = reg + .highlights_query(lang) + .unwrap_or_else(|| panic!("`{lang}` highlights compile")); + assert!( + query.capture_names().len() >= 15, + "`{lang}` composes the JavaScript base (got {} captures, delta alone is ~5)", + query.capture_names().len() + ); + } + } + + #[test] + fn javascript_shadowed_builtin_is_not_mislabeled() { + // `#is-not? local` (JS/TS use it for console/require/etc.) needs a + // scope map from the LOCALS_QUERY we don't run, so + // `compute_highlight_spans` drops captures guarded by it. Here + // `console` is a LOCAL declaration — it must not surface as a + // `*.builtin` capture (which is what a naive run of the shared JS + // query would produce). + let reg = SyntaxRegistry::new(); + let language = reg.language("javascript").expect("javascript loads"); + let query = reg + .highlights_query("javascript") + .expect("javascript highlights compile"); + let mut buf = fresh_buffer("shadow.js"); + buf.apply_edit(EditOp::Insert { + pos: 0, + bytes: b"const console = 5;\nconsole;\n", + }) + .unwrap(); + let view = ParseView::new(&buf, language, "javascript".to_owned()); + let handle = view.handle(); + let _vid = buf.attach_view(Box::new(view)); + let bundle = parse_synchronously(&handle); + let spans = compute_highlight_spans(&query, &bundle); + assert!(!spans.is_empty(), "the JS query produced highlight spans"); + let names = query.capture_names(); + let builtin: Vec<&str> = spans + .iter() + .map(|s| names[s.capture_index as usize]) + .filter(|n| n.contains("builtin")) + .collect(); + assert!( + builtin.is_empty(), + "a locally-shadowed `console` must not get a *.builtin capture; got {builtin:?}" + ); + // ...and dropping the builtin pattern must not strip *all* styling: + // each `console` occurrence still keeps its ordinary `@variable` + // capture (the fallback), so the loss is only the `.builtin` refine. + let src = "const console = 5;\nconsole;\n"; + for (pos, _) in src.match_indices("console") { + let (start, end) = (pos as u32, (pos + "console".len()) as u32); + let caps: Vec<&str> = spans + .iter() + .filter(|s| s.start_byte == start && s.end_byte == end) + .map(|s| names[s.capture_index as usize]) + .collect(); + assert!( + caps.iter().any(|n| n.starts_with("variable")), + "`console` at byte {pos} keeps a variable capture; got {caps:?}" + ); + } + } + + #[test] + fn gap_grammar_extensions_resolve() { + let reg = SyntaxRegistry::new(); + for (path, lang) in [ + ("main.py", "python"), + ("stub.pyi", "python"), + ("server.go", "go"), + ("app.js", "javascript"), + ("mod.mjs", "javascript"), + ("view.jsx", "javascriptreact"), + ("index.ts", "typescript"), + ("types.mts", "typescript"), + ("App.tsx", "typescriptreact"), + ("Cargo.toml", "toml"), + ("build.zig", "zig"), + ("config.zon", "zig"), + ] { + assert_eq!( + reg.language_name_for_path(path).as_deref(), + Some(lang), + "{path} resolves to {lang}" + ); + } + } + #[test] fn byte_to_point_handles_first_line() { let src = b"hello world"; diff --git a/tests/m4_acceptance.rs b/tests/m4_acceptance.rs index 3cc0462..f922253 100644 --- a/tests/m4_acceptance.rs +++ b/tests/m4_acceptance.rs @@ -5959,7 +5959,7 @@ fn m4_shebang_extensionless_script_resolves_bash() { #[test] fn m4_shebang_does_not_override_extension() { use pmacs::editor::EditorState; - let s = EditorState::new(); + let mut s = EditorState::new(); let dir = tempfile::tempdir().expect("tempdir"); let f = dir.path().join("tool.py"); std::fs::write(&f, b"#!/bin/sh\nprint('hi')\n").expect("write"); @@ -5972,56 +5972,58 @@ fn m4_shebang_does_not_override_extension() { )) .exec() .expect("open .py with a shell shebang"); - // Both the LSP language *and* the grammar decision must respect the - // extension: python for LSP, and NO grammar parse view (python has no - // grammar) — not a bash tree installed from the `#!/bin/sh` line. - // `_has_view` is set synchronously by `_dispatch`, so no pump is - // needed; without the precedence fix the shebang would have dispatched - // bash and this would be true. - let (lang, has_view): (Option, bool) = s + // Both the LSP language *and* the grammar must respect the extension: + // python, not bash from the `#!/bin/sh` line. (Python now has a + // grammar, so the check is "the tree is python", not "no tree at all".) + let lang: Option = s .lua_host .lua() - .load( - "return pmacs.lsp.active_buffer_language(), - pmacs.parse._has_view(pmacs.window.buffer())", - ) + .load("return pmacs.lsp.active_buffer_language()") .eval() - .expect("language + view"); + .expect("language"); assert_eq!( lang.as_deref(), Some("python"), ".py extension wins over a #!/bin/sh shebang (LSP)" ); - assert!( - !has_view, - ".py file must not get a grammar parse view from a #!/bin/sh line" + pump_async(&mut s, |st| current_tree_language(st).is_some()); + assert_eq!( + current_tree_language(&s).as_deref(), + Some("python"), + ".py gets a python grammar tree, not bash from the shebang" ); } -/// Finding-1 gate: an extensionless `#!/usr/bin/env python3` script -/// resolves to python for LSP, but python has no grammar — syntax must +/// Finding-1 gate: an extensionless `#!/usr/bin/env ruby` script resolves +/// to `ruby` via the shebang map, but ruby has no grammar — syntax must /// skip it *silently*. Without the `_has_language` gate, `_dispatch` -/// raises "unknown language: python" (caught by the after-load pcall and +/// raises "unknown language: ruby" (caught by the after-load pcall and /// reported through `pmacs.error`), which we assert does NOT happen. +/// (python/js/lua/bash all ship grammars now, so the gate needs a +/// genuinely grammarless example.) #[test] fn m4_shebang_extensionless_grammarless_language_is_silent() { use pmacs::editor::EditorState; let s = EditorState::new(); let dir = tempfile::tempdir().expect("tempdir"); let f = dir.path().join("generate"); // no extension - std::fs::write(&f, b"#!/usr/bin/env python3\nprint('hi')\n").expect("write"); + // `ruby` is deliberately grammarless (and serverless) — a language the + // shebang resolves but pmacs cannot parse. (python/js/lua/bash all have + // grammars now, so the gate needs a genuinely grammarless example.) + std::fs::write(&f, b"#!/usr/bin/env ruby\nputs 'hi'\n").expect("write"); let f_disp = f.display(); s.lua_host .lua() .load(format!( "pmacs.lsp.config = {{}} + pmacs.parse.shebangs.ruby = 'ruby' _G.__errs = {{}} local real = pmacs.error pmacs.error = function(m) table.insert(_G.__errs, m) end pmacs.buffer.find_or_open('{f_disp}')" )) .exec() - .expect("open extensionless python script"); + .expect("open extensionless ruby script"); let (lang, has_view, errs): (Option, bool, i64) = s .lua_host .lua() @@ -6032,7 +6034,11 @@ fn m4_shebang_extensionless_grammarless_language_is_silent() { ) .eval() .expect("probe"); - assert_eq!(lang.as_deref(), Some("python"), "python resolves for LSP"); + assert_eq!( + lang.as_deref(), + Some("ruby"), + "ruby resolves via the shebang" + ); assert!( !has_view, "no grammar parse view for a grammarless language" @@ -6237,6 +6243,46 @@ fn m4_filename_extensionless_dockerfile_highlights() { ); } +/// Grammar-gap languages: each bundled grammar's name matches the +/// existing `pmacs.lsp.config.` key, so grammar detection (which +/// wins over the filetype map) resolves the id the server keys off — the +/// file now gets BOTH highlighting and the right server. Verified through +/// the loaded runtime. +#[test] +fn m4_gap_grammars_align_with_lsp_configs() { + use pmacs::editor::EditorState; + let s = EditorState::new(); + for (path, id) in [ + ("app.py", "python"), + ("srv.go", "go"), + ("m.js", "javascript"), + ("v.jsx", "javascriptreact"), + ("i.ts", "typescript"), + ("A.tsx", "typescriptreact"), + ("Cargo.toml", "toml"), + ("build.zig", "zig"), + ] { + let (grammar, has_cfg): (Option, bool) = s + .lua_host + .lua() + .load(format!( + "return pmacs.parse.language_for_path('{path}'), + pmacs.lsp.config['{id}'] ~= nil" + )) + .eval() + .unwrap_or_else(|e| panic!("probe {path}: {e}")); + assert_eq!( + grammar.as_deref(), + Some(id), + "{path} grammar detection resolves to {id}" + ); + assert!( + has_cfg, + "{id} has an LSP config the grammar name aligns with" + ); + } +} + /// Typing-perf: the default bundle coalesces full-document /// `didChange` notifications instead of sending one per keystroke /// (each send copies the whole buffer several times and writes