From bcec61e020ba3a44ca7b7f8882c046ba22721747 Mon Sep 17 00:00:00 2001 From: Levi Neuwirth Date: Tue, 14 Jul 2026 17:29:54 +0100 Subject: [PATCH] feat(highlight): grammars for python/go/typescript/javascript/toml/zig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These five languages already had LSP configs (basedpyright, gopls, tsserver, taplo, zls) but shipped no tree-sitter grammar, so they rendered with no lexical color. Fill the gap — 8 BUILTIN_LANGUAGES entries across 6 crates: - python (`tree-sitter-python`, root `module`), go (`tree-sitter-go`), toml (`tree-sitter-toml-ng`), zig (`tree-sitter-zig`, +`.zon`) — each a single self-contained highlights query. - JavaScript/TypeScript family: `tree-sitter-javascript` parses both `.js` and `.jsx`; `tree-sitter-typescript` ships two grammars (LANGUAGE_TYPESCRIPT, LANGUAGE_TSX). The four entries — javascript, javascriptreact, typescript, typescriptreact — mirror the LSP filetype map so tsserver enables the JSX parser. Highlights inherit: the TS query is a ~5-capture delta over JavaScript and JSX is a further delta, so the entries compose base-first (js → jsx → ts), the same pattern as `cuda` over C/C++ (typescript resolves ~22 capture classes, typescriptreact ~24). Each grammar's name equals its existing `pmacs.lsp.config.` key, so grammar detection (which wins over the filetype map) resolves the id the server keys off — the file now gets BOTH highlighting and the right server. No lsp.lua change needed. All crates ride `tree-sitter-language 0.1` with tree-sitter dev-only — no second core in the graph. Bite-verified acceptance: - gap_grammars_load_and_parse — each grammar's ABI accepted by the 0.26 core; a snippet parses without error at its root (covers both TS grammars, incl. JSX). - typescript_highlights_compose_the_javascript_base — the compiled typescript/typescriptreact queries resolve >= 15 captures, not just the ~5-capture TS delta (the JS base is really composed in). - builtin_languages_include_gap_grammars / gap_grammar_extensions_resolve — entry presence + extension detection across all 8 ids. - m4_gap_grammars_align_with_lsp_configs — through the loaded runtime, each path's grammar id matches an existing LSP config. Bite-verified against pre-feature src/syntax.rs. Ripple: two #116 shebang tests used python as their "has-LSP-but-no- grammar" example, which this PR invalidates. Updated both — the .py + `#!/bin/sh` precedence test now asserts a python grammar tree (not "no tree"), and the grammarless-language-is-silent gate test switches to `ruby` (genuinely grammarless) via a test-local shebang mapping. Gates: fmt; clippy -D warnings; test --lib; --features crdt; m4_acceptance --skip basedpyright; GPU; full workspace sweep; git diff --check. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01YJ9FQ832QwftJXCD9LeFan --- Cargo.lock | 66 +++++++++++++++ Cargo.toml | 15 ++++ src/syntax.rs | 185 +++++++++++++++++++++++++++++++++++++++++ tests/m4_acceptance.rs | 82 +++++++++++++----- 4 files changed, 329 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef12b8c..ba0dc78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2556,10 +2556,16 @@ dependencies = [ "tree-sitter-containerfile", "tree-sitter-cpp", "tree-sitter-cuda", + "tree-sitter-go", + "tree-sitter-javascript", "tree-sitter-lua", "tree-sitter-make", "tree-sitter-md", + "tree-sitter-python", "tree-sitter-rust", + "tree-sitter-toml-ng", + "tree-sitter-typescript", + "tree-sitter-zig", "unicode-width", ] @@ -3780,6 +3786,26 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-go" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8560a4d2f835cc0d4d2c2e03cbd0dde2f6114b43bc491164238d333e28b16ea" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-javascript" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68204f2abc0627a90bdf06e605f5c470aa26fdcb2081ea553a04bdad756693f5" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-language" version = "0.1.7" @@ -3816,6 +3842,16 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-python" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf85fd39652e740bf60f46f4cda9492c3a9ad75880575bf14960f775cb74a1c" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-rust" version = "0.24.2" @@ -3826,6 +3862,36 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-toml-ng" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9adc2c898ae49730e857d75be403da3f92bb81d8e37a2f918a08dd10de5ebb1" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-typescript" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-zig" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab11fc124851b0db4dd5e55983bbd9631192e93238389dcd44521715e5d53e28" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree_magic_mini" version = "3.2.2" diff --git a/Cargo.toml b/Cargo.toml index 52b2a77..98fec84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -181,6 +181,21 @@ tree-sitter-bash = "0.25" tree-sitter-containerfile = "0.9" tree-sitter-make = "1.1" tree-sitter-cmake = "0.7" +# Grammar-gap languages: these already have LSP configs (basedpyright / +# gopls / tsserver / taplo / zls) but shipped no tree-sitter grammar, so +# they rendered with no lexical color. Fill the gap. All ride +# `tree-sitter-language 0.1` (tree-sitter dev-only) and export +# `LANGUAGE`/`HIGHLIGHTS_QUERY`. TypeScript is special: one crate ships +# TWO grammars (`LANGUAGE_TYPESCRIPT`/`LANGUAGE_TSX`) and its highlights +# inherit JavaScript, so the `typescript*` entries compose the JS query +# ahead of the TS one (see `crate::syntax::BUILTIN_LANGUAGES`). JavaScript +# parses both `.js` and `.jsx`. +tree-sitter-python = "0.25" +tree-sitter-go = "0.25" +tree-sitter-javascript = "0.25" +tree-sitter-typescript = "0.23" +tree-sitter-toml-ng = "0.7" +tree-sitter-zig = "1.1" # T M9.7: markdown grammar so prompt result buffers with # `_meta.format = "markdown"` get structured highlighting through the # same M4 path as rust/lua — no special-case painter in Lua. diff --git a/src/syntax.rs b/src/syntax.rs index c33decb..4b0b8bf 100644 --- a/src/syntax.rs +++ b/src/syntax.rs @@ -487,6 +487,78 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ loader: || tree_sitter_cmake::LANGUAGE.into(), highlights_query: &[tree_sitter_cmake::HIGHLIGHTS_QUERY], }, + // Grammar-gap languages — these already had LSP configs but no + // grammar, so they rendered without lexical color. Each language name + // matches its existing `pmacs.lsp.config.` key, so grammar + // detection (which wins over the filetype map) resolves the same id + // the server keys off. Root kinds: python `module`, go/zig + // `source_file`, js/ts family `program`, toml `document`. + LanguageEntry { + name: "python", + extensions: &["py", "pyi"], + loader: || tree_sitter_python::LANGUAGE.into(), + highlights_query: &[tree_sitter_python::HIGHLIGHTS_QUERY], + }, + LanguageEntry { + name: "go", + extensions: &["go"], + loader: || tree_sitter_go::LANGUAGE.into(), + highlights_query: &[tree_sitter_go::HIGHLIGHTS_QUERY], + }, + // JavaScript / TypeScript. One `tree-sitter-javascript` grammar parses + // both `.js` and `.jsx`; `tree-sitter-typescript` ships two grammars + // (`LANGUAGE_TYPESCRIPT`, `LANGUAGE_TSX`). Highlights inherit: the TS + // query is a ~5-capture delta over JavaScript, and JSX is a further + // `JSX_HIGHLIGHT_QUERY` delta — so the `*react` and `typescript*` + // entries compose base-first (js → jsx → ts), the same pattern as + // `cuda` over C/C++. The four names mirror the LSP filetype map + // (typescriptreact/javascriptreact) so tsserver enables the JSX parser. + LanguageEntry { + name: "javascript", + extensions: &["js", "mjs", "cjs"], + loader: || tree_sitter_javascript::LANGUAGE.into(), + highlights_query: &[tree_sitter_javascript::HIGHLIGHT_QUERY], + }, + LanguageEntry { + name: "javascriptreact", + extensions: &["jsx"], + loader: || tree_sitter_javascript::LANGUAGE.into(), + highlights_query: &[ + tree_sitter_javascript::HIGHLIGHT_QUERY, + tree_sitter_javascript::JSX_HIGHLIGHT_QUERY, + ], + }, + LanguageEntry { + name: "typescript", + extensions: &["ts", "mts", "cts"], + loader: || tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(), + highlights_query: &[ + tree_sitter_javascript::HIGHLIGHT_QUERY, + tree_sitter_typescript::HIGHLIGHTS_QUERY, + ], + }, + LanguageEntry { + name: "typescriptreact", + extensions: &["tsx"], + loader: || tree_sitter_typescript::LANGUAGE_TSX.into(), + highlights_query: &[ + tree_sitter_javascript::HIGHLIGHT_QUERY, + tree_sitter_javascript::JSX_HIGHLIGHT_QUERY, + tree_sitter_typescript::HIGHLIGHTS_QUERY, + ], + }, + LanguageEntry { + name: "toml", + extensions: &["toml"], + loader: || tree_sitter_toml_ng::LANGUAGE.into(), + highlights_query: &[tree_sitter_toml_ng::HIGHLIGHTS_QUERY], + }, + LanguageEntry { + name: "zig", + extensions: &["zig", "zon"], + loader: || tree_sitter_zig::LANGUAGE.into(), + highlights_query: &[tree_sitter_zig::HIGHLIGHTS_QUERY], + }, ]; /// Registry that the Lua surface ([`crate::lua_bindings::install_parse`]) @@ -1171,6 +1243,119 @@ mod tests { } } + #[test] + fn builtin_languages_include_gap_grammars() { + for (name, exts) in [ + ("python", &["py", "pyi"][..]), + ("go", &["go"][..]), + ("javascript", &["js", "mjs", "cjs"][..]), + ("javascriptreact", &["jsx"][..]), + ("typescript", &["ts", "mts", "cts"][..]), + ("typescriptreact", &["tsx"][..]), + ("toml", &["toml"][..]), + ("zig", &["zig", "zon"][..]), + ] { + let entry = BUILTIN_LANGUAGES + .iter() + .find(|l| l.name == name) + .unwrap_or_else(|| panic!("`{name}` language entry must be present")); + for ext in exts { + assert!(entry.extensions.contains(ext), "`{name}` claims `.{ext}`"); + } + assert!( + entry.highlights_query.iter().any(|q| !q.is_empty()), + "`{name}` ships a highlights query" + ); + } + } + + #[test] + fn gap_grammars_load_and_parse() { + // ABI acceptance for each new grammar (set_language succeeds at + // runtime) + a snippet that parses without error at the expected + // root. Covers both `tree-sitter-typescript` grammars. + let reg = SyntaxRegistry::new(); + let cases: &[(&str, &str, &[u8])] = &[ + ("python", "module", b"def f(x):\n return x + 1\n"), + ("go", "source_file", b"package main\nfunc main() {}\n"), + ("javascript", "program", b"const x = 1;\nlet y = [x];\n"), + ( + "javascriptreact", + "program", + b"const e =
;\n", + ), + ("typescript", "program", b"const x: number = 1;\n"), + ("typescriptreact", "program", b"const e =
;\n"), + ("toml", "document", b"[pkg]\nname = \"x\"\n"), + ("zig", "source_file", b"const std = @import(\"std\");\n"), + ]; + for (lang, root_kind, src) in cases { + let language = reg + .language(lang) + .unwrap_or_else(|| panic!("`{lang}` loads from BUILTIN_LANGUAGES")); + let mut buf = fresh_buffer(&format!("probe_{lang}")); + buf.apply_edit(EditOp::Insert { pos: 0, bytes: src }) + .unwrap(); + let view = ParseView::new(&buf, language, (*lang).to_owned()); + let handle = view.handle(); + let _vid = buf.attach_view(Box::new(view)); + let bundle = parse_synchronously(&handle); + assert_eq!( + bundle.tree.root_node().kind(), + *root_kind, + "`{lang}` roots at `{root_kind}`" + ); + assert!( + !bundle.tree.root_node().has_error(), + "`{lang}` parses its snippet without error" + ); + } + } + + #[test] + fn typescript_highlights_compose_the_javascript_base() { + // The bundled TypeScript highlights are a ~5-capture delta over + // JavaScript; the entries prepend the JS query (and JSX for tsx). + // Assert the COMPILED query resolves far more than the delta — the + // JS base is really there, not just the ts-specific captures. + let reg = SyntaxRegistry::new(); + for lang in ["typescript", "typescriptreact"] { + let query = reg + .highlights_query(lang) + .unwrap_or_else(|| panic!("`{lang}` highlights compile")); + assert!( + query.capture_names().len() >= 15, + "`{lang}` composes the JavaScript base (got {} captures, delta alone is ~5)", + query.capture_names().len() + ); + } + } + + #[test] + fn gap_grammar_extensions_resolve() { + let reg = SyntaxRegistry::new(); + for (path, lang) in [ + ("main.py", "python"), + ("stub.pyi", "python"), + ("server.go", "go"), + ("app.js", "javascript"), + ("mod.mjs", "javascript"), + ("view.jsx", "javascriptreact"), + ("index.ts", "typescript"), + ("types.mts", "typescript"), + ("App.tsx", "typescriptreact"), + ("Cargo.toml", "toml"), + ("build.zig", "zig"), + ("config.zon", "zig"), + ] { + assert_eq!( + reg.language_name_for_path(path).as_deref(), + Some(lang), + "{path} resolves to {lang}" + ); + } + } + #[test] fn byte_to_point_handles_first_line() { let src = b"hello world"; diff --git a/tests/m4_acceptance.rs b/tests/m4_acceptance.rs index 3cc0462..90d41c0 100644 --- a/tests/m4_acceptance.rs +++ b/tests/m4_acceptance.rs @@ -5959,7 +5959,7 @@ fn m4_shebang_extensionless_script_resolves_bash() { #[test] fn m4_shebang_does_not_override_extension() { use pmacs::editor::EditorState; - let s = EditorState::new(); + let mut s = EditorState::new(); let dir = tempfile::tempdir().expect("tempdir"); let f = dir.path().join("tool.py"); std::fs::write(&f, b"#!/bin/sh\nprint('hi')\n").expect("write"); @@ -5972,29 +5972,25 @@ fn m4_shebang_does_not_override_extension() { )) .exec() .expect("open .py with a shell shebang"); - // Both the LSP language *and* the grammar decision must respect the - // extension: python for LSP, and NO grammar parse view (python has no - // grammar) — not a bash tree installed from the `#!/bin/sh` line. - // `_has_view` is set synchronously by `_dispatch`, so no pump is - // needed; without the precedence fix the shebang would have dispatched - // bash and this would be true. - let (lang, has_view): (Option, bool) = s + // Both the LSP language *and* the grammar must respect the extension: + // python, not bash from the `#!/bin/sh` line. (Python now has a + // grammar, so the check is "the tree is python", not "no tree at all".) + let lang: Option = s .lua_host .lua() - .load( - "return pmacs.lsp.active_buffer_language(), - pmacs.parse._has_view(pmacs.window.buffer())", - ) + .load("return pmacs.lsp.active_buffer_language()") .eval() - .expect("language + view"); + .expect("language"); assert_eq!( lang.as_deref(), Some("python"), ".py extension wins over a #!/bin/sh shebang (LSP)" ); - assert!( - !has_view, - ".py file must not get a grammar parse view from a #!/bin/sh line" + pump_async(&mut s, |st| current_tree_language(st).is_some()); + assert_eq!( + current_tree_language(&s).as_deref(), + Some("python"), + ".py gets a python grammar tree, not bash from the shebang" ); } @@ -6009,19 +6005,23 @@ fn m4_shebang_extensionless_grammarless_language_is_silent() { let s = EditorState::new(); let dir = tempfile::tempdir().expect("tempdir"); let f = dir.path().join("generate"); // no extension - std::fs::write(&f, b"#!/usr/bin/env python3\nprint('hi')\n").expect("write"); + // `ruby` is deliberately grammarless (and serverless) — a language the + // shebang resolves but pmacs cannot parse. (python/js/lua/bash all have + // grammars now, so the gate needs a genuinely grammarless example.) + std::fs::write(&f, b"#!/usr/bin/env ruby\nputs 'hi'\n").expect("write"); let f_disp = f.display(); s.lua_host .lua() .load(format!( "pmacs.lsp.config = {{}} + pmacs.parse.shebangs.ruby = 'ruby' _G.__errs = {{}} local real = pmacs.error pmacs.error = function(m) table.insert(_G.__errs, m) end pmacs.buffer.find_or_open('{f_disp}')" )) .exec() - .expect("open extensionless python script"); + .expect("open extensionless ruby script"); let (lang, has_view, errs): (Option, bool, i64) = s .lua_host .lua() @@ -6032,7 +6032,11 @@ fn m4_shebang_extensionless_grammarless_language_is_silent() { ) .eval() .expect("probe"); - assert_eq!(lang.as_deref(), Some("python"), "python resolves for LSP"); + assert_eq!( + lang.as_deref(), + Some("ruby"), + "ruby resolves via the shebang" + ); assert!( !has_view, "no grammar parse view for a grammarless language" @@ -6237,6 +6241,46 @@ fn m4_filename_extensionless_dockerfile_highlights() { ); } +/// Grammar-gap languages: each bundled grammar's name matches the +/// existing `pmacs.lsp.config.` key, so grammar detection (which +/// wins over the filetype map) resolves the id the server keys off — the +/// file now gets BOTH highlighting and the right server. Verified through +/// the loaded runtime. +#[test] +fn m4_gap_grammars_align_with_lsp_configs() { + use pmacs::editor::EditorState; + let s = EditorState::new(); + for (path, id) in [ + ("app.py", "python"), + ("srv.go", "go"), + ("m.js", "javascript"), + ("v.jsx", "javascriptreact"), + ("i.ts", "typescript"), + ("A.tsx", "typescriptreact"), + ("Cargo.toml", "toml"), + ("build.zig", "zig"), + ] { + let (grammar, has_cfg): (Option, bool) = s + .lua_host + .lua() + .load(format!( + "return pmacs.parse.language_for_path('{path}'), + pmacs.lsp.config['{id}'] ~= nil" + )) + .eval() + .unwrap_or_else(|e| panic!("probe {path}: {e}")); + assert_eq!( + grammar.as_deref(), + Some(id), + "{path} grammar detection resolves to {id}" + ); + assert!( + has_cfg, + "{id} has an LSP config the grammar name aligns with" + ); + } +} + /// Typing-perf: the default bundle coalesces full-document /// `didChange` notifications instead of sending one per keystroke /// (each send copies the whole buffer several times and writes