feat(highlight): grammars for python/go/typescript/javascript/toml/zig

These five languages already had LSP configs (basedpyright, gopls,
tsserver, taplo, zls) but shipped no tree-sitter grammar, so they
rendered with no lexical color. Fill the gap — 8 BUILTIN_LANGUAGES
entries across 6 crates:

- python (`tree-sitter-python`, root `module`), go (`tree-sitter-go`),
  toml (`tree-sitter-toml-ng`), zig (`tree-sitter-zig`, +`.zon`) — each a
  single self-contained highlights query.
- JavaScript/TypeScript family: `tree-sitter-javascript` parses both
  `.js` and `.jsx`; `tree-sitter-typescript` ships two grammars
  (LANGUAGE_TYPESCRIPT, LANGUAGE_TSX). The four entries — javascript,
  javascriptreact, typescript, typescriptreact — mirror the LSP filetype
  map so tsserver enables the JSX parser. Highlights inherit: the TS
  query is a ~5-capture delta over JavaScript and JSX is a further
  delta, so the entries compose base-first (js → jsx → ts), the same
  pattern as `cuda` over C/C++ (typescript resolves ~22 capture classes,
  typescriptreact ~24).

Each grammar's name equals its existing `pmacs.lsp.config.<name>` key,
so grammar detection (which wins over the filetype map) resolves the id
the server keys off — the file now gets BOTH highlighting and the right
server. No lsp.lua change needed. All crates ride `tree-sitter-language
0.1` with tree-sitter dev-only — no second core in the graph.

Bite-verified acceptance:
- gap_grammars_load_and_parse — each grammar's ABI accepted by the 0.26
  core; a snippet parses without error at its root (covers both TS
  grammars, incl. JSX).
- typescript_highlights_compose_the_javascript_base — the compiled
  typescript/typescriptreact queries resolve >= 15 captures, not just the
  ~5-capture TS delta (the JS base is really composed in).
- builtin_languages_include_gap_grammars /
  gap_grammar_extensions_resolve — entry presence + extension detection
  across all 8 ids.
- m4_gap_grammars_align_with_lsp_configs — through the loaded runtime,
  each path's grammar id matches an existing LSP config. Bite-verified
  against pre-feature src/syntax.rs.

Ripple: two #116 shebang tests used python as their "has-LSP-but-no-
grammar" example, which this PR invalidates. Updated both — the .py +
`#!/bin/sh` precedence test now asserts a python grammar tree (not "no
tree"), and the grammarless-language-is-silent gate test switches to
`ruby` (genuinely grammarless) via a test-local shebang mapping.

Gates: fmt; clippy -D warnings; test --lib; --features crdt;
m4_acceptance --skip basedpyright; GPU; full workspace sweep;
git diff --check.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YJ9FQ832QwftJXCD9LeFan
This commit is contained in:
Levi Neuwirth 2026-07-14 17:29:54 +01:00
parent 7d1b6d6a33
commit bcec61e020
4 changed files with 329 additions and 19 deletions

66
Cargo.lock generated
View File

@ -2556,10 +2556,16 @@ dependencies = [
"tree-sitter-containerfile",
"tree-sitter-cpp",
"tree-sitter-cuda",
"tree-sitter-go",
"tree-sitter-javascript",
"tree-sitter-lua",
"tree-sitter-make",
"tree-sitter-md",
"tree-sitter-python",
"tree-sitter-rust",
"tree-sitter-toml-ng",
"tree-sitter-typescript",
"tree-sitter-zig",
"unicode-width",
]
@ -3780,6 +3786,26 @@ dependencies = [
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-go"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8560a4d2f835cc0d4d2c2e03cbd0dde2f6114b43bc491164238d333e28b16ea"
dependencies = [
"cc",
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-javascript"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68204f2abc0627a90bdf06e605f5c470aa26fdcb2081ea553a04bdad756693f5"
dependencies = [
"cc",
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-language"
version = "0.1.7"
@ -3816,6 +3842,16 @@ dependencies = [
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-python"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bf85fd39652e740bf60f46f4cda9492c3a9ad75880575bf14960f775cb74a1c"
dependencies = [
"cc",
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-rust"
version = "0.24.2"
@ -3826,6 +3862,36 @@ dependencies = [
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-toml-ng"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9adc2c898ae49730e857d75be403da3f92bb81d8e37a2f918a08dd10de5ebb1"
dependencies = [
"cc",
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-typescript"
version = "0.23.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff"
dependencies = [
"cc",
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-zig"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab11fc124851b0db4dd5e55983bbd9631192e93238389dcd44521715e5d53e28"
dependencies = [
"cc",
"tree-sitter-language",
]
[[package]]
name = "tree_magic_mini"
version = "3.2.2"

View File

@ -181,6 +181,21 @@ tree-sitter-bash = "0.25"
tree-sitter-containerfile = "0.9"
tree-sitter-make = "1.1"
tree-sitter-cmake = "0.7"
# Grammar-gap languages: these already have LSP configs (basedpyright /
# gopls / tsserver / taplo / zls) but shipped no tree-sitter grammar, so
# they rendered with no lexical color. Fill the gap. All ride
# `tree-sitter-language 0.1` (tree-sitter dev-only) and export
# `LANGUAGE`/`HIGHLIGHTS_QUERY`. TypeScript is special: one crate ships
# TWO grammars (`LANGUAGE_TYPESCRIPT`/`LANGUAGE_TSX`) and its highlights
# inherit JavaScript, so the `typescript*` entries compose the JS query
# ahead of the TS one (see `crate::syntax::BUILTIN_LANGUAGES`). JavaScript
# parses both `.js` and `.jsx`.
tree-sitter-python = "0.25"
tree-sitter-go = "0.25"
tree-sitter-javascript = "0.25"
tree-sitter-typescript = "0.23"
tree-sitter-toml-ng = "0.7"
tree-sitter-zig = "1.1"
# T M9.7: markdown grammar so prompt result buffers with
# `_meta.format = "markdown"` get structured highlighting through the
# same M4 path as rust/lua — no special-case painter in Lua.

View File

@ -487,6 +487,78 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
loader: || tree_sitter_cmake::LANGUAGE.into(),
highlights_query: &[tree_sitter_cmake::HIGHLIGHTS_QUERY],
},
// Grammar-gap languages — these already had LSP configs but no
// grammar, so they rendered without lexical color. Each language name
// matches its existing `pmacs.lsp.config.<name>` key, so grammar
// detection (which wins over the filetype map) resolves the same id
// the server keys off. Root kinds: python `module`, go/zig
// `source_file`, js/ts family `program`, toml `document`.
LanguageEntry {
name: "python",
extensions: &["py", "pyi"],
loader: || tree_sitter_python::LANGUAGE.into(),
highlights_query: &[tree_sitter_python::HIGHLIGHTS_QUERY],
},
LanguageEntry {
name: "go",
extensions: &["go"],
loader: || tree_sitter_go::LANGUAGE.into(),
highlights_query: &[tree_sitter_go::HIGHLIGHTS_QUERY],
},
// JavaScript / TypeScript. One `tree-sitter-javascript` grammar parses
// both `.js` and `.jsx`; `tree-sitter-typescript` ships two grammars
// (`LANGUAGE_TYPESCRIPT`, `LANGUAGE_TSX`). Highlights inherit: the TS
// query is a ~5-capture delta over JavaScript, and JSX is a further
// `JSX_HIGHLIGHT_QUERY` delta — so the `*react` and `typescript*`
// entries compose base-first (js → jsx → ts), the same pattern as
// `cuda` over C/C++. The four names mirror the LSP filetype map
// (typescriptreact/javascriptreact) so tsserver enables the JSX parser.
LanguageEntry {
name: "javascript",
extensions: &["js", "mjs", "cjs"],
loader: || tree_sitter_javascript::LANGUAGE.into(),
highlights_query: &[tree_sitter_javascript::HIGHLIGHT_QUERY],
},
LanguageEntry {
name: "javascriptreact",
extensions: &["jsx"],
loader: || tree_sitter_javascript::LANGUAGE.into(),
highlights_query: &[
tree_sitter_javascript::HIGHLIGHT_QUERY,
tree_sitter_javascript::JSX_HIGHLIGHT_QUERY,
],
},
LanguageEntry {
name: "typescript",
extensions: &["ts", "mts", "cts"],
loader: || tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
highlights_query: &[
tree_sitter_javascript::HIGHLIGHT_QUERY,
tree_sitter_typescript::HIGHLIGHTS_QUERY,
],
},
LanguageEntry {
name: "typescriptreact",
extensions: &["tsx"],
loader: || tree_sitter_typescript::LANGUAGE_TSX.into(),
highlights_query: &[
tree_sitter_javascript::HIGHLIGHT_QUERY,
tree_sitter_javascript::JSX_HIGHLIGHT_QUERY,
tree_sitter_typescript::HIGHLIGHTS_QUERY,
],
},
LanguageEntry {
name: "toml",
extensions: &["toml"],
loader: || tree_sitter_toml_ng::LANGUAGE.into(),
highlights_query: &[tree_sitter_toml_ng::HIGHLIGHTS_QUERY],
},
LanguageEntry {
name: "zig",
extensions: &["zig", "zon"],
loader: || tree_sitter_zig::LANGUAGE.into(),
highlights_query: &[tree_sitter_zig::HIGHLIGHTS_QUERY],
},
];
/// Registry that the Lua surface ([`crate::lua_bindings::install_parse`])
@ -1171,6 +1243,119 @@ mod tests {
}
}
#[test]
fn builtin_languages_include_gap_grammars() {
for (name, exts) in [
("python", &["py", "pyi"][..]),
("go", &["go"][..]),
("javascript", &["js", "mjs", "cjs"][..]),
("javascriptreact", &["jsx"][..]),
("typescript", &["ts", "mts", "cts"][..]),
("typescriptreact", &["tsx"][..]),
("toml", &["toml"][..]),
("zig", &["zig", "zon"][..]),
] {
let entry = BUILTIN_LANGUAGES
.iter()
.find(|l| l.name == name)
.unwrap_or_else(|| panic!("`{name}` language entry must be present"));
for ext in exts {
assert!(entry.extensions.contains(ext), "`{name}` claims `.{ext}`");
}
assert!(
entry.highlights_query.iter().any(|q| !q.is_empty()),
"`{name}` ships a highlights query"
);
}
}
#[test]
fn gap_grammars_load_and_parse() {
// ABI acceptance for each new grammar (set_language succeeds at
// runtime) + a snippet that parses without error at the expected
// root. Covers both `tree-sitter-typescript` grammars.
let reg = SyntaxRegistry::new();
let cases: &[(&str, &str, &[u8])] = &[
("python", "module", b"def f(x):\n return x + 1\n"),
("go", "source_file", b"package main\nfunc main() {}\n"),
("javascript", "program", b"const x = 1;\nlet y = [x];\n"),
(
"javascriptreact",
"program",
b"const e = <div id=\"a\"/>;\n",
),
("typescript", "program", b"const x: number = 1;\n"),
("typescriptreact", "program", b"const e = <div/>;\n"),
("toml", "document", b"[pkg]\nname = \"x\"\n"),
("zig", "source_file", b"const std = @import(\"std\");\n"),
];
for (lang, root_kind, src) in cases {
let language = reg
.language(lang)
.unwrap_or_else(|| panic!("`{lang}` loads from BUILTIN_LANGUAGES"));
let mut buf = fresh_buffer(&format!("probe_{lang}"));
buf.apply_edit(EditOp::Insert { pos: 0, bytes: src })
.unwrap();
let view = ParseView::new(&buf, language, (*lang).to_owned());
let handle = view.handle();
let _vid = buf.attach_view(Box::new(view));
let bundle = parse_synchronously(&handle);
assert_eq!(
bundle.tree.root_node().kind(),
*root_kind,
"`{lang}` roots at `{root_kind}`"
);
assert!(
!bundle.tree.root_node().has_error(),
"`{lang}` parses its snippet without error"
);
}
}
#[test]
fn typescript_highlights_compose_the_javascript_base() {
// The bundled TypeScript highlights are a ~5-capture delta over
// JavaScript; the entries prepend the JS query (and JSX for tsx).
// Assert the COMPILED query resolves far more than the delta — the
// JS base is really there, not just the ts-specific captures.
let reg = SyntaxRegistry::new();
for lang in ["typescript", "typescriptreact"] {
let query = reg
.highlights_query(lang)
.unwrap_or_else(|| panic!("`{lang}` highlights compile"));
assert!(
query.capture_names().len() >= 15,
"`{lang}` composes the JavaScript base (got {} captures, delta alone is ~5)",
query.capture_names().len()
);
}
}
#[test]
fn gap_grammar_extensions_resolve() {
let reg = SyntaxRegistry::new();
for (path, lang) in [
("main.py", "python"),
("stub.pyi", "python"),
("server.go", "go"),
("app.js", "javascript"),
("mod.mjs", "javascript"),
("view.jsx", "javascriptreact"),
("index.ts", "typescript"),
("types.mts", "typescript"),
("App.tsx", "typescriptreact"),
("Cargo.toml", "toml"),
("build.zig", "zig"),
("config.zon", "zig"),
] {
assert_eq!(
reg.language_name_for_path(path).as_deref(),
Some(lang),
"{path} resolves to {lang}"
);
}
}
#[test]
fn byte_to_point_handles_first_line() {
let src = b"hello world";

View File

@ -5959,7 +5959,7 @@ fn m4_shebang_extensionless_script_resolves_bash() {
#[test]
fn m4_shebang_does_not_override_extension() {
use pmacs::editor::EditorState;
let s = EditorState::new();
let mut s = EditorState::new();
let dir = tempfile::tempdir().expect("tempdir");
let f = dir.path().join("tool.py");
std::fs::write(&f, b"#!/bin/sh\nprint('hi')\n").expect("write");
@ -5972,29 +5972,25 @@ fn m4_shebang_does_not_override_extension() {
))
.exec()
.expect("open .py with a shell shebang");
// Both the LSP language *and* the grammar decision must respect the
// extension: python for LSP, and NO grammar parse view (python has no
// grammar) — not a bash tree installed from the `#!/bin/sh` line.
// `_has_view` is set synchronously by `_dispatch`, so no pump is
// needed; without the precedence fix the shebang would have dispatched
// bash and this would be true.
let (lang, has_view): (Option<String>, bool) = s
// Both the LSP language *and* the grammar must respect the extension:
// python, not bash from the `#!/bin/sh` line. (Python now has a
// grammar, so the check is "the tree is python", not "no tree at all".)
let lang: Option<String> = s
.lua_host
.lua()
.load(
"return pmacs.lsp.active_buffer_language(),
pmacs.parse._has_view(pmacs.window.buffer())",
)
.load("return pmacs.lsp.active_buffer_language()")
.eval()
.expect("language + view");
.expect("language");
assert_eq!(
lang.as_deref(),
Some("python"),
".py extension wins over a #!/bin/sh shebang (LSP)"
);
assert!(
!has_view,
".py file must not get a grammar parse view from a #!/bin/sh line"
pump_async(&mut s, |st| current_tree_language(st).is_some());
assert_eq!(
current_tree_language(&s).as_deref(),
Some("python"),
".py gets a python grammar tree, not bash from the shebang"
);
}
@ -6009,19 +6005,23 @@ fn m4_shebang_extensionless_grammarless_language_is_silent() {
let s = EditorState::new();
let dir = tempfile::tempdir().expect("tempdir");
let f = dir.path().join("generate"); // no extension
std::fs::write(&f, b"#!/usr/bin/env python3\nprint('hi')\n").expect("write");
// `ruby` is deliberately grammarless (and serverless) — a language the
// shebang resolves but pmacs cannot parse. (python/js/lua/bash all have
// grammars now, so the gate needs a genuinely grammarless example.)
std::fs::write(&f, b"#!/usr/bin/env ruby\nputs 'hi'\n").expect("write");
let f_disp = f.display();
s.lua_host
.lua()
.load(format!(
"pmacs.lsp.config = {{}}
pmacs.parse.shebangs.ruby = 'ruby'
_G.__errs = {{}}
local real = pmacs.error
pmacs.error = function(m) table.insert(_G.__errs, m) end
pmacs.buffer.find_or_open('{f_disp}')"
))
.exec()
.expect("open extensionless python script");
.expect("open extensionless ruby script");
let (lang, has_view, errs): (Option<String>, bool, i64) = s
.lua_host
.lua()
@ -6032,7 +6032,11 @@ fn m4_shebang_extensionless_grammarless_language_is_silent() {
)
.eval()
.expect("probe");
assert_eq!(lang.as_deref(), Some("python"), "python resolves for LSP");
assert_eq!(
lang.as_deref(),
Some("ruby"),
"ruby resolves via the shebang"
);
assert!(
!has_view,
"no grammar parse view for a grammarless language"
@ -6237,6 +6241,46 @@ fn m4_filename_extensionless_dockerfile_highlights() {
);
}
/// Grammar-gap languages: each bundled grammar's name matches the
/// existing `pmacs.lsp.config.<name>` key, so grammar detection (which
/// wins over the filetype map) resolves the id the server keys off — the
/// file now gets BOTH highlighting and the right server. Verified through
/// the loaded runtime.
#[test]
fn m4_gap_grammars_align_with_lsp_configs() {
use pmacs::editor::EditorState;
let s = EditorState::new();
for (path, id) in [
("app.py", "python"),
("srv.go", "go"),
("m.js", "javascript"),
("v.jsx", "javascriptreact"),
("i.ts", "typescript"),
("A.tsx", "typescriptreact"),
("Cargo.toml", "toml"),
("build.zig", "zig"),
] {
let (grammar, has_cfg): (Option<String>, bool) = s
.lua_host
.lua()
.load(format!(
"return pmacs.parse.language_for_path('{path}'),
pmacs.lsp.config['{id}'] ~= nil"
))
.eval()
.unwrap_or_else(|e| panic!("probe {path}: {e}"));
assert_eq!(
grammar.as_deref(),
Some(id),
"{path} grammar detection resolves to {id}"
);
assert!(
has_cfg,
"{id} has an LSP config the grammar name aligns with"
);
}
}
/// Typing-perf: the default bundle coalesces full-document
/// `didChange` notifications instead of sending one per keystroke
/// (each send copies the whole buffer several times and writes