From 9ce6f1abf3790b73e5153156a46a317067a030d4 Mon Sep 17 00:00:00 2001
From: Levi Neuwirth <ln@levineuwirth.org>
Date: Wed, 15 Jul 2026 17:44:59 +0100
Subject: [PATCH] feat(json-yaml): JSON + YAML grammars and language servers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add tree-sitter-json (0.24) and tree-sitter-yaml (0.7) to
BUILTIN_LANGUAGES (both ABI-current via tree-sitter-language, verified
compiling under tree-sitter 0.26), each self-contained highlights, no
injections of their own. Extensions json=.json, yaml=.yaml/.yml; root
kinds json `document`, yaml `stream`.

The payoff from the #122 injection engine is free: the markdown block
injection query already sets injection.language "yaml" for `---`
frontmatter (minus_metadata) and "toml" for `+++` (plus_metadata), so
registering yaml lights up YAML frontmatter highlighting with no extra
wiring, and ```json / ```yaml / ```yml fences resolve through the engine
(yml->yaml alias already present). Two acceptance tests pin this synergy.

LSP (builtin/runtime/lsp.lua): pmacs.lsp.config.json uses the maintained
extracted-bundle binary `vscode-json-language-server --stdio` (NOT the
stale standalone vscode-json-languageserver); MIT, no telemetry, remote
$schema fetch left enabled (no handledSchemaProtocols). pmacs.lsp.config
.yaml uses `yaml-language-server --stdio` with Red Hat telemetry
disabled by default. Both ship the exact workspace/configuration sections
each server pulls (json+http; yaml+http+redhat.telemetry) present-not-null
so the servers get defaults rather than erroring — the CMake #117 lesson.
Sections derived from server source/docs (neither binary installed on
this build machine to observe live; verify where present). Filetype
fallback entries added. JSON is the standing prerequisite for the Jupyter
.ipynb arc; handoff §6 updated.

Nine acceptance tests (grammar ABI, highlights compile, detection,
grammar<->LSP-key alignment, the two frontmatter/fence synergy proofs,
and the pinned LSP-config sections).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YJ9FQ832QwftJXCD9LeFan
---
 Cargo.lock              |  22 +++++
 Cargo.toml              |   7 ++
 builtin/runtime/lsp.lua |  39 +++++++++
 docs/agent-handoff.md   |  12 ++-
 src/syntax.rs           | 182 ++++++++++++++++++++++++++++++++++++++++
 tests/m4_acceptance.rs  |  69 +++++++++++++++
 6 files changed, 327 insertions(+), 4 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 15707b8..dd5b071 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2558,6 +2558,7 @@ dependencies = [
  "tree-sitter-cuda",
  "tree-sitter-go",
  "tree-sitter-javascript",
+ "tree-sitter-json",
  "tree-sitter-lua",
  "tree-sitter-make",
  "tree-sitter-md",
@@ -2565,6 +2566,7 @@ dependencies = [
  "tree-sitter-rust",
  "tree-sitter-toml-ng",
  "tree-sitter-typescript",
+ "tree-sitter-yaml",
  "tree-sitter-zig",
  "unicode-width",
 ]
@@ -3807,6 +3809,16 @@ dependencies = [
  "tree-sitter-language",
 ]
 
+[[package]]
+name = "tree-sitter-json"
+version = "0.24.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4d727acca406c0020cffc6cf35516764f36c8e3dc4408e5ebe2cb35a947ec471"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
 [[package]]
 name = "tree-sitter-language"
 version = "0.1.7"
@@ -3883,6 +3895,16 @@ dependencies = [
  "tree-sitter-language",
 ]
 
+[[package]]
+name = "tree-sitter-yaml"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53c223db85f05e34794f065454843b0668ebc15d240ada63e2b5939f43ce7c97"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
 [[package]]
 name = "tree-sitter-zig"
 version = "1.1.2"
diff --git a/Cargo.toml b/Cargo.toml
index 98fec84..c3e9ea9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -196,6 +196,13 @@ tree-sitter-javascript = "0.25"
 tree-sitter-typescript = "0.23"
 tree-sitter-toml-ng = "0.7"
 tree-sitter-zig = "1.1"
+# JSON + YAML — config formats + the honest gate on the Jupyter path
+# (JSON). Both are ABI-current (`LANGUAGE: LanguageFn` via
+# `tree-sitter-language`), NOT a `tree-sitter ^0.20` fork. Registering
+# yaml also lights up markdown `---` frontmatter through the #122
+# injection engine (the block injection query already sets yaml for it).
+tree-sitter-json = "0.24"
+tree-sitter-yaml = "0.7"
 # T M9.7: markdown grammar so prompt result buffers with
 # `_meta.format = "markdown"` get structured highlighting through the
 # same M4 path as rust/lua — no special-case painter in Lua.
diff --git a/builtin/runtime/lsp.lua b/builtin/runtime/lsp.lua
index eb4f1e4..74c3866 100644
--- a/builtin/runtime/lsp.lua
+++ b/builtin/runtime/lsp.lua
@@ -200,6 +200,40 @@ pmacs.lsp.config.zig = pmacs.lsp.config.zig or {
   args = {},
 }
 
+-- JSON via the maintained `vscode-langservers-extracted` bundle
+-- (`vscode-json-language-server`, NOT the stale standalone
+-- `vscode-json-languageserver` npm package). Schema-driven
+-- diagnostics/completion; auto-associates well-known files
+-- (`package.json`, `tsconfig.json`) via its built-in schema store. It
+-- pulls `workspace/configuration` for the `json` and `http` sections, so
+-- both ship present-not-null (empty ⇒ server defaults). The underlying
+-- Microsoft server is MIT with no telemetry path; its only outbound
+-- behavior is fetching remote `$schema` content, LEFT ENABLED by default
+-- (disabling via `handledSchemaProtocols = {"file"}` would break remote
+-- schemas without a `vscode/content` implementation). Sections are
+-- derived from the server source/docs — the binary is not installed on
+-- this build machine to observe live; verify where it is present.
+pmacs.lsp.config.json = pmacs.lsp.config.json or {
+  command = "vscode-json-language-server",
+  args = { "--stdio" },
+  settings = { json = {}, http = {} },
+}
+
+-- YAML via Red Hat `yaml-language-server`. It pulls
+-- `workspace/configuration` for `yaml`, `http`, and `redhat.telemetry`;
+-- all three ship present-not-null, with Red Hat telemetry disabled by
+-- default (privacy-respecting; inert if the server never asks). Sections
+-- derived from source/docs — verify against an installed binary.
+pmacs.lsp.config.yaml = pmacs.lsp.config.yaml or {
+  command = "yaml-language-server",
+  args = { "--stdio" },
+  settings = {
+    yaml = {},
+    http = {},
+    redhat = { telemetry = { enabled = false } },
+  },
+}
+
 -- LSP-side extension → language map, deliberately independent of the
 -- tree-sitter detection in `pmacs.parse`. Consulted only when
 -- `pmacs.parse.language_for_path` finds nothing (an extension with a
@@ -267,6 +301,11 @@ pmacs.lsp.filetypes.toml = pmacs.lsp.filetypes.toml or "toml"
 -- Zig (zls). `.zon` is Zig Object Notation, handled by the same server.
 pmacs.lsp.filetypes.zig = pmacs.lsp.filetypes.zig or "zig"
 pmacs.lsp.filetypes.zon = pmacs.lsp.filetypes.zon or "zig"
+-- JSON / YAML. Both ship grammars, so `language_for_path` already resolves
+-- these and the map is the stable-id fallback (same role as `lua`/`cuda`).
+pmacs.lsp.filetypes.json = pmacs.lsp.filetypes.json or "json"
+pmacs.lsp.filetypes.yaml = pmacs.lsp.filetypes.yaml or "yaml"
+pmacs.lsp.filetypes.yml = pmacs.lsp.filetypes.yml or "yaml"
 
 -- Per-buffer attachment record: { language, server, uri, version }.
 -- Keyed by `tostring(BufferIdLua)` because BufferIdLua hands out fresh
diff --git a/docs/agent-handoff.md b/docs/agent-handoff.md
index d9986c7..d54cba0 100644
--- a/docs/agent-handoff.md
+++ b/docs/agent-handoff.md
@@ -323,11 +323,15 @@ runtime/Lua-registered languages (v1 resolves only against
 `BUILTIN_LANGUAGES`), and the next injection *consumers* gated on new
 grammars — HTML/CSS/GraphQL/SQL (`<script>`/`<style>`, JS/TS template
 literals, doc-comment code); modeline detection as a 5th layer
-(`-*- mode: … -*-` / `# vim: ft=…`); JSON/YAML grammars+LSP;
+(`-*- mode: … -*-` / `# vim: ft=…`);
 byte-accurate multibyte cursor placement in `move_active_cursor_to`
-(still steps one codepoint per LSP byte column). A full Jupyter `.ipynb`
-setup (reader → editable → kernel execution) is a real arc now gated on
-**JSON** (injections shipped in #122), NOT a one-shot.
+(still steps one codepoint per LSP byte column). **JSON + YAML grammars
+LANDED** (grammar-gap style, `tree-sitter-json`/`-yaml`; LSP configs
+`vscode-json-language-server` / `yaml-language-server`; YAML `---` and
+TOML `+++` markdown frontmatter now highlight via the #122 engine). A
+full Jupyter `.ipynb` setup (reader → editable → kernel execution) has
+both grammar prerequisites in place now (JSON + injections); what remains
+is the reader → editable → kernel arc itself, NOT a one-shot.
 GPU: auto-reconnect after daemon restart, splits/multi-buffer, gutter
 riders (whitespace guides, folding, git markers).
 Themes (full list in theme-faces framing rev 9 "Deferred (named)"):
diff --git a/src/syntax.rs b/src/syntax.rs
index 50ef47d..cdd6d56 100644
--- a/src/syntax.rs
+++ b/src/syntax.rs
@@ -1027,6 +1027,27 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
         highlights_query: &[tree_sitter_zig::HIGHLIGHTS_QUERY],
         injections_query: &[],
     },
+    // JSON + YAML — config formats, both self-contained highlights and no
+    // injections of their own. Registering `yaml` also lights up markdown
+    // `---` frontmatter via the #122 injection engine (the markdown block
+    // injection query sets `injection.language "yaml"` for `minus_metadata`;
+    // `+++` TOML frontmatter already works). Root kinds: json `document`,
+    // yaml `stream`. `.jsonc`/`.json5` (comments / trailing commas) are a
+    // deferred variant — the plain JSON grammar rejects them.
+    LanguageEntry {
+        name: "json",
+        extensions: &["json"],
+        loader: || tree_sitter_json::LANGUAGE.into(),
+        highlights_query: &[tree_sitter_json::HIGHLIGHTS_QUERY],
+        injections_query: &[],
+    },
+    LanguageEntry {
+        name: "yaml",
+        extensions: &["yaml", "yml"],
+        loader: || tree_sitter_yaml::LANGUAGE.into(),
+        highlights_query: &[tree_sitter_yaml::HIGHLIGHTS_QUERY],
+        injections_query: &[],
+    },
 ];
 
 /// Registry that the Lua surface ([`crate::lua_bindings::install_parse`])
@@ -2062,6 +2083,167 @@ mod tests {
         }
     }
 
+    #[test]
+    fn builtin_languages_include_json_and_yaml() {
+        // Framing acceptance #1: both entries present, claim their
+        // extensions, ship non-empty highlights.
+        let json = BUILTIN_LANGUAGES
+            .iter()
+            .find(|l| l.name == "json")
+            .expect("`json` entry present");
+        assert!(json.extensions.contains(&"json"), "`json` claims `.json`");
+        assert!(!json.highlights_query.is_empty(), "`json` ships highlights");
+        let yaml = BUILTIN_LANGUAGES
+            .iter()
+            .find(|l| l.name == "yaml")
+            .expect("`yaml` entry present");
+        assert!(yaml.extensions.contains(&"yaml"), "`yaml` claims `.yaml`");
+        assert!(yaml.extensions.contains(&"yml"), "`yaml` claims `.yml`");
+        assert!(!yaml.highlights_query.is_empty(), "`yaml` ships highlights");
+    }
+
+    #[test]
+    fn json_grammar_loads_and_parses() {
+        // Framing acceptance #2 / ABI pin: `tree-sitter-json` 0.24 is
+        // accepted by our tree-sitter 0.26 core; a JSON object parses to a
+        // `document` root without error.
+        let reg = SyntaxRegistry::new();
+        let language = reg.language("json").expect("`json` loads");
+        let mut buf = fresh_buffer("data.json");
+        buf.apply_edit(EditOp::Insert {
+            pos: 0,
+            bytes: b"{\n  \"name\": \"pmacs\",\n  \"nums\": [1, 2, 3],\n  \"ok\": true\n}\n",
+        })
+        .unwrap();
+        let view = ParseView::new(&buf, language, "json".to_owned());
+        let handle = view.handle();
+        let _vid = buf.attach_view(Box::new(view));
+        let bundle = parse_synchronously(&handle);
+        assert_eq!(
+            bundle.root_tree().root_node().kind(),
+            "document",
+            "json grammar roots at `document`"
+        );
+        assert!(
+            !bundle.root_tree().root_node().has_error(),
+            "json grammar parses an object without error"
+        );
+    }
+
+    #[test]
+    fn yaml_grammar_loads_and_parses() {
+        // Framing acceptance #3 / ABI pin: `tree-sitter-yaml` 0.7 loads and
+        // a YAML mapping parses to a `stream` root without error.
+        let reg = SyntaxRegistry::new();
+        let language = reg.language("yaml").expect("`yaml` loads");
+        let mut buf = fresh_buffer("config.yaml");
+        buf.apply_edit(EditOp::Insert {
+            pos: 0,
+            bytes: b"name: pmacs\nversion: 1\ntags:\n  - a\n  - b\n",
+        })
+        .unwrap();
+        let view = ParseView::new(&buf, language, "yaml".to_owned());
+        let handle = view.handle();
+        let _vid = buf.attach_view(Box::new(view));
+        let bundle = parse_synchronously(&handle);
+        assert_eq!(
+            bundle.root_tree().root_node().kind(),
+            "stream",
+            "yaml grammar roots at `stream`"
+        );
+        assert!(
+            !bundle.root_tree().root_node().has_error(),
+            "yaml grammar parses a mapping without error"
+        );
+    }
+
+    #[test]
+    fn json_yaml_highlights_compile() {
+        // Framing acceptance #4: both highlights queries compile against
+        // their grammars and resolve capture classes.
+        let reg = SyntaxRegistry::new();
+        let json = reg
+            .highlights_query("json")
+            .expect("json highlights compile");
+        assert!(
+            json.capture_names().len() >= 3,
+            "json highlights resolve capture classes; got {}",
+            json.capture_names().len()
+        );
+        let yaml = reg
+            .highlights_query("yaml")
+            .expect("yaml highlights compile");
+        assert!(
+            yaml.capture_names().len() >= 3,
+            "yaml highlights resolve capture classes; got {}",
+            yaml.capture_names().len()
+        );
+    }
+
+    #[test]
+    fn language_for_path_resolves_json_yaml() {
+        // Framing acceptance #5.
+        let reg = SyntaxRegistry::new();
+        assert_eq!(
+            reg.language_name_for_path("tsconfig.json").as_deref(),
+            Some("json")
+        );
+        assert_eq!(
+            reg.language_name_for_path("config.yaml").as_deref(),
+            Some("yaml")
+        );
+        assert_eq!(
+            reg.language_name_for_path("ci.yml").as_deref(),
+            Some("yaml")
+        );
+    }
+
+    #[test]
+    fn yaml_frontmatter_injects_in_markdown() {
+        // Framing acceptance #7 — THE headline synergy with #122: a markdown
+        // `---` frontmatter block (a `minus_metadata` node) is injected as
+        // yaml by the bundled markdown injection query, so registering the
+        // yaml grammar lights it up with no extra wiring.
+        let reg = SyntaxRegistry::new();
+        let src = b"---\ntitle: Hello\ntags: [a, b]\n---\n\n# Body\n";
+        let bundle = parse_layered(&reg, "markdown", src);
+        let yaml = bundle
+            .layers
+            .iter()
+            .find(|l| l.language_name == "yaml")
+            .expect("`---` frontmatter yields a yaml child layer");
+        assert_eq!(
+            yaml.tree.root_node().kind(),
+            "stream",
+            "yaml layer roots at stream"
+        );
+        let query = yaml
+            .highlight_query
+            .as_ref()
+            .expect("yaml highlights resolved");
+        let spans = compute_highlight_spans_for(query, &yaml.tree, &bundle.source, None);
+        assert!(!spans.is_empty(), "the yaml frontmatter layer highlights");
+    }
+
+    #[test]
+    fn json_fence_injects_in_markdown() {
+        // Framing acceptance #8: a ```json fence yields a json child layer
+        // through the #122 engine.
+        let reg = SyntaxRegistry::new();
+        let src = b"# Doc\n\n```json\n{\"a\": 1, \"b\": [2, 3]}\n```\n";
+        let bundle = parse_layered(&reg, "markdown", src);
+        let json = bundle
+            .layers
+            .iter()
+            .find(|l| l.language_name == "json")
+            .expect("a ```json fence yields a json child layer");
+        assert_eq!(
+            json.tree.root_node().kind(),
+            "document",
+            "json layer roots at document"
+        );
+    }
+
     #[test]
     fn builtin_languages_include_dockerfile_make_cmake() {
         for (name, exts) in [
diff --git a/tests/m4_acceptance.rs b/tests/m4_acceptance.rs
index a20cd16..fff4ff7 100644
--- a/tests/m4_acceptance.rs
+++ b/tests/m4_acceptance.rs
@@ -6265,6 +6265,9 @@ fn m4_gap_grammars_align_with_lsp_configs() {
         ("A.tsx", "typescriptreact"),
         ("Cargo.toml", "toml"),
         ("build.zig", "zig"),
+        ("tsconfig.json", "json"),
+        ("config.yaml", "yaml"),
+        ("ci.yml", "yaml"),
     ] {
         let (grammar, has_cfg): (Option<String>, bool) = s
             .lua_host
@@ -6287,6 +6290,72 @@ fn m4_gap_grammars_align_with_lsp_configs() {
     }
 }
 
+/// JSON/YAML LSP configs pin the exact server binary and the
+/// `workspace/configuration` sections each server pulls (framing Q#JY2).
+/// The servers are not installed on the build machine, so the section set
+/// is derived from server source/docs and pinned here as the config
+/// contract — a machine with the binaries should confirm them live. The
+/// point is to pin the sections, not merely that some settings table
+/// exists.
+#[test]
+fn m4_json_yaml_lsp_configs_pin_command_and_sections() {
+    use pmacs::editor::EditorState;
+    let s = EditorState::new();
+    let lua = s.lua_host.lua();
+
+    // json: the maintained extracted-bundle binary (NOT the stale
+    // standalone `vscode-json-languageserver`), `--stdio`, and the `json`
+    // + `http` workspace-config sections present (empty ⇒ server defaults;
+    // remote `$schema` fetch left enabled — no `handledSchemaProtocols`).
+    let json_command: String = lua
+        .load("return pmacs.lsp.config.json.command")
+        .eval()
+        .unwrap();
+    assert_eq!(
+        json_command, "vscode-json-language-server",
+        "json uses the extracted-bundle binary, not the stale standalone"
+    );
+    let json_ok: bool = lua
+        .load(
+            "local c = pmacs.lsp.config.json
+             return c.args[1] == '--stdio'
+                 and c.settings.json ~= nil
+                 and c.settings.http ~= nil
+                 and c.settings.handledSchemaProtocols == nil",
+        )
+        .eval()
+        .unwrap();
+    assert!(
+        json_ok,
+        "json config: --stdio, json + http sections present, remote schemas left on"
+    );
+
+    // yaml: `yaml-language-server --stdio`; `yaml` + `http` +
+    // `redhat.telemetry` sections, Red Hat telemetry disabled by default.
+    let yaml_command: String = lua
+        .load("return pmacs.lsp.config.yaml.command")
+        .eval()
+        .unwrap();
+    assert_eq!(
+        yaml_command, "yaml-language-server",
+        "yaml uses the Red Hat yaml-language-server"
+    );
+    let yaml_ok: bool = lua
+        .load(
+            "local c = pmacs.lsp.config.yaml
+             return c.args[1] == '--stdio'
+                 and c.settings.yaml ~= nil
+                 and c.settings.http ~= nil
+                 and c.settings.redhat.telemetry.enabled == false",
+        )
+        .eval()
+        .unwrap();
+    assert!(
+        yaml_ok,
+        "yaml config: --stdio, yaml + http + redhat.telemetry sections, telemetry off"
+    );
+}
+
 /// Typing-perf: the default bundle coalesces full-document
 /// `didChange` notifications instead of sending one per keystroke
 /// (each send copies the whole buffer several times and writes