From 4282b1c3336567a100267571d7e5152806827e1b Mon Sep 17 00:00:00 2001 From: Levi Neuwirth Date: Wed, 15 Jul 2026 10:57:41 +0100 Subject: [PATCH] feat(injections): multi-language injection layers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Teach the syntax engine that one buffer can hold more than one language. After the root parse, run the grammar's injections.scm, parse each embedded region with the injected language, and merge every layer's highlight spans. First consumer: markdown fenced code + inline (zero new grammars — the block grammar already ships an injection query and the injected langs already have grammars from #118). Engine (src/syntax.rs): - ParseTreeBundle now holds Vec (root layer 0 + injected children, depth-ascending); installed atomically so the existing Arc::ptr_eq style gate and highlight cache keep working (Q#IJ1). - run_parse builds layers on the worker: run injections.scm, resolve the injected language, compute Vec (exclude NAMED children, intersect the parent's ranges), set_included_ranges cold-parse, recurse — bounded by depth (3), a layer backstop (4096), and a (lang,ranges) visited guard; any child failure drops that child only (Q#IJ3/IJ5). LanguageEntry gains injections_query; markdown_inline is registered (retires the M9.7 block-only floor); markdown/rust carry injection queries. - Injected languages resolve off the static BUILTIN_LANGUAGES table (Send loaders + query sources), preserving lazy loading. Dynamic fence names go through a case-folded alias map seeded with defaults and Lua-extensible via pmacs.parse.injection_aliases, snapshotted into ParseRequest at dispatch so the worker never touches the Rc registry or a Lua table (Q#IJ2/IJ4). Highlight queries are resolved at settle (resolve_layer_queries), keeping query compilation main-thread/cached. Producers: - SyntaxHighlightView (grid) iterates layers shallow-to-deep so a deeper layer's styling wins within its region (Q#IJ6/IJ7). - scoped_style_spans (wire) flattens all layers into DISJOINT effective spans via a boundary sweep, since the GPU re-sorts spans by start (replace_style_spans / merge_style_spans) and would otherwise destroy producer order. The GPU source_color_at consumer is fixed to fold all covering spans (matching semantic_client's effective_style_at) rather than returning the first. Named-children exclusion: content ranges exclude only NAMED children (matching tree-sitter-md's own inline splitter) — excluding a block inline node's anonymous text tokens would shred the paragraph into unparseable fragments. 13 acceptance gates (framing docs/multi-language-injections-framing.md): layer structure, absolute child offsets, alias resolution (static + case-folded dynamic + unknown-skip + Lua-async override), multi-range inline, recursion bounds, wire + grid + GPU producers, incremental edit / new fence, many-paragraph settle budget with tail coverage, and the single-layer regression guard. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01YJ9FQ832QwftJXCD9LeFan --- builtin/runtime/syntax.lua | 14 + pmacs-gpu/src/main.rs | 56 ++- src/async_runtime.rs | 4 +- src/highlight.rs | 257 ++++++---- src/lua_bindings/mod.rs | 50 +- src/semantic_render.rs | 222 +++++++-- src/syntax.rs | 852 +++++++++++++++++++++++++++++++++- tests/injection_acceptance.rs | 165 +++++++ tests/m4_acceptance.rs | 9 +- 9 files changed, 1465 insertions(+), 164 deletions(-) create mode 100644 tests/injection_acceptance.rs diff --git a/builtin/runtime/syntax.lua b/builtin/runtime/syntax.lua index b639377..6ed7d86 100644 --- a/builtin/runtime/syntax.lua +++ b/builtin/runtime/syntax.lua @@ -44,6 +44,20 @@ function pmacs.parse._dispatch(buf, lang) return job_id end +-- Injection language aliases (framing Q#IJ4). The registry holds the +-- merged map (seeded with defaults on the Rust side), and each dispatch +-- snapshots it into the parse request so the worker can resolve dynamic +-- fence names (`py` → python, `ts` → typescript). Exposed as a +-- write-through proxy so users add fence-name aliases from init.lua: +-- pmacs.parse.injection_aliases.mylang = "rust" +-- Reads are not proxied (the canonical map lives Rust-side); this is a +-- write-only extension surface. +pmacs.parse.injection_aliases = setmetatable({}, { + __newindex = function(_, alias, lang) + pmacs.parse._register_injection_alias(alias, lang) + end, +}) + -- Shebang → language detection ------------------------------------------ -- -- Extension detection (`language_for_path`) misses extensionless scripts diff --git a/pmacs-gpu/src/main.rs b/pmacs-gpu/src/main.rs index 79567e4..a932e68 100644 --- a/pmacs-gpu/src/main.rs +++ b/pmacs-gpu/src/main.rs @@ -6732,12 +6732,24 @@ fn adornment_text_color(fg: CellColor) -> glyphon::Color { } fn source_color_at(byte: u64, spans: &[StyleSpan]) -> Option { - for sp in spans { - if sp.range.start <= byte && byte < sp.range.end { - return cell_color_to_glyphon(sp.style.fg); + // Fold every covering span in order, matching the semantic-client + // `effective_style_at` contract (last covering span with a non-default + // fg wins) rather than returning the first. The daemon flattens + // injection layers into disjoint spans (framing Q#IJ6), so usually at + // most one covers a byte — but where spans do overlap (a styled + // markdown parent under an injected child), the topmost color must win, + // never the outermost. Spans arrive start-sorted, so a narrower nested + // span (later start) folds after and overrides its enclosing span. + let mut color = None; + for sp in spans + .iter() + .filter(|sp| sp.range.start <= byte && byte < sp.range.end) + { + if let Some(c) = cell_color_to_glyphon(sp.style.fg) { + color = Some(c); } } - None + color } /// Convert a `pmacs-protocol::cell::Color` to a `glyphon::Color`. @@ -8969,4 +8981,40 @@ mod tests { "the peer cursor line must keep the CurrentLine constant" ); } + + #[test] + fn source_color_folds_overlapping_child_over_parent() { + // Framing acceptance #9 (GPU consumer): a styled markdown parent + // span [0,10) (red) with an injected child span [3,6) (green) on top. + // `replace_style_spans` sorts incoming spans by start, so we mirror + // that here; `source_color_at` must FOLD the covering spans (child + // green wins at a shared byte), not return the first (parent) span. + // Bite: the pre-fix first-covering-span code returns red at byte 4. + let red = style_with_fg(CellColor::Rgb(200, 0, 0)); + let green = style_with_fg(CellColor::Rgb(0, 200, 0)); + let mut spans = vec![ + StyleSpan { + range: ByteRange { start: 0, end: 10 }, + style: red, + }, + StyleSpan { + range: ByteRange { start: 3, end: 6 }, + style: green, + }, + ]; + spans.sort_by_key(|s| s.range.start); // as replace_style_spans does + + // Byte 4 is covered by both: the child (green) wins the fold. + assert_eq!( + source_color_at(4, &spans), + Some(glyphon::Color::rgb(0, 200, 0)), + "the injected child color wins over the parent at a shared byte" + ); + // Byte 1 is parent-only: stays red. + assert_eq!( + source_color_at(1, &spans), + Some(glyphon::Color::rgb(200, 0, 0)), + "a parent-only byte keeps the parent color" + ); + } } diff --git a/src/async_runtime.rs b/src/async_runtime.rs index 94ae001..6bc8292 100644 --- a/src/async_runtime.rs +++ b/src/async_runtime.rs @@ -2700,6 +2700,7 @@ mod tests { language_name: "rust".to_owned(), prior_tree: None, edits: Vec::new(), + injection_aliases: Arc::new(std::collections::HashMap::new()), }; let id = rt.dispatch_parse(req, None); pump_until(&rt, || rt.is_complete(id)); @@ -2713,7 +2714,7 @@ mod tests { other => panic!("unexpected outcome: {other:?}"), } assert_eq!(bundle.language_name, "rust"); - assert_eq!(bundle.tree.root_node().kind(), "source_file"); + assert_eq!(bundle.root_tree().root_node().kind(), "source_file"); // take_parse_tree was already drained, so handoff is empty. assert_eq!(rt.parse_handoff_len(), 0); } @@ -2729,6 +2730,7 @@ mod tests { language_name: "rust".to_owned(), prior_tree: None, edits: Vec::new(), + injection_aliases: Arc::new(std::collections::HashMap::new()), }; let id = rt.dispatch_parse(req, None); pump_until(&rt, || rt.is_complete(id)); diff --git a/src/highlight.rs b/src/highlight.rs index 8b10981..a74176a 100644 --- a/src/highlight.rs +++ b/src/highlight.rs @@ -39,7 +39,7 @@ use crate::buffer::Buffer; use crate::cell::{CellCoord, CellGrid, Color, Style, UnderlineStyle}; use crate::lsp::SharedLspManager; use crate::overlay::merge_styles; -use crate::syntax::{HighlightSpan, ParseTreeBundle, ParseViewHandle, compute_highlight_spans}; +use crate::syntax::{HighlightSpan, ParseTreeBundle, ParseViewHandle, compute_highlight_spans_for}; use crate::view::{View, Viewport}; // --------------------------------------------------------------------------- @@ -248,6 +248,19 @@ pub type ThemeHandle = Arc>; // SyntaxHighlightView // --------------------------------------------------------------------------- +/// Per-layer cached highlight spans + capture names (framing Q#IJ7). +/// One entry per bundle layer that has a highlight query; kept in the +/// bundle's depth order so the render paints shallow-to-deep. +struct LayerSpans { + /// Spans for this layer, sorted wider-first per + /// [`compute_highlight_spans_for`]. + spans: Vec, + /// Capture names indexed by `HighlightSpan::capture_index` for this + /// layer's own query. Layers may use different grammars, so this is + /// per-layer, not shared. + capture_names: Arc<[String]>, +} + /// Cached per-bundle highlight state. Keyed by the underlying /// `Arc`'s identity (compared via `Arc::ptr_eq`), /// so a freshly-installed bundle invalidates the cache. @@ -255,25 +268,19 @@ struct HighlightCache { /// The bundle the cache was built against. `None` until the /// first render observes a settled bundle. bundle: Option>, - /// Compiled spans, sorted wider-first per - /// [`compute_highlight_spans`]. - spans: Vec, + /// Per-layer spans in depth-ascending order (framing Q#IJ6). + layers: Vec, /// Per-row first-byte offsets into `bundle.source`. `Vec` /// because pmacs files cap at 4 GiB. line_offsets: Vec, - /// Capture names indexed by `HighlightSpan::capture_index`. - /// Populated alongside `spans` so the render path doesn't need - /// to keep a reference into the [`tree_sitter::Query`]. - capture_names: Arc<[String]>, } impl HighlightCache { fn empty() -> Self { Self { bundle: None, - spans: Vec::new(), + layers: Vec::new(), line_offsets: Vec::new(), - capture_names: Arc::from(Vec::::new().into_boxed_slice()), } } } @@ -283,39 +290,41 @@ impl HighlightCache { const TAB_WIDTH: u32 = 8; /// View that renders syntax highlighting from a tree-sitter parse -/// tree. Composes over [`crate::text_view::TextView`] per the M2.9 -/// view-composition contract --- it never writes glyphs, only merges -/// styles into cells the base view has already painted. +/// tree, including its injection layers. Composes over +/// [`crate::text_view::TextView`] per the M2.9 view-composition +/// contract --- it never writes glyphs, only merges styles into cells +/// the base view has already painted. pub struct SyntaxHighlightView { parse: ParseViewHandle, - query: Arc, theme: ThemeHandle, cache: HighlightCache, } impl SyntaxHighlightView { - /// Construct a highlight view over `parse` using `query` and - /// `theme`. The initial cache is empty; the first render with - /// a settled bundle populates it. + /// Construct a highlight view over `parse` using `theme`. Per-layer + /// highlight queries come from each `Layer::highlight_query` in the + /// bundle (resolved at settle, framing Q#IJ2), so no query is passed + /// here. The initial cache is empty; the first render with a settled + /// bundle populates it. #[must_use] - pub fn new(parse: ParseViewHandle, query: Arc, theme: ThemeHandle) -> Self { + pub fn new(parse: ParseViewHandle, theme: ThemeHandle) -> Self { Self { parse, - query, theme, cache: HighlightCache::empty(), } } - /// Test helper: number of cached highlight spans. + /// Test helper: total cached highlight spans across all layers. #[must_use] pub fn cached_span_count(&self) -> usize { - self.cache.spans.len() + self.cache.layers.iter().map(|l| l.spans.len()).sum() } /// Refresh `self.cache` if the parse view's current bundle /// differs from the cached one. No-op when the bundle pointer - /// is unchanged --- the steady-state cost between parses. + /// is unchanged --- the steady-state cost between parses. Rebuilds + /// spans for every layer that carries a highlight query. fn refresh_cache_if_stale(&mut self) { let Some(bundle) = self.parse.current() else { return; @@ -328,26 +337,39 @@ impl SyntaxHighlightView { if !stale { return; } - let spans = compute_highlight_spans(&self.query, &bundle); - let line_offsets = compute_line_offsets(bundle.source.as_ref()); - let capture_names: Arc<[String]> = self - .query - .capture_names() - .iter() - .map(|s| (*s).to_owned()) - .collect(); + let source = bundle.source.as_ref(); + let mut layers = Vec::new(); + for layer in &bundle.layers { + let Some(query) = layer.highlight_query.as_ref() else { + continue; + }; + let spans = compute_highlight_spans_for(query, &layer.tree, source, None); + if spans.is_empty() { + continue; + } + let capture_names: Arc<[String]> = query + .capture_names() + .iter() + .map(|s| (*s).to_owned()) + .collect(); + layers.push(LayerSpans { + spans, + capture_names, + }); + } + let line_offsets = compute_line_offsets(source); self.cache = HighlightCache { bundle: Some(bundle), - spans, + layers, line_offsets, - capture_names, }; } - /// Look up the style for span `s`, consulting the active theme. - fn style_for(&self, theme: &Theme, s: HighlightSpan) -> Style { + /// Look up the style for span `s` within `layer`, consulting the + /// active theme. + fn style_for(theme: &Theme, layer: &LayerSpans, s: HighlightSpan) -> Style { let idx = s.capture_index as usize; - let Some(name) = self.cache.capture_names.get(idx) else { + let Some(name) = layer.capture_names.get(idx) else { return theme.default_style; }; theme.lookup(name) @@ -364,7 +386,7 @@ impl View for SyntaxHighlightView { let Some(bundle) = self.cache.bundle.clone() else { return; }; - if self.cache.spans.is_empty() || self.cache.line_offsets.is_empty() { + if self.cache.layers.is_empty() || self.cache.line_offsets.is_empty() { return; } let source: &[u8] = bundle.source.as_ref(); @@ -376,61 +398,64 @@ impl View for SyntaxHighlightView { let cell_origin = viewport.cell_origin; let total_lines = self.cache.line_offsets.len() as u32; - for row_offset in 0..max_rows { - let line_idx = start_line + row_offset; - if line_idx >= total_lines { - break; - } - let line_start = self.cache.line_offsets[line_idx as usize]; - let line_end = self - .cache - .line_offsets - .get(line_idx as usize + 1) - .copied() - .unwrap_or(source.len() as u32); - // Trim a single trailing newline if any --- the text - // view doesn't paint it as a glyph either. - let line_end_no_nl = if line_end > line_start - && source.get(line_end as usize - 1).copied() == Some(b'\n') - { - line_end - 1 - } else { - line_end - }; - let line_bytes = &source[line_start as usize..line_end_no_nl as usize]; + // Paint layers shallow-to-deep (framing Q#IJ6): a deeper layer + // merges on top, so an injected child's styling wins within its + // region. Within a layer, wider-first ordering (from + // `compute_highlight_spans_for`) lets narrower captures override. + for layer in &self.cache.layers { + for row_offset in 0..max_rows { + let line_idx = start_line + row_offset; + if line_idx >= total_lines { + break; + } + let line_start = self.cache.line_offsets[line_idx as usize]; + let line_end = self + .cache + .line_offsets + .get(line_idx as usize + 1) + .copied() + .unwrap_or(source.len() as u32); + // Trim a single trailing newline if any --- the text + // view doesn't paint it as a glyph either. + let line_end_no_nl = if line_end > line_start + && source.get(line_end as usize - 1).copied() == Some(b'\n') + { + line_end - 1 + } else { + line_end + }; + let line_bytes = &source[line_start as usize..line_end_no_nl as usize]; - // Spans whose start lies on this line. Wider-first - // ordering means parents apply before children. - // Spans that *cross* lines apply on every line they - // touch --- this loop only filters by start, then - // intersects with the line range below. - for span in self - .cache - .spans - .iter() - .filter(|s| s.start_byte < line_end_no_nl && s.end_byte > line_start) - .copied() - { - let style = self.style_for(&theme, span); - if style == Style::default() { - // Nothing to merge --- skip the per-cell loop. - continue; - } - let s_start = span.start_byte.max(line_start); - let s_end = span.end_byte.min(line_end_no_nl); - let byte_col_start = (s_start - line_start) as usize; - let byte_col_end = (s_end - line_start) as usize; - let (start_col, end_col) = - byte_range_to_display_cols(line_bytes, byte_col_start, byte_col_end); - if end_col <= start_col { - continue; - } - let cell_row = cell_origin.row + row_offset; - let clamped_start = start_col.min(max_cols); - let clamped_end = end_col.min(max_cols); - for col in clamped_start..clamped_end { - let cell = cells.at(CellCoord::new(cell_row, cell_origin.col + col)); - cell.style = merge_styles(cell.style, style); + // Spans whose start lies on this line. Spans that *cross* + // lines apply on every line they touch --- this loop filters + // by intersection, then clips to the line range below. + for span in layer + .spans + .iter() + .filter(|s| s.start_byte < line_end_no_nl && s.end_byte > line_start) + .copied() + { + let style = Self::style_for(&theme, layer, span); + if style == Style::default() { + // Nothing to merge --- skip the per-cell loop. + continue; + } + let s_start = span.start_byte.max(line_start); + let s_end = span.end_byte.min(line_end_no_nl); + let byte_col_start = (s_start - line_start) as usize; + let byte_col_end = (s_end - line_start) as usize; + let (start_col, end_col) = + byte_range_to_display_cols(line_bytes, byte_col_start, byte_col_end); + if end_col <= start_col { + continue; + } + let cell_row = cell_origin.row + row_offset; + let clamped_start = start_col.min(max_cols); + let clamped_end = end_col.min(max_cols); + for col in clamped_start..clamped_end { + let cell = cells.at(CellCoord::new(cell_row, cell_origin.col + col)); + cell.style = merge_styles(cell.style, style); + } } } } @@ -1228,4 +1253,56 @@ mod tests { ); assert!(!c.style.bold); } + + #[test] + fn grid_paints_injected_child_keyword() { + // Framing acceptance #10: the grid `SyntaxHighlightView` paints a + // rust keyword cell INSIDE a markdown ```rust fence — styling only + // the injected rust layer can produce. Layers paint shallow-to-deep + // (framing Q#IJ6), so the rust keyword wins over the markdown root. + use crate::buffer::{Buffer, BufferId, EditOp}; + use crate::cell::{Cell, CellSize}; + use crate::syntax::{ParseView, SyntaxRegistry}; + + let reg = SyntaxRegistry::new(); + let language = reg.language("markdown").expect("markdown grammar"); + // Line 0: ```rust ; line 1: fn demo() {} ; line 2: ``` + let src = b"```rust\nfn demo() {}\n```\n"; + let mut buf = Buffer::new(BufferId::next(), "doc.md"); + buf.apply_edit(EditOp::Insert { pos: 0, bytes: src }) + .unwrap(); + let view = ParseView::new(&buf, language, "markdown".to_owned()); + let handle = view.handle(); + let _vid = buf.attach_view(Box::new(view)); + let mut req = handle.make_request(); + req.injection_aliases = reg.injection_alias_snapshot(); + let bundle = crate::syntax::run_parse(req).expect("markdown parse"); + handle.install(reg.resolve_layer_queries(&bundle)); + + let mut hv = SyntaxHighlightView::new(handle, reg.theme()); + let cols = 40usize; + let rows = 3usize; + let mut backing: Vec = vec![Cell::default(); rows * cols]; + let mut grid = CellGrid { + cells: &mut backing, + stride: cols as u32, + size: CellSize::new(rows as u32, cols as u32), + }; + let viewport = Viewport { + buffer_start: 0, + buffer_end: u64::MAX, + cell_origin: CellCoord::new(0, 0), + cell_size: CellSize::new(rows as u32, cols as u32), + gutter_w: 0, + }; + let registry = buf; // keep buf alive + hv.render(®istry, viewport, &mut grid); + + // `fn` is at line 1, cols 0..2. default_dark styles `keyword` bold. + let c = grid.get(CellCoord::new(1, 0)); + assert!( + c.style.bold, + "the injected rust `fn` keyword is painted (bold) inside the fence" + ); + } } diff --git a/src/lua_bindings/mod.rs b/src/lua_bindings/mod.rs index b153ffd..487d098 100644 --- a/src/lua_bindings/mod.rs +++ b/src/lua_bindings/mod.rs @@ -6362,7 +6362,7 @@ pub struct ParseNodeLua { impl ParseNodeLua { fn resolve(&self) -> Option> { - let mut node = self.bundle.tree.root_node(); + let mut node = self.bundle.root_tree().root_node(); for &idx in &self.path { node = node.child(idx)?; } @@ -6395,7 +6395,9 @@ impl UserData for ParseTreeLua { methods.add_method("text", |lua, this, ()| { lua.create_string(this.0.source.as_ref()) }); - methods.add_method("sexp", |_, this, ()| Ok(this.0.tree.root_node().to_sexp())); + methods.add_method("sexp", |_, this, ()| { + Ok(this.0.root_tree().root_node().to_sexp()) + }); } } @@ -6617,6 +6619,21 @@ pub fn install_parse( )?; } + // Injection alias write-through (framing Q#IJ4). `syntax.lua` wraps + // this in a `pmacs.parse.injection_aliases` proxy table so users write + // `pmacs.parse.injection_aliases.mylang = "rust"`. The registry holds + // the merged map (defaults + overrides); each dispatch snapshots it. + { + let s = syntax.clone(); + parse_mod.set( + "_register_injection_alias", + lua.create_function(move |_, (alias, lang): (String, String)| { + s.register_injection_alias(alias, lang); + Ok(()) + })?, + )?; + } + { let s = syntax.clone(); parse_mod.set( @@ -6658,7 +6675,10 @@ pub fn install_parse( let handle = get_or_create_parse_view(&s, ®, id.0, &lang)?; let req = handle.make_request(); let bundle = syntax::run_parse(req).map_err(mlua::Error::external)?; - let arc = Arc::new(bundle); + // Resolve each layer's highlight query from the registry + // cache before install so producers can style every layer + // (framing Q#IJ2 stage 2). + let arc = s.resolve_layer_queries(&bundle); handle.install(arc.clone()); Ok(ParseTreeLua(arc)) })?, @@ -6676,7 +6696,10 @@ pub fn install_parse( "_dispatch", lua.create_function(move |_, (id, lang): (BufferIdLua, String)| { let handle = get_or_create_parse_view(&s, ®, id.0, &lang)?; - let req = handle.make_request(); + let mut req = handle.make_request(); + // Snapshot the alias map into the request so the worker can + // resolve dynamic fence names off the main thread (Q#IJ4). + req.injection_aliases = s.injection_alias_snapshot(); let job_id = rt.dispatch_parse(req, None); s.record_parse_job(job_id, id.0); Ok(job_id) @@ -6707,7 +6730,10 @@ pub fn install_parse( return Ok(false); }; if let Some(handle) = s.view(buf_id) { - handle.install(bundle); + // Stage 2 (framing Q#IJ2): resolve each layer's highlight + // query on the main thread before install. + let resolved = s.resolve_layer_queries(&bundle); + handle.install(resolved); Ok(true) } else { Ok(false) @@ -6737,12 +6763,14 @@ pub fn install_parse( id.0 ))); }; - let Some(query) = s.highlights_query(&lang) else { - // No highlights query for this language --- treat as - // a benign no-op so callers don't need to special-case - // grammars without highlights bundled. + if s.highlights_query(&lang).is_none() { + // Root language ships no highlights --- treat as a benign + // no-op so callers don't need to special-case grammars + // without highlights bundled. (Injected child layers still + // resolve their own queries at settle; a root-highlight-less + // injector is out of v1 scope.) return Ok(false); - }; + } let theme = s.theme(); let core = lua .app_data_ref::() @@ -6755,7 +6783,7 @@ pub fn install_parse( id.0 ))); } - let overlay = SyntaxHighlightView::new(handle, query, theme); + let overlay = SyntaxHighlightView::new(handle, theme); win.push_overlay(Box::new(overlay)); Ok(true) })?, diff --git a/src/semantic_render.rs b/src/semantic_render.rs index cf64e7a..f7132a4 100644 --- a/src/semantic_render.rs +++ b/src/semantic_render.rs @@ -1743,12 +1743,6 @@ fn scoped_style_spans(state: &EditorState, vp: &DeclaredViewport) -> Vec Vec = Vec::new(); + for layer in &bundle.layers { + let Some(query) = layer.highlight_query.as_ref() else { continue; }; - let style = theme.lookup(name); + let names = query.capture_names(); + let highlights = crate::syntax::compute_highlight_spans_for( + query, + &layer.tree, + source, + Some(vis_start as usize..vis_end as usize), + ); + for (order, hs) in highlights.iter().enumerate() { + let s = u64::from(hs.start_byte).max(vis_start); + let e = u64::from(hs.end_byte).min(vis_end); + if e <= s { + continue; + } + let Some(name) = names.get(hs.capture_index as usize) else { + continue; + }; + let style = theme.lookup(name); + if style == Style::default() { + continue; + } + styled.push(StyledLayerSpan { + start: s, + end: e, + style, + priority: (layer.depth, order as u32), + }); + } + } + flatten_layer_spans(&styled) +} + +/// One styled span from a single injection layer, tagged with a priority +/// used to resolve overlaps: `(depth, order)`, higher wins. `order` is the +/// index in the layer's wider-first list, so a narrower capture (later) +/// overrides a wider one within the same layer. +struct StyledLayerSpan { + start: u64, + end: u64, + style: Style, + priority: (u16, u32), +} + +/// Flatten possibly-overlapping per-layer styled spans into **disjoint** +/// `StyleSpan`s whose per-byte style is the priority-ordered fold of every +/// covering span (framing Q#IJ6). Emitting disjoint spans makes the result +/// robust to the GPU wire re-sorting spans by start (`replace_style_spans` +/// / `merge_style_spans`), which would otherwise destroy producer order. +/// A boundary sweep over the (viewport-bounded) span endpoints; adjacent +/// equal-style runs are merged for wire economy. +fn flatten_layer_spans(styled: &[StyledLayerSpan]) -> Vec { + if styled.is_empty() { + return Vec::new(); + } + let mut bounds: Vec = Vec::with_capacity(styled.len() * 2); + for sp in styled { + bounds.push(sp.start); + bounds.push(sp.end); + } + bounds.sort_unstable(); + bounds.dedup(); + + let mut out: Vec = Vec::new(); + for win in bounds.windows(2) { + let (a, b) = (win[0], win[1]); + if b <= a { + continue; + } + // Fold every span covering [a, b) in ascending priority so a + // deeper/narrower span overrides, while a span that only sets a + // non-color attribute still composes (matches the semantic-client + // `effective_style_at` contract). + let mut covering: Vec<&StyledLayerSpan> = styled + .iter() + .filter(|sp| sp.start <= a && sp.end >= b) + .collect(); + if covering.is_empty() { + continue; + } + covering.sort_by_key(|sp| sp.priority); + let mut style = Style::default(); + for sp in covering { + style = crate::overlay::merge_styles(style, sp.style); + } if style == Style::default() { - continue; // Nothing to render — skip the wire byte. + continue; + } + if let Some(last) = out.last_mut() + && last.range.end == a + && last.style == style + { + last.range.end = b; + continue; } out.push(StyleSpan { - range: ByteRange { start: s, end: e }, + range: ByteRange { start: a, end: b }, style, }); } @@ -3300,7 +3376,9 @@ mod tests { let handle = parse_view.handle(); let req = handle.make_request(); let bundle = crate::syntax::run_parse(req).expect("initial rust parse"); - handle.install(std::sync::Arc::new(bundle)); + // Mirror the production settle path: resolve each layer's highlight + // query before install so the producer can style it (framing Q#IJ2). + handle.install(state.syntax_registry.resolve_layer_queries(&bundle)); buf.attach_view(Box::new(parse_view)); drop(registry); core.set_buffer_path(buffer_id, Some(std::path::PathBuf::from("/tmp/x.rs"))); @@ -3309,6 +3387,90 @@ mod tests { handle } + fn seed_markdown_parse_view( + state: &EditorState, + buffer_id: BufferId, + text: &[u8], + ) -> crate::syntax::ParseViewHandle { + let language = state + .syntax_registry + .language("markdown") + .expect("markdown grammar"); + let mut core = state.core.borrow_mut(); + let registry_handle = core.registry.clone(); + let mut registry = registry_handle.borrow_mut(); + let buf = registry.get_mut(buffer_id).expect("active buffer"); + if !text.is_empty() { + buf.apply_edit(crate::buffer::EditOp::Insert { + pos: 0, + bytes: text, + }) + .expect("seed markdown text"); + } + let parse_view = crate::syntax::ParseView::new(buf, language, "markdown".to_owned()); + let handle = parse_view.handle(); + let mut req = handle.make_request(); + req.injection_aliases = state.syntax_registry.injection_alias_snapshot(); + let bundle = crate::syntax::run_parse(req).expect("markdown parse"); + handle.install(state.syntax_registry.resolve_layer_queries(&bundle)); + buf.attach_view(Box::new(parse_view)); + drop(registry); + core.set_buffer_path(buffer_id, Some(std::path::PathBuf::from("/tmp/x.md"))); + drop(core); + state.syntax_registry.attach_view(buffer_id, handle.clone()); + handle + } + + #[test] + fn wire_producer_emits_disjoint_child_spans_in_fence() { + // Framing acceptance #8: `scoped_style_spans` over a ```rust fence + // emits a styled span on the `fn` keyword INSIDE the fence — which + // only the injected rust layer can produce (the markdown root has + // no keyword styling there). The pre-injection single-layer producer + // fails this. Emitted spans are disjoint (framing Q#IJ6 flatten). + let state = empty_state(); + let bid = active_buffer(&state); + let src = b"# T\n\n```rust\nfn demo() {}\n```\n"; + seed_markdown_parse_view(&state, bid, src); + let vp = DeclaredViewport { + buffer_id: bid, + visible: ByteRange { + start: 0, + end: src.len() as u64, + }, + frontend_generation: 0, + }; + let spans = scoped_style_spans(&state, &vp); + assert!(!spans.is_empty(), "layered producer emits style spans"); + + // Disjoint (flattened): no two spans overlap. + let mut sorted = spans.clone(); + sorted.sort_by_key(|s| s.range.start); + for w in sorted.windows(2) { + assert!( + w[0].range.end <= w[1].range.start, + "flattened spans must be disjoint: {:?} then {:?}", + w[0].range, + w[1].range + ); + } + + // The `fn` keyword inside the fence carries a non-default style. + let fn_off = src + .windows(2) + .position(|w| w == b"fn") + .expect("`fn` present in source") as u64; + let covering = spans + .iter() + .find(|s| s.range.start <= fn_off && fn_off < s.range.end) + .expect("a span covers the `fn` keyword inside the fence"); + assert_ne!( + covering.style, + Style::default(), + "the injected rust keyword is styled" + ); + } + #[test] fn cpp_style_comes_from_lsp_when_no_tree_sitter_grammar() { let state = empty_state(); @@ -3402,7 +3564,7 @@ mod tests { ); let bundle = crate::syntax::run_parse(req).expect("settled rust parse"); - handle.install(std::sync::Arc::new(bundle)); + handle.install(state.syntax_registry.resolve_layer_queries(&bundle)); assert_eq!(state.syntax_registry.take_parse_job(9001), Some(bid)); let settled = s.render_frame(&state); assert!( diff --git a/src/syntax.rs b/src/syntax.rs index 33577e0..ef01159 100644 --- a/src/syntax.rs +++ b/src/syntax.rs @@ -32,12 +32,12 @@ //! tests. use std::cell::RefCell; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::rc::Rc; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; -use tree_sitter::StreamingIterator; +use tree_sitter::{Node, Point, Range, StreamingIterator}; use crate::async_runtime::JobId; use crate::buffer::{Buffer, BufferError, BufferId}; @@ -71,6 +71,14 @@ pub struct ParseRequest { /// produced. Empty for cold parses; non-empty drives incremental /// re-parse. pub edits: Vec, + /// Snapshot of the injection alias map (framing Q#IJ4). The worker + /// resolves a dynamic `@injection.language` fence-name (`py`, `ts`, + /// `c++`) through this map — case-folded — before matching it against + /// [`BUILTIN_LANGUAGES`]. Snapshotted from the registry at dispatch so + /// the worker never touches the main-thread `Rc` registry or a Lua + /// table. Empty for the non-layered/legacy callers (no injections + /// resolve, root parse unaffected). + pub injection_aliases: Arc>, } /// Output of [`run_parse`]. The runtime's parse-handoff side map @@ -78,22 +86,58 @@ pub struct ParseRequest { /// resolves a buffer id to its current bundle and walks the tree. #[derive(Debug)] pub struct ParseTreeBundle { - /// The freshly-produced parse tree. - pub tree: tree_sitter::Tree, - /// Source bytes the tree was parsed against. Co-owned with the - /// request so node-byte-range lookups can read the underlying + /// Injection layers (framing Q#IJ1). `layers[0]` is the root layer + /// (the whole buffer, parsed with the buffer's own grammar); + /// subsequent entries are injected child layers in depth-ascending + /// order. Always non-empty — a parse produces at least the root, so + /// [`Self::root_tree`] never panics. + pub layers: Vec, + /// Source bytes every layer's tree was parsed against. Co-owned with + /// the request so node-byte-range lookups can read the underlying /// text (T M4.1 acceptance: "parse tree introspectable via Lua" - /// implies the source the tree references). + /// implies the source the tree references). Child layers parse the + /// *same* full source via `set_included_ranges`, so their node + /// offsets are absolute into these bytes (framing mechanic #1). pub source: Arc<[u8]>, - /// Language label. Stored alongside the tree so Lua can ask - /// "what grammar produced this?". + /// Root language label (`layers[0].language_name`). Kept here so Lua + /// and the `*workers*` buffer can ask "what grammar produced this?" + /// without indexing the layer vec. pub language_name: String, - /// Wall-clock duration of the parse itself (excludes dispatch + /// Wall-clock duration of the whole layered parse (excludes dispatch /// queueing, source materialization, and bus delivery). T M4.1 /// acceptance criteria are stated in this metric. pub parse_duration: Duration, } +/// One injection layer within a [`ParseTreeBundle`] (framing Q#IJ1). A +/// layer pairs a parse tree with the language that produced it and the +/// injection-nesting depth (root = 0). `highlight_query` is resolved on +/// the main thread at settle from the registry cache (framing Q#IJ2) — +/// the worker leaves it `None`. +#[derive(Debug)] +pub struct Layer { + /// Canonical language name of the grammar that produced `tree`. + pub language_name: String, + /// The layer's parse tree. Node offsets are absolute into the + /// bundle's `source` (child layers use `set_included_ranges`). + pub tree: tree_sitter::Tree, + /// Injection depth: 0 for the root, 1 for a direct injection, etc. + pub depth: u16, + /// Compiled `highlights.scm` for `language_name`, resolved at settle. + /// `None` when the language ships no highlights, or on the worker + /// (pre-settle). Producers read it to style this layer. + pub highlight_query: Option>, +} + +impl ParseTreeBundle { + /// The root layer's tree (`layers[0]`) — the whole-buffer parse. + /// Never panics: [`run_parse`] always seeds the root layer. + #[must_use] + pub fn root_tree(&self) -> &tree_sitter::Tree { + &self.layers[0].tree + } +} + /// Run a parse. This is the worker-side body that the runtime's /// `dispatch_parse` closure invokes after pulling a job from the /// queue. Always synchronous --- there is no internal yielding. @@ -115,18 +159,391 @@ pub fn run_parse(req: ParseRequest) -> Result { } } let started = Instant::now(); - let tree = parser + let root_tree = parser .parse(req.source.as_ref(), prior.as_ref()) .ok_or_else(|| "parser produced no tree".to_owned())?; + // `parse_duration` measures the root parse only — the metric the M4.1 + // acceptance gates are stated in. Injection layer building (below) is an + // additive phase separately guarded by the settle-time budget test; it + // must not retroactively inflate this metric. let parse_duration = started.elapsed(); + + // Seed the root layer, then expand injection layers (framing Q#IJ1). + // Injection expansion is best-effort and isolated to the child + // (Q#IJ3): a failed/unknown/over-budget child drops that child only — + // the root always installs, so this returns `Ok` whenever the root + // parsed. + let mut layers = vec![Layer { + language_name: req.language_name.clone(), + tree: root_tree, + depth: 0, + highlight_query: None, + }]; + build_injection_layers(&mut layers, req.source.as_ref(), &req.injection_aliases); Ok(ParseTreeBundle { - tree, + layers, source: req.source, language_name: req.language_name, parse_duration, }) } +// --------------------------------------------------------------------------- +// Injection layers (framing Q#IJ2 -- Q#IJ5). Worker-side: this runs on a +// parse worker, so it touches no `Rc` registry and no Lua — it resolves +// injected languages by indexing the `&'static BUILTIN_LANGUAGES` table +// (loaders + `injections_query` sources are `Send`) and case-folds fence +// names through the `ParseRequest`'s alias snapshot. +// --------------------------------------------------------------------------- + +/// Max injection nesting depth (framing Q#IJ3). markdown→rust is depth 1. +const MAX_INJECTION_DEPTH: u16 = 3; +/// Runaway backstop on total layers per buffer (framing Q#IJ3) — set well +/// above any real document (a markdown doc's one-inline-layer-per-paragraph +/// sits far under this). Purely anti-runaway; the perf bound is the +/// settle-time acceptance guard, not this number. If hit, tail layers are +/// dropped (degraded highlighting on a pathological file only). +const MAX_INJECTION_LAYERS: usize = 4096; + +/// The default fence-name → canonical-language alias map (framing Q#IJ4). +/// Keys are lowercase; the resolver case-folds before lookup. Seeded into +/// the registry and snapshotted into each [`ParseRequest`]; also handy for +/// tests that build a request without the registry. +#[must_use] +pub fn default_injection_aliases() -> HashMap { + [ + ("js", "javascript"), + ("jsx", "javascriptreact"), + ("ts", "typescript"), + ("tsx", "typescriptreact"), + ("py", "python"), + ("py3", "python"), + ("python3", "python"), + ("rs", "rust"), + ("sh", "bash"), + ("shell", "bash"), + ("shellscript", "bash"), + ("zsh", "bash"), + ("c++", "cpp"), + ("cxx", "cpp"), + ("cc", "cpp"), + ("golang", "go"), + ("yml", "yaml"), + ("md", "markdown"), + ] + .into_iter() + .map(|(a, b)| (a.to_owned(), b.to_owned())) + .collect() +} + +/// One injection region resolved from a parent layer's `injections.scm`. +/// Ranges are already child-excluded and normalized (Q#IJ5) but not yet +/// intersected with the parent layer's ranges (that happens per-parent in +/// [`build_injection_layers`], which knows the parent's included ranges). +struct InjectionMatch { + /// Raw language name — dynamic capture text or a static `#set!` value. + language: String, + /// Child-excluded content ranges for this match, sorted/non-overlapping. + ranges: Vec, +} + +/// Expand injection layers under the already-parsed root (`layers[0]`), +/// appending children in depth-ascending order (Q#IJ1, Q#IJ6 rely on this +/// ordering). Bounded by depth, total layer count, and a +/// `(language, ranges)` visited guard (Q#IJ3). BFS by depth so siblings +/// at a level are grouped before descending. +fn build_injection_layers( + layers: &mut Vec, + source: &[u8], + aliases: &HashMap, +) { + let mut query_cache: HashMap>> = HashMap::new(); + let mut visited: HashSet<(String, Vec<(usize, usize)>)> = HashSet::new(); + // Frontier entries are (layer index, that layer's included ranges). + let mut frontier: Vec<(usize, Vec)> = vec![(0, vec![whole_source_range(source)])]; + let mut depth: u16 = 0; + + while depth < MAX_INJECTION_DEPTH && !frontier.is_empty() { + // Children discovered this level: (layer, its ranges) to append and + // (if any injections themselves) descend into next level. + let mut children: Vec<(Layer, Vec)> = Vec::new(); + 'parents: for (parent_idx, parent_ranges) in &frontier { + let parent_lang = layers[*parent_idx].language_name.clone(); + let Some(query) = injection_query_cached(&mut query_cache, &parent_lang) else { + continue; + }; + for m in collect_injection_matches(&query, &layers[*parent_idx].tree, source) { + if layers.len() + children.len() >= MAX_INJECTION_LAYERS { + break 'parents; // runaway backstop; tail dropped + } + let Some(child_lang) = resolve_injected_language(&m.language, aliases) else { + continue; // unknown/unaliased language — skip this child only + }; + let mut ranges = intersect_ranges(&m.ranges, parent_ranges, source); + normalize_ranges(&mut ranges); + if ranges.is_empty() { + continue; + } + let key = (child_lang.to_owned(), ranges_key(&ranges)); + if !visited.insert(key) { + continue; // same (language, ranges) already parsed — cycle guard + } + let Some(tree) = parse_child(child_lang, &ranges, source) else { + continue; // child parse failed — skip this child only + }; + children.push(( + Layer { + language_name: child_lang.to_owned(), + tree, + depth: depth + 1, + highlight_query: None, + }, + ranges, + )); + } + } + if children.is_empty() { + break; + } + let mut next_frontier = Vec::with_capacity(children.len()); + for (layer, ranges) in children { + let idx = layers.len(); + layers.push(layer); + next_frontier.push((idx, ranges)); + } + frontier = next_frontier; + depth += 1; + } +} + +/// Compile (once, cached) the `injections.scm` for `lang` from the static +/// [`BUILTIN_LANGUAGES`] table, or `None` if the language ships none. +fn injection_query_cached( + cache: &mut HashMap>>, + lang: &str, +) -> Option> { + if let Some(slot) = cache.get(lang) { + return slot.clone(); + } + let compiled = BUILTIN_LANGUAGES + .iter() + .find(|e| e.name == lang) + .and_then(|entry| { + let source = entry.injections_query.join("\n"); + if source.trim().is_empty() { + return None; + } + let language = (entry.loader)(); + tree_sitter::Query::new(&language, &source) + .ok() + .map(Arc::new) + }); + cache.insert(lang.to_owned(), compiled.clone()); + compiled +} + +/// Run `query` over `tree` and return each injection region: its raw +/// language name (dynamic `@injection.language` node text, or static +/// `#set! injection.language`) and its child-excluded content ranges. +fn collect_injection_matches( + query: &tree_sitter::Query, + tree: &tree_sitter::Tree, + source: &[u8], +) -> Vec { + let names = query.capture_names(); + let content_cap = names.iter().position(|n| *n == "injection.content"); + let Some(content_cap) = content_cap.map(|i| i as u32) else { + return Vec::new(); + }; + let lang_cap = names + .iter() + .position(|n| *n == "injection.language") + .map(|i| i as u32); + + let mut out = Vec::new(); + let mut cursor = tree_sitter::QueryCursor::new(); + let mut it = cursor.matches(query, tree.root_node(), source); + while let Some(m) = it.next() { + // Static language + include-children from `#set!` property settings. + let mut static_lang: Option = None; + let mut include_children = false; + for prop in query.property_settings(m.pattern_index) { + match &*prop.key { + "injection.language" => { + static_lang = prop.value.as_deref().map(str::to_owned); + } + "injection.include-children" => include_children = true, + _ => {} + } + } + let mut dyn_lang: Option = None; + let mut ranges: Vec = Vec::new(); + for cap in m.captures { + if Some(cap.index) == lang_cap { + if let Ok(text) = cap.node.utf8_text(source) { + dyn_lang = Some(text.to_owned()); + } + } else if cap.index == content_cap { + ranges.extend(content_node_ranges(cap.node, include_children)); + } + } + let Some(language) = static_lang.or(dyn_lang) else { + continue; + }; + normalize_ranges(&mut ranges); + if ranges.is_empty() { + continue; + } + out.push(InjectionMatch { language, ranges }); + } + out +} + +/// The included ranges for one `@injection.content` node (framing Q#IJ5 / +/// mechanic #3). With `include_children`, the whole node span; otherwise +/// the node's extent minus its **named** children's ranges. Anonymous +/// token children are *kept* — they are the injected text itself, not +/// structure to exclude. (This matches `tree-sitter-md`'s own inline +/// splitter, `bindings/rust/parser.rs:410`, which filters on `is_named()`: +/// excluding a block `inline` node's anonymous text tokens would shred the +/// paragraph into unparseable fragments. Our real injection sites — a +/// childless `code_fence_content`, an `inline` with only anonymous +/// children, an `include-children` macro `token_tree` — all resolve +/// correctly under this rule.) A node with no named children yields its +/// whole span. +fn content_node_ranges(node: Node, include_children: bool) -> Vec { + if include_children { + return vec![node.range()]; + } + let mut ranges = Vec::new(); + let mut start_byte = node.start_byte(); + let mut start_point = node.start_position(); + let mut cursor = node.walk(); + if cursor.goto_first_child() { + loop { + let child = cursor.node(); + if child.is_named() { + if child.start_byte() > start_byte { + ranges.push(Range { + start_byte, + end_byte: child.start_byte(), + start_point, + end_point: child.start_position(), + }); + } + start_byte = child.end_byte(); + start_point = child.end_position(); + } + if !cursor.goto_next_sibling() { + break; + } + } + } + if node.end_byte() > start_byte { + ranges.push(Range { + start_byte, + end_byte: node.end_byte(), + start_point, + end_point: node.end_position(), + }); + } + ranges +} + +/// Clip `candidate` ranges to `parent` ranges (framing Q#IJ5): a nested +/// injection cannot reintroduce bytes its parent excluded. Points are +/// recomputed only for a clipped edge (unclipped edges keep the node's +/// exact point). At depth 1 the parent is the whole buffer, so this is a +/// pass-through. +fn intersect_ranges(candidate: &[Range], parent: &[Range], source: &[u8]) -> Vec { + let mut out = Vec::new(); + for c in candidate { + for p in parent { + let start = c.start_byte.max(p.start_byte); + let end = c.end_byte.min(p.end_byte); + if end > start { + out.push(Range { + start_byte: start, + end_byte: end, + start_point: if start == c.start_byte { + c.start_point + } else { + byte_to_point(source, start) + }, + end_point: if end == c.end_byte { + c.end_point + } else { + byte_to_point(source, end) + }, + }); + } + } + } + out +} + +/// Sort, drop empty, and merge overlapping ranges so the result satisfies +/// `set_included_ranges`' sorted/non-overlapping/non-empty contract. +fn normalize_ranges(ranges: &mut Vec) { + ranges.retain(|r| r.end_byte > r.start_byte); + ranges.sort_by_key(|r| r.start_byte); + let mut merged: Vec = Vec::with_capacity(ranges.len()); + for r in ranges.drain(..) { + if let Some(last) = merged.last_mut() + && r.start_byte < last.end_byte + { + if r.end_byte > last.end_byte { + last.end_byte = r.end_byte; + last.end_point = r.end_point; + } + continue; + } + merged.push(r); + } + *ranges = merged; +} + +/// A hashable identity for a range set (framing Q#IJ3 visited guard). +fn ranges_key(ranges: &[Range]) -> Vec<(usize, usize)> { + ranges.iter().map(|r| (r.start_byte, r.end_byte)).collect() +} + +/// Case-fold `raw`, apply the alias map, then resolve against the bundled +/// table (framing Q#IJ4). Returns the canonical `&'static` name, or `None` +/// for an unknown language. +fn resolve_injected_language(raw: &str, aliases: &HashMap) -> Option<&'static str> { + let lower = raw.trim().to_ascii_lowercase(); + if lower.is_empty() { + return None; + } + let candidate: &str = aliases.get(&lower).map_or(lower.as_str(), String::as_str); + BUILTIN_LANGUAGES + .iter() + .find(|e| e.name == candidate) + .map(|e| e.name) +} + +/// Cold-parse `source` restricted to `ranges` with `lang`'s grammar. Node +/// offsets in the returned tree are absolute into `source` (mechanic #1). +fn parse_child(lang: &str, ranges: &[Range], source: &[u8]) -> Option { + let entry = BUILTIN_LANGUAGES.iter().find(|e| e.name == lang)?; + let language = (entry.loader)(); + let mut parser = tree_sitter::Parser::new(); + parser.set_language(&language).ok()?; + parser.set_included_ranges(ranges).ok()?; + parser.parse(source, None) +} + +/// The whole-buffer range, the root layer's parent range. +fn whole_source_range(source: &[u8]) -> Range { + Range { + start_byte: 0, + end_byte: source.len(), + start_point: Point::new(0, 0), + end_point: byte_to_point(source, source.len()), + } +} + /// Convert a byte offset within `source` to a tree-sitter /// `(row, column)` [`tree_sitter::Point`]. `byte` is clamped to /// `source.len()`. @@ -285,13 +702,17 @@ impl ParseViewHandle { pub fn make_request(&self) -> ParseRequest { let mut inner = self.inner.lock().expect("ParseView mutex poisoned"); let edits = std::mem::take(&mut inner.pending); - let prior_tree = inner.current.as_ref().map(|b| b.tree.clone()); + let prior_tree = inner.current.as_ref().map(|b| b.root_tree().clone()); ParseRequest { source: Arc::from(inner.source.clone()), language: inner.language.clone(), language_name: inner.language_name.clone(), prior_tree, edits, + // Empty by default; the dispatch binding overrides with the + // registry's alias snapshot (framing Q#IJ4). Callers that need + // injections and bypass the registry set this themselves. + injection_aliases: Arc::new(HashMap::new()), } } @@ -339,6 +760,13 @@ pub struct LanguageEntry { /// slice (or all-empty fragments) means no highlights: the view /// runs but emits nothing. pub highlights_query: &'static [&'static str], + /// Bundled `injections.scm` fragments (framing Q#IJ2), joined with a + /// newline and compiled on the parse worker to find embedded-language + /// regions. Empty for the many grammars that ship none (or don't + /// inject). Names are inconsistent across crates — markdown exposes + /// `INJECTION_QUERY_BLOCK`, rust `INJECTIONS_QUERY`, most none — the + /// same shape `highlights_query` already absorbs. + pub injections_query: &'static [&'static str], } /// Bundled grammars (T M4.2 + M4.3). The order is significant only @@ -360,12 +788,14 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["rs"], loader: || tree_sitter_rust::LANGUAGE.into(), highlights_query: &[tree_sitter_rust::HIGHLIGHTS_QUERY], + injections_query: &[tree_sitter_rust::INJECTIONS_QUERY], }, LanguageEntry { name: "lua", extensions: &["lua"], loader: || tree_sitter_lua::LANGUAGE.into(), highlights_query: &[tree_sitter_lua::HIGHLIGHTS_QUERY], + injections_query: &[], }, // T M9.7: markdown grammar for prompt-result buffers with // `_meta.format = "markdown"`. Uses only the block grammar @@ -385,6 +815,20 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["md", "markdown"], loader: || tree_sitter_md::LANGUAGE.into(), highlights_query: &[tree_sitter_md::HIGHLIGHT_QUERY_BLOCK], + injections_query: &[tree_sitter_md::INJECTION_QUERY_BLOCK], + }, + // markdown_inline (framing Q#IJ10) — the inline grammar the block + // grammar injects for paragraph/heading text (`#set! injection.language + // "markdown_inline"`). No file extension: it is injection-only, never + // opened directly by name. Ships an inline highlights query (emphasis, + // links, code spans) and its own injections (e.g. inline HTML), so it + // recurses like any other layer. Retires the M9.7 block-only floor. + LanguageEntry { + name: "markdown_inline", + extensions: &[], + loader: || tree_sitter_md::INLINE_LANGUAGE.into(), + highlights_query: &[tree_sitter_md::HIGHLIGHT_QUERY_INLINE], + injections_query: &[tree_sitter_md::INJECTION_QUERY_INLINE], }, // T M_B3 — C / C++. Lexical highlighting (keywords / strings / // operators) so the grid TUI shows code-shaped C++ on first open. @@ -406,12 +850,14 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["c", "h"], loader: || tree_sitter_c::LANGUAGE.into(), highlights_query: &[tree_sitter_c::HIGHLIGHT_QUERY], + injections_query: &[], }, LanguageEntry { name: "cpp", extensions: &["cpp", "cc", "cxx", "hpp", "hh", "hxx", "ipp", "inl", "cppm"], loader: || tree_sitter_cpp::LANGUAGE.into(), highlights_query: &[tree_sitter_cpp::HIGHLIGHT_QUERY], + injections_query: &[], }, // CUDA (`.cu` source, `.cuh` header). A dedicated grammar rather // than reusing `cpp`: CUDA extends C++ with `__global__`/`__device__` @@ -442,6 +888,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ tree_sitter_cpp::HIGHLIGHT_QUERY, tree_sitter_cuda::HIGHLIGHTS_QUERY, ], + injections_query: &[], }, // Shell / bash. Lexical highlighting for the shell family; the LSP // half (bash-language-server) was already wired in `lsp.lua`. Unlike @@ -459,6 +906,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["sh", "bash", "zsh", "ksh", "ash", "bats"], loader: || tree_sitter_bash::LANGUAGE.into(), highlights_query: &[tree_sitter_bash::HIGHLIGHT_QUERY], + injections_query: &[], }, // Filename-identified languages. These files usually have no useful // extension (`Dockerfile`, `Makefile`, `CMakeLists.txt`), so the bulk @@ -474,18 +922,21 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["dockerfile", "containerfile"], loader: || tree_sitter_containerfile::LANGUAGE.into(), highlights_query: &[tree_sitter_containerfile::HIGHLIGHTS_QUERY], + injections_query: &[], }, LanguageEntry { name: "make", extensions: &["mk", "make"], loader: || tree_sitter_make::LANGUAGE.into(), highlights_query: &[tree_sitter_make::HIGHLIGHTS_QUERY], + injections_query: &[], }, LanguageEntry { name: "cmake", extensions: &["cmake"], loader: || tree_sitter_cmake::LANGUAGE.into(), highlights_query: &[tree_sitter_cmake::HIGHLIGHTS_QUERY], + injections_query: &[], }, // Grammar-gap languages — these already had LSP configs but no // grammar, so they rendered without lexical color. Each language name @@ -498,12 +949,14 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["py", "pyi"], loader: || tree_sitter_python::LANGUAGE.into(), highlights_query: &[tree_sitter_python::HIGHLIGHTS_QUERY], + injections_query: &[], }, LanguageEntry { name: "go", extensions: &["go"], loader: || tree_sitter_go::LANGUAGE.into(), highlights_query: &[tree_sitter_go::HIGHLIGHTS_QUERY], + injections_query: &[], }, // JavaScript / TypeScript. One `tree-sitter-javascript` grammar parses // both `.js` and `.jsx`; `tree-sitter-typescript` ships two grammars @@ -518,6 +971,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["js", "mjs", "cjs"], loader: || tree_sitter_javascript::LANGUAGE.into(), highlights_query: &[tree_sitter_javascript::HIGHLIGHT_QUERY], + injections_query: &[], }, LanguageEntry { name: "javascriptreact", @@ -527,6 +981,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ tree_sitter_javascript::HIGHLIGHT_QUERY, tree_sitter_javascript::JSX_HIGHLIGHT_QUERY, ], + injections_query: &[], }, LanguageEntry { name: "typescript", @@ -536,6 +991,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ tree_sitter_javascript::HIGHLIGHT_QUERY, tree_sitter_typescript::HIGHLIGHTS_QUERY, ], + injections_query: &[], }, LanguageEntry { name: "typescriptreact", @@ -546,18 +1002,21 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ tree_sitter_javascript::JSX_HIGHLIGHT_QUERY, tree_sitter_typescript::HIGHLIGHTS_QUERY, ], + injections_query: &[], }, LanguageEntry { name: "toml", extensions: &["toml"], loader: || tree_sitter_toml_ng::LANGUAGE.into(), highlights_query: &[tree_sitter_toml_ng::HIGHLIGHTS_QUERY], + injections_query: &[], }, LanguageEntry { name: "zig", extensions: &["zig", "zon"], loader: || tree_sitter_zig::LANGUAGE.into(), highlights_query: &[tree_sitter_zig::HIGHLIGHTS_QUERY], + injections_query: &[], }, ]; @@ -587,6 +1046,11 @@ pub struct SyntaxRegistry { /// compilation failure (e.g. grammar / query ABI skew) is /// cached as `Err(message)` so we don't burn cycles re-trying. queries: RefCell, String>>>, + /// Fence-name → canonical-language alias map (framing Q#IJ4). Seeded + /// with [`default_injection_aliases`]; Lua adds to it through + /// [`Self::register_injection_alias`]. Snapshotted into each + /// [`ParseRequest`] at dispatch so the worker reads a `Send` copy. + injection_aliases: RefCell>, /// Active theme (T M4.3). Shared with every /// [`crate::highlight::SyntaxHighlightView`] attached through /// this registry --- editing the theme through Lua updates all @@ -613,6 +1077,7 @@ impl SyntaxRegistry { parse_jobs: RefCell::new(HashMap::new()), extra_extensions: RefCell::new(HashMap::new()), queries: RefCell::new(HashMap::new()), + injection_aliases: RefCell::new(default_injection_aliases()), theme: Arc::new(Mutex::new(Theme::default_dark())), } } @@ -777,6 +1242,47 @@ impl SyntaxRegistry { .insert(lang_name.to_owned(), compiled); result } + + /// Add or override a fence-name → language alias (framing Q#IJ4). The + /// alias key is case-folded to match the resolver. Called from Lua via + /// `pmacs.parse.injection_aliases`. + pub fn register_injection_alias(&self, alias: impl Into, lang: impl Into) { + self.injection_aliases + .borrow_mut() + .insert(alias.into().to_ascii_lowercase(), lang.into()); + } + + /// A `Send` snapshot of the alias map for a [`ParseRequest`] (Q#IJ4). + #[must_use] + pub fn injection_alias_snapshot(&self) -> Arc> { + Arc::new(self.injection_aliases.borrow().clone()) + } + + /// Stage 2 of the injection handoff (framing Q#IJ2): fill each layer's + /// `highlight_query` from this registry's cache and return the resolved + /// bundle. The worker leaves the queries `None`; this runs on the main + /// thread at settle where the `Rc` query cache lives. Tree clones are + /// cheap (`ts_tree_copy` shares subtrees), so rebuilding the bundle is + /// near-free. + #[must_use] + pub fn resolve_layer_queries(&self, raw: &ParseTreeBundle) -> Arc { + let layers = raw + .layers + .iter() + .map(|l| Layer { + language_name: l.language_name.clone(), + tree: l.tree.clone(), + depth: l.depth, + highlight_query: self.highlights_query(&l.language_name), + }) + .collect(); + Arc::new(ParseTreeBundle { + layers, + source: raw.source.clone(), + language_name: raw.language_name.clone(), + parse_duration: raw.parse_duration, + }) + } } impl Default for SyntaxRegistry { @@ -876,14 +1382,33 @@ pub fn compute_highlight_spans_in_range( query: &tree_sitter::Query, bundle: &ParseTreeBundle, byte_range: Option>, +) -> Vec { + compute_highlight_spans_for( + query, + bundle.root_tree(), + bundle.source.as_ref(), + byte_range, + ) +} + +/// Like [`compute_highlight_spans_in_range`] but over an explicit +/// `(tree, source)` — the per-layer form the producers call for each +/// injection layer (framing Q#IJ7). `source` is the whole buffer; a +/// child layer's tree carries absolute offsets into it, so the same +/// capture walk works unchanged. +#[must_use] +pub fn compute_highlight_spans_for( + query: &tree_sitter::Query, + tree: &tree_sitter::Tree, + source: &[u8], + byte_range: Option>, ) -> Vec { let mut spans = Vec::new(); let mut cursor = tree_sitter::QueryCursor::new(); if let Some(range) = byte_range { cursor.set_byte_range(range); } - let source: &[u8] = bundle.source.as_ref(); - let root = bundle.tree.root_node(); + let root = tree.root_node(); let mut iter = cursor.captures(query, root, source); while let Some((qmatch, capture_idx)) = iter.next() { // Fail-closed on the locals property predicate. The capture @@ -945,6 +1470,283 @@ mod tests { bundle } + /// Parse `src` as `lang` through `reg` with injection layers expanded + /// (worker) and each layer's highlight query resolved (settle) — the + /// full framing Q#IJ2 handoff. Returns the resolved bundle. + fn parse_layered(reg: &SyntaxRegistry, lang: &str, src: &[u8]) -> Arc { + let language = reg.language(lang).expect("grammar loads"); + let mut buf = fresh_buffer("doc"); + buf.apply_edit(EditOp::Insert { pos: 0, bytes: src }) + .unwrap(); + let view = ParseView::new(&buf, language, lang.to_owned()); + let handle = view.handle(); + let _vid = buf.attach_view(Box::new(view)); + let mut req = handle.make_request(); + req.injection_aliases = reg.injection_alias_snapshot(); + let bundle = run_parse(req).expect("parse succeeds"); + reg.resolve_layer_queries(&bundle) + } + + #[test] + fn injection_query_block_compiles() { + // ABI guard (framing acceptance #1): the markdown block + inline + // injection queries must compile against their grammars. A crate + // bump that drifted query and grammar apart surfaces here. + let mut cache = HashMap::new(); + assert!( + injection_query_cached(&mut cache, "markdown").is_some(), + "markdown ships a compilable injection query" + ); + assert!( + injection_query_cached(&mut cache, "markdown_inline").is_some(), + "markdown_inline ships a compilable injection query" + ); + // A language with no injections resolves to None, not an error. + assert!(injection_query_cached(&mut cache, "toml").is_none()); + } + + #[test] + fn markdown_fence_builds_rust_child_layer() { + // Framing acceptance #2. + let reg = SyntaxRegistry::new(); + let src = b"# Title\n\n```rust\nfn demo() { let x = 1; }\n```\n\nText.\n"; + let bundle = parse_layered(®, "markdown", src); + assert!( + bundle.layers.len() >= 2, + "fenced markdown is layered; got {} layer(s)", + bundle.layers.len() + ); + assert_eq!( + bundle.layers[0].language_name, "markdown", + "root is markdown" + ); + let rust = bundle + .layers + .iter() + .find(|l| l.language_name == "rust") + .expect("a rust child layer for the ```rust fence"); + assert_eq!( + rust.tree.root_node().kind(), + "source_file", + "rust root kind" + ); + assert_eq!(rust.depth, 1, "the fence child is at depth 1"); + } + + #[test] + fn child_layer_offsets_are_absolute() { + // Framing acceptance #3 / mechanic #1: a node inside the fence has + // byte offsets absolute into the FULL markdown source. + let reg = SyntaxRegistry::new(); + let prefix = "# Title\n\n```rust\n"; + let src = format!("{prefix}fn demo() {{}}\n```\n"); + let bundle = parse_layered(®, "markdown", src.as_bytes()); + let rust = bundle + .layers + .iter() + .find(|l| l.language_name == "rust") + .expect("rust child layer"); + let fi = rust.tree.root_node().child(0).expect("function_item"); + assert!( + fi.start_byte() >= prefix.len(), + "child offset {} is absolute (>= prefix len {})", + fi.start_byte(), + prefix.len() + ); + let text = &bundle.source[fi.start_byte()..fi.end_byte()]; + assert!( + std::str::from_utf8(text).unwrap().contains("fn demo"), + "absolute offsets index the rust code within the full source" + ); + } + + #[test] + fn dynamic_alias_resolves_and_unknown_skips() { + // Framing acceptance #4: case-folded alias resolution + graceful + // skip of an unknown fence language. + let reg = SyntaxRegistry::new(); + for (fence, lang) in [("py", "python"), ("rs", "rust"), ("JS", "javascript")] { + let src = format!("```{fence}\nvalue\n```\n"); + let bundle = parse_layered(®, "markdown", src.as_bytes()); + assert!( + bundle.layers.iter().any(|l| l.language_name == lang), + "fence ```{fence} resolves to {lang}" + ); + } + let bundle = parse_layered(®, "markdown", b"```nonsense\nvalue\n```\n"); + assert!( + !bundle.layers.iter().any(|l| l.language_name == "nonsense"), + "unknown fence language produces no child layer" + ); + assert_eq!(bundle.layers[0].language_name, "markdown", "root intact"); + assert_eq!(bundle.root_tree().root_node().kind(), "document"); + } + + #[test] + fn registry_alias_override_resolves_via_snapshot() { + // Framing acceptance #5 (Rust-level bridge): a registered alias is + // snapshotted into the request and resolved by the worker. The full + // Lua-async path is covered in `tests/injection_acceptance.rs`. + let reg = SyntaxRegistry::new(); + reg.register_injection_alias("MyLang", "rust"); // case-folded to `mylang` + let bundle = parse_layered(®, "markdown", b"```mylang\nfn f() {}\n```\n"); + assert!( + bundle.layers.iter().any(|l| l.language_name == "rust"), + "a registered alias resolves the fence to its target grammar" + ); + } + + #[test] + fn inline_layer_multi_range_link_and_emphasis() { + // Framing acceptance #6: a paragraph with a link AND emphasis + // becomes a markdown_inline layer whose included ranges exclude the + // block grammar's named children — the inline grammar still parses + // the emphasis run in a surviving text range. + let reg = SyntaxRegistry::new(); + let src = b"See [the docs](http://example.com) and *emphasis* here.\n"; + let bundle = parse_layered(®, "markdown", src); + let inline = bundle + .layers + .iter() + .find(|l| l.language_name == "markdown_inline") + .expect("inline paragraph becomes a markdown_inline layer"); + assert_eq!( + inline.depth, 1, + "inline is a depth-1 injection of the block grammar" + ); + let sexp = inline.tree.root_node().to_sexp(); + assert!( + sexp.contains("emphasis"), + "the inline layer parsed the *emphasis* run: {sexp}" + ); + let query = inline + .highlight_query + .as_ref() + .expect("inline highlights resolved at settle"); + let spans = compute_highlight_spans_for(query, &inline.tree, &bundle.source, None); + assert!( + !spans.is_empty(), + "the inline layer produces highlight spans" + ); + } + + #[test] + fn recursion_bounds_terminate() { + // Framing acceptance #7: rust self-injects into macro token-trees. + // Nested injections must terminate (depth cap + visited guard), + // never loop — if the guard failed this test would hang. + let reg = SyntaxRegistry::new(); + let src = + b"macro_rules! m { () => { println!(\"{}\", vec![1, 2, 3]); }; }\nfn f() { m!(); }\n"; + let bundle = parse_layered(®, "rust", src); + assert!( + bundle.layers.len() <= MAX_INJECTION_LAYERS, + "layer count within the backstop" + ); + let max_depth = bundle.layers.iter().map(|l| l.depth).max().unwrap_or(0); + assert!( + max_depth <= MAX_INJECTION_DEPTH, + "max depth {max_depth} within cap {MAX_INJECTION_DEPTH}" + ); + assert_eq!(bundle.layers[0].language_name, "rust", "root is rust"); + assert!( + bundle.layers.iter().all(|l| l.language_name == "rust"), + "all layers are rust (self-injection)" + ); + } + + #[test] + fn non_injecting_buffer_single_layer() { + // Framing acceptance #13: a plain rust file with no macros produces + // exactly one layer — no behavior change for non-injecting content. + let reg = SyntaxRegistry::new(); + let bundle = parse_layered(®, "rust", b"fn main() { let x = 1; }\n"); + assert_eq!( + bundle.layers.len(), + 1, + "no injections yields the single root layer" + ); + assert_eq!(bundle.layers[0].depth, 0); + } + + #[test] + fn incremental_edit_reflects_in_child_and_new_fence_adds_layer() { + // Framing acceptance #11: editing inside a fence reflects in the + // child layer after reparse; a NEW fence adds a layer. + let reg = SyntaxRegistry::new(); + let language = reg.language("markdown").expect("markdown"); + let mut buf = fresh_buffer("doc"); + buf.apply_edit(EditOp::Insert { + pos: 0, + bytes: b"```rust\nfn a() {}\n```\n", + }) + .unwrap(); + let view = ParseView::new(&buf, language, "markdown".to_owned()); + let handle = view.handle(); + let _vid = buf.attach_view(Box::new(view)); + + let reparse = |handle: &ParseViewHandle| -> Arc { + let mut req = handle.make_request(); + req.injection_aliases = reg.injection_alias_snapshot(); + let bundle = reg.resolve_layer_queries(&run_parse(req).expect("parse")); + handle.install(bundle.clone()); + bundle + }; + let count_fns = |b: &ParseTreeBundle| -> usize { + b.layers + .iter() + .find(|l| l.language_name == "rust") + .map_or(0, |l| { + l.tree + .root_node() + .to_sexp() + .matches("function_item") + .count() + }) + }; + + let b0 = reparse(&handle); + assert_eq!( + b0.layers + .iter() + .filter(|l| l.language_name == "rust") + .count(), + 1, + "initial: one rust fence layer" + ); + assert_eq!(count_fns(&b0), 1, "initial rust child has one function"); + + // Edit inside the fence: add a second function before the closer. + let src = handle.source_snapshot(); + let at = src + .windows(4) + .position(|w| w == b"\n```") + .expect("closing fence"); + buf.apply_edit(EditOp::Insert { + pos: at as u64, + bytes: b"\nfn b() {}", + }) + .unwrap(); + let b1 = reparse(&handle); + assert!( + count_fns(&b1) >= 2, + "an edit inside the fence is reflected in the rust child layer" + ); + + // Append a NEW python fence → a new layer appears. + let end = buf.len(); + buf.apply_edit(EditOp::Insert { + pos: end, + bytes: b"\n```py\nx = 1\n```\n", + }) + .unwrap(); + let b2 = reparse(&handle); + assert!( + b2.layers.iter().any(|l| l.language_name == "python"), + "a newly-added fence adds its child layer" + ); + } + #[test] fn builtin_languages_include_c_and_cpp() { // M_B3 regression guard: a future refactor must not silently @@ -1060,12 +1862,12 @@ mod tests { let _vid = buf.attach_view(Box::new(view)); let bundle = parse_synchronously(&handle); assert_eq!( - bundle.tree.root_node().kind(), + bundle.root_tree().root_node().kind(), "translation_unit", "CUDA grammar (C-derived) roots at translation_unit" ); assert!( - !bundle.tree.root_node().has_error(), + !bundle.root_tree().root_node().has_error(), "CUDA grammar parses the `<<<...>>>` kernel launch without error" ); } @@ -1128,12 +1930,12 @@ mod tests { let _vid = buf.attach_view(Box::new(view)); let bundle = parse_synchronously(&handle); assert_eq!( - bundle.tree.root_node().kind(), + bundle.root_tree().root_node().kind(), "program", "bash grammar roots at `program`" ); assert!( - !bundle.tree.root_node().has_error(), + !bundle.root_tree().root_node().has_error(), "bash grammar parses a representative script without error" ); } @@ -1232,12 +2034,12 @@ mod tests { let _vid = buf.attach_view(Box::new(view)); let bundle = parse_synchronously(&handle); assert_eq!( - bundle.tree.root_node().kind(), + bundle.root_tree().root_node().kind(), *root_kind, "`{lang}` roots at `{root_kind}`" ); assert!( - !bundle.tree.root_node().has_error(), + !bundle.root_tree().root_node().has_error(), "`{lang}` parses its snippet without error" ); } @@ -1319,12 +2121,12 @@ mod tests { let _vid = buf.attach_view(Box::new(view)); let bundle = parse_synchronously(&handle); assert_eq!( - bundle.tree.root_node().kind(), + bundle.root_tree().root_node().kind(), *root_kind, "`{lang}` roots at `{root_kind}`" ); assert!( - !bundle.tree.root_node().has_error(), + !bundle.root_tree().root_node().has_error(), "`{lang}` parses its snippet without error" ); } @@ -1486,7 +2288,7 @@ mod tests { // Cold parse. let bundle = parse_synchronously(&handle); - assert_eq!(bundle.tree.root_node().kind(), "source_file"); + assert_eq!(bundle.root_tree().root_node().kind(), "source_file"); assert_eq!(handle.pending_edit_count(), 0); // One incremental edit, then re-parse with the new source + @@ -1498,7 +2300,7 @@ mod tests { .unwrap(); assert_eq!(handle.pending_edit_count(), 1); let bundle = parse_synchronously(&handle); - assert_eq!(bundle.tree.root_node().kind(), "source_file"); + assert_eq!(bundle.root_tree().root_node().kind(), "source_file"); assert_eq!( bundle.source.as_ref(), b"fn main() { let _ = 1;}\n", diff --git a/tests/injection_acceptance.rs b/tests/injection_acceptance.rs new file mode 100644 index 0000000..a5129e6 --- /dev/null +++ b/tests/injection_acceptance.rs @@ -0,0 +1,165 @@ +// injection_acceptance.rs --- multi-language injection acceptance gates. + +//! Multi-language injection acceptance gates that need the public API +//! surface (the Lua async path and a settle-time perf budget). The +//! layer-structure, alias-resolution, producer, and GPU gates live next to +//! the code they exercise (`syntax.rs` / `semantic_render.rs` / +//! `highlight.rs` / `pmacs-gpu`), where `run_parse`, `scoped_style_spans`, +//! and `source_color_at` are reachable. See the framing acceptance list in +//! `docs/multi-language-injections-framing.md`. + +use std::fmt::Write as _; +use std::time::{Duration, Instant}; + +use pmacs::buffer::{Buffer, BufferId, EditOp}; +use pmacs::editor::EditorState; +use pmacs::lua_bindings::BufferIdLua; +use pmacs::syntax::{self, ParseView, SyntaxRegistry}; + +/// Drive `tick_async` until `predicate` holds or a deadline passes. +fn pump_async bool>(state: &mut EditorState, predicate: F) { + let deadline = Instant::now() + Duration::from_secs(3); + while !predicate(state) { + assert!(Instant::now() < deadline, "async pump deadline exceeded"); + state.tick_async(); + std::thread::sleep(Duration::from_millis(2)); + } +} + +/// Framing acceptance #5 (full Lua async path): an alias added from Lua via +/// `pmacs.parse.injection_aliases` must reach the parse worker through the +/// dispatch snapshot, so an *asynchronously* dispatched injected parse +/// resolves a fence named by the new alias. Testing only the static +/// resolver would leave the Lua write-through + snapshot bridge unproven. +#[test] +fn lua_alias_override_resolves_on_async_parse() { + let mut state = EditorState::new(); + + // Add a bespoke fence alias from Lua (write-through to the registry). + state + .lua_host + .lua() + .load(r#"pmacs.parse.injection_aliases.mydsl = "rust""#) + .exec() + .expect("set injection alias from Lua"); + + // A markdown buffer whose fence uses the new alias. + let src = b"# Doc\n\n```mydsl\nfn injected() { let x = 1; }\n```\n"; + let buf_id = state + .lua_host + .registry() + .borrow_mut() + .create_from_bytes("doc.md".to_owned(), src); + state + .lua_host + .lua() + .globals() + .set("BUF", BufferIdLua(buf_id)) + .expect("bind BUF"); + + // Dispatch asynchronously (the wrapped `_dispatch` records the job; the + // per-tick settle path installs the resolved bundle). + state + .lua_host + .lua() + .load("pmacs.parse._dispatch(BUF, 'markdown')") + .exec() + .expect("async dispatch"); + + pump_async(&mut state, |s| { + s.syntax_registry + .view(buf_id) + .and_then(|h| h.current()) + .is_some() + }); + + let bundle = state + .syntax_registry + .view(buf_id) + .and_then(|h| h.current()) + .expect("settled bundle"); + assert!( + bundle.layers.iter().any(|l| l.language_name == "rust"), + "the Lua-set `mydsl` alias resolved the fence to a rust child layer; \ + layers: {:?}", + bundle + .layers + .iter() + .map(|l| l.language_name.as_str()) + .collect::>() + ); +} + +/// Framing acceptance #12: a large all-inline markdown buffer settles +/// (root + one cold inline layer per paragraph) within a comfortable +/// budget, and the FINAL paragraph still receives an inline layer — the +/// tail is not silently dropped. This is the measured guard that keeps +/// child-incrementality (Q#IJ8) out of v1. +#[test] +fn many_paragraph_settle_under_budget_with_tail_covered() { + let reg = SyntaxRegistry::new(); + let n = 200usize; + let mut src = String::new(); + for i in 0..n { + // A blank line separates paragraphs; each carries emphasis + a link + // so the block grammar injects a markdown_inline layer for it. + writeln!( + src, + "Paragraph {i} with *emphasis* and a [link](http://x).\n" + ) + .expect("write"); + } + // A distinctly-marked final paragraph. + let marker = "FINALPARAGRAPH"; + writeln!(src, "{marker} with *stress*.\n").expect("write"); + + let language = reg.language("markdown").expect("markdown grammar"); + let mut buf = Buffer::new(BufferId::next(), "big.md"); + buf.apply_edit(EditOp::Insert { + pos: 0, + bytes: src.as_bytes(), + }) + .expect("seed markdown"); + let view = ParseView::new(&buf, language, "markdown".to_owned()); + let handle = view.handle(); + let _vid = buf.attach_view(Box::new(view)); + let mut req = handle.make_request(); + req.injection_aliases = reg.injection_alias_snapshot(); + + let start = Instant::now(); + let bundle = syntax::run_parse(req).expect("layered markdown parse"); + let elapsed = start.elapsed(); + + // Catastrophic-regression guard: cold-parsing ~200 tiny inline layers + // is milliseconds of work; a generous ceiling avoids debug/CI flakiness + // while still catching a blow-up (e.g. accidental O(n^2) layer work). + assert!( + elapsed < Duration::from_secs(2), + "many-paragraph settle took {elapsed:?}, exceeds the budget" + ); + + let inline_count = bundle + .layers + .iter() + .filter(|l| l.language_name == "markdown_inline") + .count(); + assert!( + inline_count >= 100, + "most paragraphs produce an inline layer; got {inline_count}" + ); + + // Tail coverage: an inline layer's tree spans the final paragraph. + let marker_off = src.find(marker).expect("marker present"); + let tail_covered = bundle + .layers + .iter() + .filter(|l| l.language_name == "markdown_inline") + .any(|l| { + let r = l.tree.root_node(); + (r.start_byte()..r.end_byte()).contains(&marker_off) + }); + assert!( + tail_covered, + "the final paragraph still receives an inline layer (no tail loss)" + ); +} diff --git a/tests/m4_acceptance.rs b/tests/m4_acceptance.rs index f922253..59dde84 100644 --- a/tests/m4_acceptance.rs +++ b/tests/m4_acceptance.rs @@ -125,9 +125,10 @@ fn m4_1_initial_parse_of_5000_line_file_under_100ms() { language_name: "rust".to_owned(), prior_tree: None, edits: Vec::new(), + injection_aliases: Arc::new(std::collections::HashMap::new()), }; let bundle = syntax::run_parse(req).expect("parse succeeds"); - assert_eq!(bundle.tree.root_node().kind(), "source_file"); + assert_eq!(bundle.root_tree().root_node().kind(), "source_file"); assert!( bundle.parse_duration < Duration::from_millis(100), "5000-line cold parse took {:?}, exceeds 100 ms budget", @@ -204,7 +205,7 @@ fn m4_1_incremental_parse_under_5ms() { let req = handle.make_request(); let bundle = syntax::run_parse(req).expect("incremental parse"); - assert_eq!(bundle.tree.root_node().kind(), "source_file"); + assert_eq!(bundle.root_tree().root_node().kind(), "source_file"); assert!( bundle.parse_duration < Duration::from_millis(5), "incremental parse took {:?}, exceeds 5 ms budget", @@ -226,6 +227,7 @@ fn m4_1_dispatch_parse_round_trips_via_runtime() { language_name: "rust".to_owned(), prior_tree: None, edits: Vec::new(), + injection_aliases: Arc::new(std::collections::HashMap::new()), }; let id = rt.dispatch_parse(req, None); let deadline = Instant::now() + Duration::from_secs(5); @@ -243,7 +245,7 @@ fn m4_1_dispatch_parse_round_trips_via_runtime() { } other => panic!("unexpected outcome: {other:?}"), } - assert_eq!(bundle.tree.root_node().kind(), "source_file"); + assert_eq!(bundle.root_tree().root_node().kind(), "source_file"); assert_eq!(bundle.language_name, "rust"); } @@ -542,6 +544,7 @@ fn m4_3_open_rust_file_highlights_under_100ms() { language_name: "rust".to_owned(), prior_tree: None, edits: Vec::new(), + injection_aliases: Arc::new(std::collections::HashMap::new()), }; let bundle = syntax::run_parse(req).expect("parse succeeds"); let spans = syntax::compute_highlight_spans(&query, &bundle);