diff --git a/builtin/runtime/syntax.lua b/builtin/runtime/syntax.lua index b639377..6ed7d86 100644 --- a/builtin/runtime/syntax.lua +++ b/builtin/runtime/syntax.lua @@ -44,6 +44,20 @@ function pmacs.parse._dispatch(buf, lang) return job_id end +-- Injection language aliases (framing Q#IJ4). The registry holds the +-- merged map (seeded with defaults on the Rust side), and each dispatch +-- snapshots it into the parse request so the worker can resolve dynamic +-- fence names (`py` → python, `ts` → typescript). Exposed as a +-- write-through proxy so users add fence-name aliases from init.lua: +-- pmacs.parse.injection_aliases.mylang = "rust" +-- Reads are not proxied (the canonical map lives Rust-side); this is a +-- write-only extension surface. +pmacs.parse.injection_aliases = setmetatable({}, { + __newindex = function(_, alias, lang) + pmacs.parse._register_injection_alias(alias, lang) + end, +}) + -- Shebang → language detection ------------------------------------------ -- -- Extension detection (`language_for_path`) misses extensionless scripts diff --git a/pmacs-gpu/src/main.rs b/pmacs-gpu/src/main.rs index 79567e4..a932e68 100644 --- a/pmacs-gpu/src/main.rs +++ b/pmacs-gpu/src/main.rs @@ -6732,12 +6732,24 @@ fn adornment_text_color(fg: CellColor) -> glyphon::Color { } fn source_color_at(byte: u64, spans: &[StyleSpan]) -> Option { - for sp in spans { - if sp.range.start <= byte && byte < sp.range.end { - return cell_color_to_glyphon(sp.style.fg); + // Fold every covering span in order, matching the semantic-client + // `effective_style_at` contract (last covering span with a non-default + // fg wins) rather than returning the first. The daemon flattens + // injection layers into disjoint spans (framing Q#IJ6), so usually at + // most one covers a byte — but where spans do overlap (a styled + // markdown parent under an injected child), the topmost color must win, + // never the outermost. Spans arrive start-sorted, so a narrower nested + // span (later start) folds after and overrides its enclosing span. + let mut color = None; + for sp in spans + .iter() + .filter(|sp| sp.range.start <= byte && byte < sp.range.end) + { + if let Some(c) = cell_color_to_glyphon(sp.style.fg) { + color = Some(c); } } - None + color } /// Convert a `pmacs-protocol::cell::Color` to a `glyphon::Color`. @@ -8969,4 +8981,40 @@ mod tests { "the peer cursor line must keep the CurrentLine constant" ); } + + #[test] + fn source_color_folds_overlapping_child_over_parent() { + // Framing acceptance #9 (GPU consumer): a styled markdown parent + // span [0,10) (red) with an injected child span [3,6) (green) on top. + // `replace_style_spans` sorts incoming spans by start, so we mirror + // that here; `source_color_at` must FOLD the covering spans (child + // green wins at a shared byte), not return the first (parent) span. + // Bite: the pre-fix first-covering-span code returns red at byte 4. + let red = style_with_fg(CellColor::Rgb(200, 0, 0)); + let green = style_with_fg(CellColor::Rgb(0, 200, 0)); + let mut spans = vec![ + StyleSpan { + range: ByteRange { start: 0, end: 10 }, + style: red, + }, + StyleSpan { + range: ByteRange { start: 3, end: 6 }, + style: green, + }, + ]; + spans.sort_by_key(|s| s.range.start); // as replace_style_spans does + + // Byte 4 is covered by both: the child (green) wins the fold. + assert_eq!( + source_color_at(4, &spans), + Some(glyphon::Color::rgb(0, 200, 0)), + "the injected child color wins over the parent at a shared byte" + ); + // Byte 1 is parent-only: stays red. + assert_eq!( + source_color_at(1, &spans), + Some(glyphon::Color::rgb(200, 0, 0)), + "a parent-only byte keeps the parent color" + ); + } } diff --git a/src/async_runtime.rs b/src/async_runtime.rs index 94ae001..6bc8292 100644 --- a/src/async_runtime.rs +++ b/src/async_runtime.rs @@ -2700,6 +2700,7 @@ mod tests { language_name: "rust".to_owned(), prior_tree: None, edits: Vec::new(), + injection_aliases: Arc::new(std::collections::HashMap::new()), }; let id = rt.dispatch_parse(req, None); pump_until(&rt, || rt.is_complete(id)); @@ -2713,7 +2714,7 @@ mod tests { other => panic!("unexpected outcome: {other:?}"), } assert_eq!(bundle.language_name, "rust"); - assert_eq!(bundle.tree.root_node().kind(), "source_file"); + assert_eq!(bundle.root_tree().root_node().kind(), "source_file"); // take_parse_tree was already drained, so handoff is empty. assert_eq!(rt.parse_handoff_len(), 0); } @@ -2729,6 +2730,7 @@ mod tests { language_name: "rust".to_owned(), prior_tree: None, edits: Vec::new(), + injection_aliases: Arc::new(std::collections::HashMap::new()), }; let id = rt.dispatch_parse(req, None); pump_until(&rt, || rt.is_complete(id)); diff --git a/src/highlight.rs b/src/highlight.rs index 8b10981..a74176a 100644 --- a/src/highlight.rs +++ b/src/highlight.rs @@ -39,7 +39,7 @@ use crate::buffer::Buffer; use crate::cell::{CellCoord, CellGrid, Color, Style, UnderlineStyle}; use crate::lsp::SharedLspManager; use crate::overlay::merge_styles; -use crate::syntax::{HighlightSpan, ParseTreeBundle, ParseViewHandle, compute_highlight_spans}; +use crate::syntax::{HighlightSpan, ParseTreeBundle, ParseViewHandle, compute_highlight_spans_for}; use crate::view::{View, Viewport}; // --------------------------------------------------------------------------- @@ -248,6 +248,19 @@ pub type ThemeHandle = Arc>; // SyntaxHighlightView // --------------------------------------------------------------------------- +/// Per-layer cached highlight spans + capture names (framing Q#IJ7). +/// One entry per bundle layer that has a highlight query; kept in the +/// bundle's depth order so the render paints shallow-to-deep. +struct LayerSpans { + /// Spans for this layer, sorted wider-first per + /// [`compute_highlight_spans_for`]. + spans: Vec, + /// Capture names indexed by `HighlightSpan::capture_index` for this + /// layer's own query. Layers may use different grammars, so this is + /// per-layer, not shared. + capture_names: Arc<[String]>, +} + /// Cached per-bundle highlight state. Keyed by the underlying /// `Arc`'s identity (compared via `Arc::ptr_eq`), /// so a freshly-installed bundle invalidates the cache. @@ -255,25 +268,19 @@ struct HighlightCache { /// The bundle the cache was built against. `None` until the /// first render observes a settled bundle. bundle: Option>, - /// Compiled spans, sorted wider-first per - /// [`compute_highlight_spans`]. - spans: Vec, + /// Per-layer spans in depth-ascending order (framing Q#IJ6). + layers: Vec, /// Per-row first-byte offsets into `bundle.source`. `Vec` /// because pmacs files cap at 4 GiB. line_offsets: Vec, - /// Capture names indexed by `HighlightSpan::capture_index`. - /// Populated alongside `spans` so the render path doesn't need - /// to keep a reference into the [`tree_sitter::Query`]. - capture_names: Arc<[String]>, } impl HighlightCache { fn empty() -> Self { Self { bundle: None, - spans: Vec::new(), + layers: Vec::new(), line_offsets: Vec::new(), - capture_names: Arc::from(Vec::::new().into_boxed_slice()), } } } @@ -283,39 +290,41 @@ impl HighlightCache { const TAB_WIDTH: u32 = 8; /// View that renders syntax highlighting from a tree-sitter parse -/// tree. Composes over [`crate::text_view::TextView`] per the M2.9 -/// view-composition contract --- it never writes glyphs, only merges -/// styles into cells the base view has already painted. +/// tree, including its injection layers. Composes over +/// [`crate::text_view::TextView`] per the M2.9 view-composition +/// contract --- it never writes glyphs, only merges styles into cells +/// the base view has already painted. pub struct SyntaxHighlightView { parse: ParseViewHandle, - query: Arc, theme: ThemeHandle, cache: HighlightCache, } impl SyntaxHighlightView { - /// Construct a highlight view over `parse` using `query` and - /// `theme`. The initial cache is empty; the first render with - /// a settled bundle populates it. + /// Construct a highlight view over `parse` using `theme`. Per-layer + /// highlight queries come from each `Layer::highlight_query` in the + /// bundle (resolved at settle, framing Q#IJ2), so no query is passed + /// here. The initial cache is empty; the first render with a settled + /// bundle populates it. #[must_use] - pub fn new(parse: ParseViewHandle, query: Arc, theme: ThemeHandle) -> Self { + pub fn new(parse: ParseViewHandle, theme: ThemeHandle) -> Self { Self { parse, - query, theme, cache: HighlightCache::empty(), } } - /// Test helper: number of cached highlight spans. + /// Test helper: total cached highlight spans across all layers. #[must_use] pub fn cached_span_count(&self) -> usize { - self.cache.spans.len() + self.cache.layers.iter().map(|l| l.spans.len()).sum() } /// Refresh `self.cache` if the parse view's current bundle /// differs from the cached one. No-op when the bundle pointer - /// is unchanged --- the steady-state cost between parses. + /// is unchanged --- the steady-state cost between parses. Rebuilds + /// spans for every layer that carries a highlight query. fn refresh_cache_if_stale(&mut self) { let Some(bundle) = self.parse.current() else { return; @@ -328,26 +337,39 @@ impl SyntaxHighlightView { if !stale { return; } - let spans = compute_highlight_spans(&self.query, &bundle); - let line_offsets = compute_line_offsets(bundle.source.as_ref()); - let capture_names: Arc<[String]> = self - .query - .capture_names() - .iter() - .map(|s| (*s).to_owned()) - .collect(); + let source = bundle.source.as_ref(); + let mut layers = Vec::new(); + for layer in &bundle.layers { + let Some(query) = layer.highlight_query.as_ref() else { + continue; + }; + let spans = compute_highlight_spans_for(query, &layer.tree, source, None); + if spans.is_empty() { + continue; + } + let capture_names: Arc<[String]> = query + .capture_names() + .iter() + .map(|s| (*s).to_owned()) + .collect(); + layers.push(LayerSpans { + spans, + capture_names, + }); + } + let line_offsets = compute_line_offsets(source); self.cache = HighlightCache { bundle: Some(bundle), - spans, + layers, line_offsets, - capture_names, }; } - /// Look up the style for span `s`, consulting the active theme. - fn style_for(&self, theme: &Theme, s: HighlightSpan) -> Style { + /// Look up the style for span `s` within `layer`, consulting the + /// active theme. + fn style_for(theme: &Theme, layer: &LayerSpans, s: HighlightSpan) -> Style { let idx = s.capture_index as usize; - let Some(name) = self.cache.capture_names.get(idx) else { + let Some(name) = layer.capture_names.get(idx) else { return theme.default_style; }; theme.lookup(name) @@ -364,7 +386,7 @@ impl View for SyntaxHighlightView { let Some(bundle) = self.cache.bundle.clone() else { return; }; - if self.cache.spans.is_empty() || self.cache.line_offsets.is_empty() { + if self.cache.layers.is_empty() || self.cache.line_offsets.is_empty() { return; } let source: &[u8] = bundle.source.as_ref(); @@ -376,61 +398,64 @@ impl View for SyntaxHighlightView { let cell_origin = viewport.cell_origin; let total_lines = self.cache.line_offsets.len() as u32; - for row_offset in 0..max_rows { - let line_idx = start_line + row_offset; - if line_idx >= total_lines { - break; - } - let line_start = self.cache.line_offsets[line_idx as usize]; - let line_end = self - .cache - .line_offsets - .get(line_idx as usize + 1) - .copied() - .unwrap_or(source.len() as u32); - // Trim a single trailing newline if any --- the text - // view doesn't paint it as a glyph either. - let line_end_no_nl = if line_end > line_start - && source.get(line_end as usize - 1).copied() == Some(b'\n') - { - line_end - 1 - } else { - line_end - }; - let line_bytes = &source[line_start as usize..line_end_no_nl as usize]; + // Paint layers shallow-to-deep (framing Q#IJ6): a deeper layer + // merges on top, so an injected child's styling wins within its + // region. Within a layer, wider-first ordering (from + // `compute_highlight_spans_for`) lets narrower captures override. + for layer in &self.cache.layers { + for row_offset in 0..max_rows { + let line_idx = start_line + row_offset; + if line_idx >= total_lines { + break; + } + let line_start = self.cache.line_offsets[line_idx as usize]; + let line_end = self + .cache + .line_offsets + .get(line_idx as usize + 1) + .copied() + .unwrap_or(source.len() as u32); + // Trim a single trailing newline if any --- the text + // view doesn't paint it as a glyph either. + let line_end_no_nl = if line_end > line_start + && source.get(line_end as usize - 1).copied() == Some(b'\n') + { + line_end - 1 + } else { + line_end + }; + let line_bytes = &source[line_start as usize..line_end_no_nl as usize]; - // Spans whose start lies on this line. Wider-first - // ordering means parents apply before children. - // Spans that *cross* lines apply on every line they - // touch --- this loop only filters by start, then - // intersects with the line range below. - for span in self - .cache - .spans - .iter() - .filter(|s| s.start_byte < line_end_no_nl && s.end_byte > line_start) - .copied() - { - let style = self.style_for(&theme, span); - if style == Style::default() { - // Nothing to merge --- skip the per-cell loop. - continue; - } - let s_start = span.start_byte.max(line_start); - let s_end = span.end_byte.min(line_end_no_nl); - let byte_col_start = (s_start - line_start) as usize; - let byte_col_end = (s_end - line_start) as usize; - let (start_col, end_col) = - byte_range_to_display_cols(line_bytes, byte_col_start, byte_col_end); - if end_col <= start_col { - continue; - } - let cell_row = cell_origin.row + row_offset; - let clamped_start = start_col.min(max_cols); - let clamped_end = end_col.min(max_cols); - for col in clamped_start..clamped_end { - let cell = cells.at(CellCoord::new(cell_row, cell_origin.col + col)); - cell.style = merge_styles(cell.style, style); + // Spans whose start lies on this line. Spans that *cross* + // lines apply on every line they touch --- this loop filters + // by intersection, then clips to the line range below. + for span in layer + .spans + .iter() + .filter(|s| s.start_byte < line_end_no_nl && s.end_byte > line_start) + .copied() + { + let style = Self::style_for(&theme, layer, span); + if style == Style::default() { + // Nothing to merge --- skip the per-cell loop. + continue; + } + let s_start = span.start_byte.max(line_start); + let s_end = span.end_byte.min(line_end_no_nl); + let byte_col_start = (s_start - line_start) as usize; + let byte_col_end = (s_end - line_start) as usize; + let (start_col, end_col) = + byte_range_to_display_cols(line_bytes, byte_col_start, byte_col_end); + if end_col <= start_col { + continue; + } + let cell_row = cell_origin.row + row_offset; + let clamped_start = start_col.min(max_cols); + let clamped_end = end_col.min(max_cols); + for col in clamped_start..clamped_end { + let cell = cells.at(CellCoord::new(cell_row, cell_origin.col + col)); + cell.style = merge_styles(cell.style, style); + } } } } @@ -1228,4 +1253,56 @@ mod tests { ); assert!(!c.style.bold); } + + #[test] + fn grid_paints_injected_child_keyword() { + // Framing acceptance #10: the grid `SyntaxHighlightView` paints a + // rust keyword cell INSIDE a markdown ```rust fence — styling only + // the injected rust layer can produce. Layers paint shallow-to-deep + // (framing Q#IJ6), so the rust keyword wins over the markdown root. + use crate::buffer::{Buffer, BufferId, EditOp}; + use crate::cell::{Cell, CellSize}; + use crate::syntax::{ParseView, SyntaxRegistry}; + + let reg = SyntaxRegistry::new(); + let language = reg.language("markdown").expect("markdown grammar"); + // Line 0: ```rust ; line 1: fn demo() {} ; line 2: ``` + let src = b"```rust\nfn demo() {}\n```\n"; + let mut buf = Buffer::new(BufferId::next(), "doc.md"); + buf.apply_edit(EditOp::Insert { pos: 0, bytes: src }) + .unwrap(); + let view = ParseView::new(&buf, language, "markdown".to_owned()); + let handle = view.handle(); + let _vid = buf.attach_view(Box::new(view)); + let mut req = handle.make_request(); + req.injection_aliases = reg.injection_alias_snapshot(); + let bundle = crate::syntax::run_parse(req).expect("markdown parse"); + handle.install(reg.resolve_layer_queries(&bundle)); + + let mut hv = SyntaxHighlightView::new(handle, reg.theme()); + let cols = 40usize; + let rows = 3usize; + let mut backing: Vec = vec![Cell::default(); rows * cols]; + let mut grid = CellGrid { + cells: &mut backing, + stride: cols as u32, + size: CellSize::new(rows as u32, cols as u32), + }; + let viewport = Viewport { + buffer_start: 0, + buffer_end: u64::MAX, + cell_origin: CellCoord::new(0, 0), + cell_size: CellSize::new(rows as u32, cols as u32), + gutter_w: 0, + }; + let registry = buf; // keep buf alive + hv.render(®istry, viewport, &mut grid); + + // `fn` is at line 1, cols 0..2. default_dark styles `keyword` bold. + let c = grid.get(CellCoord::new(1, 0)); + assert!( + c.style.bold, + "the injected rust `fn` keyword is painted (bold) inside the fence" + ); + } } diff --git a/src/lua_bindings/mod.rs b/src/lua_bindings/mod.rs index b153ffd..487d098 100644 --- a/src/lua_bindings/mod.rs +++ b/src/lua_bindings/mod.rs @@ -6362,7 +6362,7 @@ pub struct ParseNodeLua { impl ParseNodeLua { fn resolve(&self) -> Option> { - let mut node = self.bundle.tree.root_node(); + let mut node = self.bundle.root_tree().root_node(); for &idx in &self.path { node = node.child(idx)?; } @@ -6395,7 +6395,9 @@ impl UserData for ParseTreeLua { methods.add_method("text", |lua, this, ()| { lua.create_string(this.0.source.as_ref()) }); - methods.add_method("sexp", |_, this, ()| Ok(this.0.tree.root_node().to_sexp())); + methods.add_method("sexp", |_, this, ()| { + Ok(this.0.root_tree().root_node().to_sexp()) + }); } } @@ -6617,6 +6619,21 @@ pub fn install_parse( )?; } + // Injection alias write-through (framing Q#IJ4). `syntax.lua` wraps + // this in a `pmacs.parse.injection_aliases` proxy table so users write + // `pmacs.parse.injection_aliases.mylang = "rust"`. The registry holds + // the merged map (defaults + overrides); each dispatch snapshots it. + { + let s = syntax.clone(); + parse_mod.set( + "_register_injection_alias", + lua.create_function(move |_, (alias, lang): (String, String)| { + s.register_injection_alias(alias, lang); + Ok(()) + })?, + )?; + } + { let s = syntax.clone(); parse_mod.set( @@ -6658,7 +6675,10 @@ pub fn install_parse( let handle = get_or_create_parse_view(&s, ®, id.0, &lang)?; let req = handle.make_request(); let bundle = syntax::run_parse(req).map_err(mlua::Error::external)?; - let arc = Arc::new(bundle); + // Resolve each layer's highlight query from the registry + // cache before install so producers can style every layer + // (framing Q#IJ2 stage 2). + let arc = s.resolve_layer_queries(&bundle); handle.install(arc.clone()); Ok(ParseTreeLua(arc)) })?, @@ -6676,7 +6696,10 @@ pub fn install_parse( "_dispatch", lua.create_function(move |_, (id, lang): (BufferIdLua, String)| { let handle = get_or_create_parse_view(&s, ®, id.0, &lang)?; - let req = handle.make_request(); + let mut req = handle.make_request(); + // Snapshot the alias map into the request so the worker can + // resolve dynamic fence names off the main thread (Q#IJ4). + req.injection_aliases = s.injection_alias_snapshot(); let job_id = rt.dispatch_parse(req, None); s.record_parse_job(job_id, id.0); Ok(job_id) @@ -6707,7 +6730,10 @@ pub fn install_parse( return Ok(false); }; if let Some(handle) = s.view(buf_id) { - handle.install(bundle); + // Stage 2 (framing Q#IJ2): resolve each layer's highlight + // query on the main thread before install. + let resolved = s.resolve_layer_queries(&bundle); + handle.install(resolved); Ok(true) } else { Ok(false) @@ -6737,12 +6763,14 @@ pub fn install_parse( id.0 ))); }; - let Some(query) = s.highlights_query(&lang) else { - // No highlights query for this language --- treat as - // a benign no-op so callers don't need to special-case - // grammars without highlights bundled. + if s.highlights_query(&lang).is_none() { + // Root language ships no highlights --- treat as a benign + // no-op so callers don't need to special-case grammars + // without highlights bundled. (Injected child layers still + // resolve their own queries at settle; a root-highlight-less + // injector is out of v1 scope.) return Ok(false); - }; + } let theme = s.theme(); let core = lua .app_data_ref::() @@ -6755,7 +6783,7 @@ pub fn install_parse( id.0 ))); } - let overlay = SyntaxHighlightView::new(handle, query, theme); + let overlay = SyntaxHighlightView::new(handle, theme); win.push_overlay(Box::new(overlay)); Ok(true) })?, diff --git a/src/semantic_render.rs b/src/semantic_render.rs index cf64e7a..f7132a4 100644 --- a/src/semantic_render.rs +++ b/src/semantic_render.rs @@ -1743,12 +1743,6 @@ fn scoped_style_spans(state: &EditorState, vp: &DeclaredViewport) -> Vec Vec = Vec::new(); + for layer in &bundle.layers { + let Some(query) = layer.highlight_query.as_ref() else { continue; }; - let style = theme.lookup(name); + let names = query.capture_names(); + let highlights = crate::syntax::compute_highlight_spans_for( + query, + &layer.tree, + source, + Some(vis_start as usize..vis_end as usize), + ); + for (order, hs) in highlights.iter().enumerate() { + let s = u64::from(hs.start_byte).max(vis_start); + let e = u64::from(hs.end_byte).min(vis_end); + if e <= s { + continue; + } + let Some(name) = names.get(hs.capture_index as usize) else { + continue; + }; + let style = theme.lookup(name); + if style == Style::default() { + continue; + } + styled.push(StyledLayerSpan { + start: s, + end: e, + style, + priority: (layer.depth, order as u32), + }); + } + } + flatten_layer_spans(&styled) +} + +/// One styled span from a single injection layer, tagged with a priority +/// used to resolve overlaps: `(depth, order)`, higher wins. `order` is the +/// index in the layer's wider-first list, so a narrower capture (later) +/// overrides a wider one within the same layer. +struct StyledLayerSpan { + start: u64, + end: u64, + style: Style, + priority: (u16, u32), +} + +/// Flatten possibly-overlapping per-layer styled spans into **disjoint** +/// `StyleSpan`s whose per-byte style is the priority-ordered fold of every +/// covering span (framing Q#IJ6). Emitting disjoint spans makes the result +/// robust to the GPU wire re-sorting spans by start (`replace_style_spans` +/// / `merge_style_spans`), which would otherwise destroy producer order. +/// A boundary sweep over the (viewport-bounded) span endpoints; adjacent +/// equal-style runs are merged for wire economy. +fn flatten_layer_spans(styled: &[StyledLayerSpan]) -> Vec { + if styled.is_empty() { + return Vec::new(); + } + let mut bounds: Vec = Vec::with_capacity(styled.len() * 2); + for sp in styled { + bounds.push(sp.start); + bounds.push(sp.end); + } + bounds.sort_unstable(); + bounds.dedup(); + + let mut out: Vec = Vec::new(); + for win in bounds.windows(2) { + let (a, b) = (win[0], win[1]); + if b <= a { + continue; + } + // Fold every span covering [a, b) in ascending priority so a + // deeper/narrower span overrides, while a span that only sets a + // non-color attribute still composes (matches the semantic-client + // `effective_style_at` contract). + let mut covering: Vec<&StyledLayerSpan> = styled + .iter() + .filter(|sp| sp.start <= a && sp.end >= b) + .collect(); + if covering.is_empty() { + continue; + } + covering.sort_by_key(|sp| sp.priority); + let mut style = Style::default(); + for sp in covering { + style = crate::overlay::merge_styles(style, sp.style); + } if style == Style::default() { - continue; // Nothing to render — skip the wire byte. + continue; + } + if let Some(last) = out.last_mut() + && last.range.end == a + && last.style == style + { + last.range.end = b; + continue; } out.push(StyleSpan { - range: ByteRange { start: s, end: e }, + range: ByteRange { start: a, end: b }, style, }); } @@ -3300,7 +3376,9 @@ mod tests { let handle = parse_view.handle(); let req = handle.make_request(); let bundle = crate::syntax::run_parse(req).expect("initial rust parse"); - handle.install(std::sync::Arc::new(bundle)); + // Mirror the production settle path: resolve each layer's highlight + // query before install so the producer can style it (framing Q#IJ2). + handle.install(state.syntax_registry.resolve_layer_queries(&bundle)); buf.attach_view(Box::new(parse_view)); drop(registry); core.set_buffer_path(buffer_id, Some(std::path::PathBuf::from("/tmp/x.rs"))); @@ -3309,6 +3387,90 @@ mod tests { handle } + fn seed_markdown_parse_view( + state: &EditorState, + buffer_id: BufferId, + text: &[u8], + ) -> crate::syntax::ParseViewHandle { + let language = state + .syntax_registry + .language("markdown") + .expect("markdown grammar"); + let mut core = state.core.borrow_mut(); + let registry_handle = core.registry.clone(); + let mut registry = registry_handle.borrow_mut(); + let buf = registry.get_mut(buffer_id).expect("active buffer"); + if !text.is_empty() { + buf.apply_edit(crate::buffer::EditOp::Insert { + pos: 0, + bytes: text, + }) + .expect("seed markdown text"); + } + let parse_view = crate::syntax::ParseView::new(buf, language, "markdown".to_owned()); + let handle = parse_view.handle(); + let mut req = handle.make_request(); + req.injection_aliases = state.syntax_registry.injection_alias_snapshot(); + let bundle = crate::syntax::run_parse(req).expect("markdown parse"); + handle.install(state.syntax_registry.resolve_layer_queries(&bundle)); + buf.attach_view(Box::new(parse_view)); + drop(registry); + core.set_buffer_path(buffer_id, Some(std::path::PathBuf::from("/tmp/x.md"))); + drop(core); + state.syntax_registry.attach_view(buffer_id, handle.clone()); + handle + } + + #[test] + fn wire_producer_emits_disjoint_child_spans_in_fence() { + // Framing acceptance #8: `scoped_style_spans` over a ```rust fence + // emits a styled span on the `fn` keyword INSIDE the fence — which + // only the injected rust layer can produce (the markdown root has + // no keyword styling there). The pre-injection single-layer producer + // fails this. Emitted spans are disjoint (framing Q#IJ6 flatten). + let state = empty_state(); + let bid = active_buffer(&state); + let src = b"# T\n\n```rust\nfn demo() {}\n```\n"; + seed_markdown_parse_view(&state, bid, src); + let vp = DeclaredViewport { + buffer_id: bid, + visible: ByteRange { + start: 0, + end: src.len() as u64, + }, + frontend_generation: 0, + }; + let spans = scoped_style_spans(&state, &vp); + assert!(!spans.is_empty(), "layered producer emits style spans"); + + // Disjoint (flattened): no two spans overlap. + let mut sorted = spans.clone(); + sorted.sort_by_key(|s| s.range.start); + for w in sorted.windows(2) { + assert!( + w[0].range.end <= w[1].range.start, + "flattened spans must be disjoint: {:?} then {:?}", + w[0].range, + w[1].range + ); + } + + // The `fn` keyword inside the fence carries a non-default style. + let fn_off = src + .windows(2) + .position(|w| w == b"fn") + .expect("`fn` present in source") as u64; + let covering = spans + .iter() + .find(|s| s.range.start <= fn_off && fn_off < s.range.end) + .expect("a span covers the `fn` keyword inside the fence"); + assert_ne!( + covering.style, + Style::default(), + "the injected rust keyword is styled" + ); + } + #[test] fn cpp_style_comes_from_lsp_when_no_tree_sitter_grammar() { let state = empty_state(); @@ -3402,7 +3564,7 @@ mod tests { ); let bundle = crate::syntax::run_parse(req).expect("settled rust parse"); - handle.install(std::sync::Arc::new(bundle)); + handle.install(state.syntax_registry.resolve_layer_queries(&bundle)); assert_eq!(state.syntax_registry.take_parse_job(9001), Some(bid)); let settled = s.render_frame(&state); assert!( diff --git a/src/syntax.rs b/src/syntax.rs index 33577e0..ef01159 100644 --- a/src/syntax.rs +++ b/src/syntax.rs @@ -32,12 +32,12 @@ //! tests. use std::cell::RefCell; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::rc::Rc; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; -use tree_sitter::StreamingIterator; +use tree_sitter::{Node, Point, Range, StreamingIterator}; use crate::async_runtime::JobId; use crate::buffer::{Buffer, BufferError, BufferId}; @@ -71,6 +71,14 @@ pub struct ParseRequest { /// produced. Empty for cold parses; non-empty drives incremental /// re-parse. pub edits: Vec, + /// Snapshot of the injection alias map (framing Q#IJ4). The worker + /// resolves a dynamic `@injection.language` fence-name (`py`, `ts`, + /// `c++`) through this map — case-folded — before matching it against + /// [`BUILTIN_LANGUAGES`]. Snapshotted from the registry at dispatch so + /// the worker never touches the main-thread `Rc` registry or a Lua + /// table. Empty for the non-layered/legacy callers (no injections + /// resolve, root parse unaffected). + pub injection_aliases: Arc>, } /// Output of [`run_parse`]. The runtime's parse-handoff side map @@ -78,22 +86,58 @@ pub struct ParseRequest { /// resolves a buffer id to its current bundle and walks the tree. #[derive(Debug)] pub struct ParseTreeBundle { - /// The freshly-produced parse tree. - pub tree: tree_sitter::Tree, - /// Source bytes the tree was parsed against. Co-owned with the - /// request so node-byte-range lookups can read the underlying + /// Injection layers (framing Q#IJ1). `layers[0]` is the root layer + /// (the whole buffer, parsed with the buffer's own grammar); + /// subsequent entries are injected child layers in depth-ascending + /// order. Always non-empty — a parse produces at least the root, so + /// [`Self::root_tree`] never panics. + pub layers: Vec, + /// Source bytes every layer's tree was parsed against. Co-owned with + /// the request so node-byte-range lookups can read the underlying /// text (T M4.1 acceptance: "parse tree introspectable via Lua" - /// implies the source the tree references). + /// implies the source the tree references). Child layers parse the + /// *same* full source via `set_included_ranges`, so their node + /// offsets are absolute into these bytes (framing mechanic #1). pub source: Arc<[u8]>, - /// Language label. Stored alongside the tree so Lua can ask - /// "what grammar produced this?". + /// Root language label (`layers[0].language_name`). Kept here so Lua + /// and the `*workers*` buffer can ask "what grammar produced this?" + /// without indexing the layer vec. pub language_name: String, - /// Wall-clock duration of the parse itself (excludes dispatch + /// Wall-clock duration of the whole layered parse (excludes dispatch /// queueing, source materialization, and bus delivery). T M4.1 /// acceptance criteria are stated in this metric. pub parse_duration: Duration, } +/// One injection layer within a [`ParseTreeBundle`] (framing Q#IJ1). A +/// layer pairs a parse tree with the language that produced it and the +/// injection-nesting depth (root = 0). `highlight_query` is resolved on +/// the main thread at settle from the registry cache (framing Q#IJ2) — +/// the worker leaves it `None`. +#[derive(Debug)] +pub struct Layer { + /// Canonical language name of the grammar that produced `tree`. + pub language_name: String, + /// The layer's parse tree. Node offsets are absolute into the + /// bundle's `source` (child layers use `set_included_ranges`). + pub tree: tree_sitter::Tree, + /// Injection depth: 0 for the root, 1 for a direct injection, etc. + pub depth: u16, + /// Compiled `highlights.scm` for `language_name`, resolved at settle. + /// `None` when the language ships no highlights, or on the worker + /// (pre-settle). Producers read it to style this layer. + pub highlight_query: Option>, +} + +impl ParseTreeBundle { + /// The root layer's tree (`layers[0]`) — the whole-buffer parse. + /// Never panics: [`run_parse`] always seeds the root layer. + #[must_use] + pub fn root_tree(&self) -> &tree_sitter::Tree { + &self.layers[0].tree + } +} + /// Run a parse. This is the worker-side body that the runtime's /// `dispatch_parse` closure invokes after pulling a job from the /// queue. Always synchronous --- there is no internal yielding. @@ -115,18 +159,391 @@ pub fn run_parse(req: ParseRequest) -> Result { } } let started = Instant::now(); - let tree = parser + let root_tree = parser .parse(req.source.as_ref(), prior.as_ref()) .ok_or_else(|| "parser produced no tree".to_owned())?; + // `parse_duration` measures the root parse only — the metric the M4.1 + // acceptance gates are stated in. Injection layer building (below) is an + // additive phase separately guarded by the settle-time budget test; it + // must not retroactively inflate this metric. let parse_duration = started.elapsed(); + + // Seed the root layer, then expand injection layers (framing Q#IJ1). + // Injection expansion is best-effort and isolated to the child + // (Q#IJ3): a failed/unknown/over-budget child drops that child only — + // the root always installs, so this returns `Ok` whenever the root + // parsed. + let mut layers = vec![Layer { + language_name: req.language_name.clone(), + tree: root_tree, + depth: 0, + highlight_query: None, + }]; + build_injection_layers(&mut layers, req.source.as_ref(), &req.injection_aliases); Ok(ParseTreeBundle { - tree, + layers, source: req.source, language_name: req.language_name, parse_duration, }) } +// --------------------------------------------------------------------------- +// Injection layers (framing Q#IJ2 -- Q#IJ5). Worker-side: this runs on a +// parse worker, so it touches no `Rc` registry and no Lua — it resolves +// injected languages by indexing the `&'static BUILTIN_LANGUAGES` table +// (loaders + `injections_query` sources are `Send`) and case-folds fence +// names through the `ParseRequest`'s alias snapshot. +// --------------------------------------------------------------------------- + +/// Max injection nesting depth (framing Q#IJ3). markdown→rust is depth 1. +const MAX_INJECTION_DEPTH: u16 = 3; +/// Runaway backstop on total layers per buffer (framing Q#IJ3) — set well +/// above any real document (a markdown doc's one-inline-layer-per-paragraph +/// sits far under this). Purely anti-runaway; the perf bound is the +/// settle-time acceptance guard, not this number. If hit, tail layers are +/// dropped (degraded highlighting on a pathological file only). +const MAX_INJECTION_LAYERS: usize = 4096; + +/// The default fence-name → canonical-language alias map (framing Q#IJ4). +/// Keys are lowercase; the resolver case-folds before lookup. Seeded into +/// the registry and snapshotted into each [`ParseRequest`]; also handy for +/// tests that build a request without the registry. +#[must_use] +pub fn default_injection_aliases() -> HashMap { + [ + ("js", "javascript"), + ("jsx", "javascriptreact"), + ("ts", "typescript"), + ("tsx", "typescriptreact"), + ("py", "python"), + ("py3", "python"), + ("python3", "python"), + ("rs", "rust"), + ("sh", "bash"), + ("shell", "bash"), + ("shellscript", "bash"), + ("zsh", "bash"), + ("c++", "cpp"), + ("cxx", "cpp"), + ("cc", "cpp"), + ("golang", "go"), + ("yml", "yaml"), + ("md", "markdown"), + ] + .into_iter() + .map(|(a, b)| (a.to_owned(), b.to_owned())) + .collect() +} + +/// One injection region resolved from a parent layer's `injections.scm`. +/// Ranges are already child-excluded and normalized (Q#IJ5) but not yet +/// intersected with the parent layer's ranges (that happens per-parent in +/// [`build_injection_layers`], which knows the parent's included ranges). +struct InjectionMatch { + /// Raw language name — dynamic capture text or a static `#set!` value. + language: String, + /// Child-excluded content ranges for this match, sorted/non-overlapping. + ranges: Vec, +} + +/// Expand injection layers under the already-parsed root (`layers[0]`), +/// appending children in depth-ascending order (Q#IJ1, Q#IJ6 rely on this +/// ordering). Bounded by depth, total layer count, and a +/// `(language, ranges)` visited guard (Q#IJ3). BFS by depth so siblings +/// at a level are grouped before descending. +fn build_injection_layers( + layers: &mut Vec, + source: &[u8], + aliases: &HashMap, +) { + let mut query_cache: HashMap>> = HashMap::new(); + let mut visited: HashSet<(String, Vec<(usize, usize)>)> = HashSet::new(); + // Frontier entries are (layer index, that layer's included ranges). + let mut frontier: Vec<(usize, Vec)> = vec![(0, vec![whole_source_range(source)])]; + let mut depth: u16 = 0; + + while depth < MAX_INJECTION_DEPTH && !frontier.is_empty() { + // Children discovered this level: (layer, its ranges) to append and + // (if any injections themselves) descend into next level. + let mut children: Vec<(Layer, Vec)> = Vec::new(); + 'parents: for (parent_idx, parent_ranges) in &frontier { + let parent_lang = layers[*parent_idx].language_name.clone(); + let Some(query) = injection_query_cached(&mut query_cache, &parent_lang) else { + continue; + }; + for m in collect_injection_matches(&query, &layers[*parent_idx].tree, source) { + if layers.len() + children.len() >= MAX_INJECTION_LAYERS { + break 'parents; // runaway backstop; tail dropped + } + let Some(child_lang) = resolve_injected_language(&m.language, aliases) else { + continue; // unknown/unaliased language — skip this child only + }; + let mut ranges = intersect_ranges(&m.ranges, parent_ranges, source); + normalize_ranges(&mut ranges); + if ranges.is_empty() { + continue; + } + let key = (child_lang.to_owned(), ranges_key(&ranges)); + if !visited.insert(key) { + continue; // same (language, ranges) already parsed — cycle guard + } + let Some(tree) = parse_child(child_lang, &ranges, source) else { + continue; // child parse failed — skip this child only + }; + children.push(( + Layer { + language_name: child_lang.to_owned(), + tree, + depth: depth + 1, + highlight_query: None, + }, + ranges, + )); + } + } + if children.is_empty() { + break; + } + let mut next_frontier = Vec::with_capacity(children.len()); + for (layer, ranges) in children { + let idx = layers.len(); + layers.push(layer); + next_frontier.push((idx, ranges)); + } + frontier = next_frontier; + depth += 1; + } +} + +/// Compile (once, cached) the `injections.scm` for `lang` from the static +/// [`BUILTIN_LANGUAGES`] table, or `None` if the language ships none. +fn injection_query_cached( + cache: &mut HashMap>>, + lang: &str, +) -> Option> { + if let Some(slot) = cache.get(lang) { + return slot.clone(); + } + let compiled = BUILTIN_LANGUAGES + .iter() + .find(|e| e.name == lang) + .and_then(|entry| { + let source = entry.injections_query.join("\n"); + if source.trim().is_empty() { + return None; + } + let language = (entry.loader)(); + tree_sitter::Query::new(&language, &source) + .ok() + .map(Arc::new) + }); + cache.insert(lang.to_owned(), compiled.clone()); + compiled +} + +/// Run `query` over `tree` and return each injection region: its raw +/// language name (dynamic `@injection.language` node text, or static +/// `#set! injection.language`) and its child-excluded content ranges. +fn collect_injection_matches( + query: &tree_sitter::Query, + tree: &tree_sitter::Tree, + source: &[u8], +) -> Vec { + let names = query.capture_names(); + let content_cap = names.iter().position(|n| *n == "injection.content"); + let Some(content_cap) = content_cap.map(|i| i as u32) else { + return Vec::new(); + }; + let lang_cap = names + .iter() + .position(|n| *n == "injection.language") + .map(|i| i as u32); + + let mut out = Vec::new(); + let mut cursor = tree_sitter::QueryCursor::new(); + let mut it = cursor.matches(query, tree.root_node(), source); + while let Some(m) = it.next() { + // Static language + include-children from `#set!` property settings. + let mut static_lang: Option = None; + let mut include_children = false; + for prop in query.property_settings(m.pattern_index) { + match &*prop.key { + "injection.language" => { + static_lang = prop.value.as_deref().map(str::to_owned); + } + "injection.include-children" => include_children = true, + _ => {} + } + } + let mut dyn_lang: Option = None; + let mut ranges: Vec = Vec::new(); + for cap in m.captures { + if Some(cap.index) == lang_cap { + if let Ok(text) = cap.node.utf8_text(source) { + dyn_lang = Some(text.to_owned()); + } + } else if cap.index == content_cap { + ranges.extend(content_node_ranges(cap.node, include_children)); + } + } + let Some(language) = static_lang.or(dyn_lang) else { + continue; + }; + normalize_ranges(&mut ranges); + if ranges.is_empty() { + continue; + } + out.push(InjectionMatch { language, ranges }); + } + out +} + +/// The included ranges for one `@injection.content` node (framing Q#IJ5 / +/// mechanic #3). With `include_children`, the whole node span; otherwise +/// the node's extent minus its **named** children's ranges. Anonymous +/// token children are *kept* — they are the injected text itself, not +/// structure to exclude. (This matches `tree-sitter-md`'s own inline +/// splitter, `bindings/rust/parser.rs:410`, which filters on `is_named()`: +/// excluding a block `inline` node's anonymous text tokens would shred the +/// paragraph into unparseable fragments. Our real injection sites — a +/// childless `code_fence_content`, an `inline` with only anonymous +/// children, an `include-children` macro `token_tree` — all resolve +/// correctly under this rule.) A node with no named children yields its +/// whole span. +fn content_node_ranges(node: Node, include_children: bool) -> Vec { + if include_children { + return vec![node.range()]; + } + let mut ranges = Vec::new(); + let mut start_byte = node.start_byte(); + let mut start_point = node.start_position(); + let mut cursor = node.walk(); + if cursor.goto_first_child() { + loop { + let child = cursor.node(); + if child.is_named() { + if child.start_byte() > start_byte { + ranges.push(Range { + start_byte, + end_byte: child.start_byte(), + start_point, + end_point: child.start_position(), + }); + } + start_byte = child.end_byte(); + start_point = child.end_position(); + } + if !cursor.goto_next_sibling() { + break; + } + } + } + if node.end_byte() > start_byte { + ranges.push(Range { + start_byte, + end_byte: node.end_byte(), + start_point, + end_point: node.end_position(), + }); + } + ranges +} + +/// Clip `candidate` ranges to `parent` ranges (framing Q#IJ5): a nested +/// injection cannot reintroduce bytes its parent excluded. Points are +/// recomputed only for a clipped edge (unclipped edges keep the node's +/// exact point). At depth 1 the parent is the whole buffer, so this is a +/// pass-through. +fn intersect_ranges(candidate: &[Range], parent: &[Range], source: &[u8]) -> Vec { + let mut out = Vec::new(); + for c in candidate { + for p in parent { + let start = c.start_byte.max(p.start_byte); + let end = c.end_byte.min(p.end_byte); + if end > start { + out.push(Range { + start_byte: start, + end_byte: end, + start_point: if start == c.start_byte { + c.start_point + } else { + byte_to_point(source, start) + }, + end_point: if end == c.end_byte { + c.end_point + } else { + byte_to_point(source, end) + }, + }); + } + } + } + out +} + +/// Sort, drop empty, and merge overlapping ranges so the result satisfies +/// `set_included_ranges`' sorted/non-overlapping/non-empty contract. +fn normalize_ranges(ranges: &mut Vec) { + ranges.retain(|r| r.end_byte > r.start_byte); + ranges.sort_by_key(|r| r.start_byte); + let mut merged: Vec = Vec::with_capacity(ranges.len()); + for r in ranges.drain(..) { + if let Some(last) = merged.last_mut() + && r.start_byte < last.end_byte + { + if r.end_byte > last.end_byte { + last.end_byte = r.end_byte; + last.end_point = r.end_point; + } + continue; + } + merged.push(r); + } + *ranges = merged; +} + +/// A hashable identity for a range set (framing Q#IJ3 visited guard). +fn ranges_key(ranges: &[Range]) -> Vec<(usize, usize)> { + ranges.iter().map(|r| (r.start_byte, r.end_byte)).collect() +} + +/// Case-fold `raw`, apply the alias map, then resolve against the bundled +/// table (framing Q#IJ4). Returns the canonical `&'static` name, or `None` +/// for an unknown language. +fn resolve_injected_language(raw: &str, aliases: &HashMap) -> Option<&'static str> { + let lower = raw.trim().to_ascii_lowercase(); + if lower.is_empty() { + return None; + } + let candidate: &str = aliases.get(&lower).map_or(lower.as_str(), String::as_str); + BUILTIN_LANGUAGES + .iter() + .find(|e| e.name == candidate) + .map(|e| e.name) +} + +/// Cold-parse `source` restricted to `ranges` with `lang`'s grammar. Node +/// offsets in the returned tree are absolute into `source` (mechanic #1). +fn parse_child(lang: &str, ranges: &[Range], source: &[u8]) -> Option { + let entry = BUILTIN_LANGUAGES.iter().find(|e| e.name == lang)?; + let language = (entry.loader)(); + let mut parser = tree_sitter::Parser::new(); + parser.set_language(&language).ok()?; + parser.set_included_ranges(ranges).ok()?; + parser.parse(source, None) +} + +/// The whole-buffer range, the root layer's parent range. +fn whole_source_range(source: &[u8]) -> Range { + Range { + start_byte: 0, + end_byte: source.len(), + start_point: Point::new(0, 0), + end_point: byte_to_point(source, source.len()), + } +} + /// Convert a byte offset within `source` to a tree-sitter /// `(row, column)` [`tree_sitter::Point`]. `byte` is clamped to /// `source.len()`. @@ -285,13 +702,17 @@ impl ParseViewHandle { pub fn make_request(&self) -> ParseRequest { let mut inner = self.inner.lock().expect("ParseView mutex poisoned"); let edits = std::mem::take(&mut inner.pending); - let prior_tree = inner.current.as_ref().map(|b| b.tree.clone()); + let prior_tree = inner.current.as_ref().map(|b| b.root_tree().clone()); ParseRequest { source: Arc::from(inner.source.clone()), language: inner.language.clone(), language_name: inner.language_name.clone(), prior_tree, edits, + // Empty by default; the dispatch binding overrides with the + // registry's alias snapshot (framing Q#IJ4). Callers that need + // injections and bypass the registry set this themselves. + injection_aliases: Arc::new(HashMap::new()), } } @@ -339,6 +760,13 @@ pub struct LanguageEntry { /// slice (or all-empty fragments) means no highlights: the view /// runs but emits nothing. pub highlights_query: &'static [&'static str], + /// Bundled `injections.scm` fragments (framing Q#IJ2), joined with a + /// newline and compiled on the parse worker to find embedded-language + /// regions. Empty for the many grammars that ship none (or don't + /// inject). Names are inconsistent across crates — markdown exposes + /// `INJECTION_QUERY_BLOCK`, rust `INJECTIONS_QUERY`, most none — the + /// same shape `highlights_query` already absorbs. + pub injections_query: &'static [&'static str], } /// Bundled grammars (T M4.2 + M4.3). The order is significant only @@ -360,12 +788,14 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["rs"], loader: || tree_sitter_rust::LANGUAGE.into(), highlights_query: &[tree_sitter_rust::HIGHLIGHTS_QUERY], + injections_query: &[tree_sitter_rust::INJECTIONS_QUERY], }, LanguageEntry { name: "lua", extensions: &["lua"], loader: || tree_sitter_lua::LANGUAGE.into(), highlights_query: &[tree_sitter_lua::HIGHLIGHTS_QUERY], + injections_query: &[], }, // T M9.7: markdown grammar for prompt-result buffers with // `_meta.format = "markdown"`. Uses only the block grammar @@ -385,6 +815,20 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["md", "markdown"], loader: || tree_sitter_md::LANGUAGE.into(), highlights_query: &[tree_sitter_md::HIGHLIGHT_QUERY_BLOCK], + injections_query: &[tree_sitter_md::INJECTION_QUERY_BLOCK], + }, + // markdown_inline (framing Q#IJ10) — the inline grammar the block + // grammar injects for paragraph/heading text (`#set! injection.language + // "markdown_inline"`). No file extension: it is injection-only, never + // opened directly by name. Ships an inline highlights query (emphasis, + // links, code spans) and its own injections (e.g. inline HTML), so it + // recurses like any other layer. Retires the M9.7 block-only floor. + LanguageEntry { + name: "markdown_inline", + extensions: &[], + loader: || tree_sitter_md::INLINE_LANGUAGE.into(), + highlights_query: &[tree_sitter_md::HIGHLIGHT_QUERY_INLINE], + injections_query: &[tree_sitter_md::INJECTION_QUERY_INLINE], }, // T M_B3 — C / C++. Lexical highlighting (keywords / strings / // operators) so the grid TUI shows code-shaped C++ on first open. @@ -406,12 +850,14 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["c", "h"], loader: || tree_sitter_c::LANGUAGE.into(), highlights_query: &[tree_sitter_c::HIGHLIGHT_QUERY], + injections_query: &[], }, LanguageEntry { name: "cpp", extensions: &["cpp", "cc", "cxx", "hpp", "hh", "hxx", "ipp", "inl", "cppm"], loader: || tree_sitter_cpp::LANGUAGE.into(), highlights_query: &[tree_sitter_cpp::HIGHLIGHT_QUERY], + injections_query: &[], }, // CUDA (`.cu` source, `.cuh` header). A dedicated grammar rather // than reusing `cpp`: CUDA extends C++ with `__global__`/`__device__` @@ -442,6 +888,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ tree_sitter_cpp::HIGHLIGHT_QUERY, tree_sitter_cuda::HIGHLIGHTS_QUERY, ], + injections_query: &[], }, // Shell / bash. Lexical highlighting for the shell family; the LSP // half (bash-language-server) was already wired in `lsp.lua`. Unlike @@ -459,6 +906,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["sh", "bash", "zsh", "ksh", "ash", "bats"], loader: || tree_sitter_bash::LANGUAGE.into(), highlights_query: &[tree_sitter_bash::HIGHLIGHT_QUERY], + injections_query: &[], }, // Filename-identified languages. These files usually have no useful // extension (`Dockerfile`, `Makefile`, `CMakeLists.txt`), so the bulk @@ -474,18 +922,21 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["dockerfile", "containerfile"], loader: || tree_sitter_containerfile::LANGUAGE.into(), highlights_query: &[tree_sitter_containerfile::HIGHLIGHTS_QUERY], + injections_query: &[], }, LanguageEntry { name: "make", extensions: &["mk", "make"], loader: || tree_sitter_make::LANGUAGE.into(), highlights_query: &[tree_sitter_make::HIGHLIGHTS_QUERY], + injections_query: &[], }, LanguageEntry { name: "cmake", extensions: &["cmake"], loader: || tree_sitter_cmake::LANGUAGE.into(), highlights_query: &[tree_sitter_cmake::HIGHLIGHTS_QUERY], + injections_query: &[], }, // Grammar-gap languages — these already had LSP configs but no // grammar, so they rendered without lexical color. Each language name @@ -498,12 +949,14 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["py", "pyi"], loader: || tree_sitter_python::LANGUAGE.into(), highlights_query: &[tree_sitter_python::HIGHLIGHTS_QUERY], + injections_query: &[], }, LanguageEntry { name: "go", extensions: &["go"], loader: || tree_sitter_go::LANGUAGE.into(), highlights_query: &[tree_sitter_go::HIGHLIGHTS_QUERY], + injections_query: &[], }, // JavaScript / TypeScript. One `tree-sitter-javascript` grammar parses // both `.js` and `.jsx`; `tree-sitter-typescript` ships two grammars @@ -518,6 +971,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["js", "mjs", "cjs"], loader: || tree_sitter_javascript::LANGUAGE.into(), highlights_query: &[tree_sitter_javascript::HIGHLIGHT_QUERY], + injections_query: &[], }, LanguageEntry { name: "javascriptreact", @@ -527,6 +981,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ tree_sitter_javascript::HIGHLIGHT_QUERY, tree_sitter_javascript::JSX_HIGHLIGHT_QUERY, ], + injections_query: &[], }, LanguageEntry { name: "typescript", @@ -536,6 +991,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ tree_sitter_javascript::HIGHLIGHT_QUERY, tree_sitter_typescript::HIGHLIGHTS_QUERY, ], + injections_query: &[], }, LanguageEntry { name: "typescriptreact", @@ -546,18 +1002,21 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ tree_sitter_javascript::JSX_HIGHLIGHT_QUERY, tree_sitter_typescript::HIGHLIGHTS_QUERY, ], + injections_query: &[], }, LanguageEntry { name: "toml", extensions: &["toml"], loader: || tree_sitter_toml_ng::LANGUAGE.into(), highlights_query: &[tree_sitter_toml_ng::HIGHLIGHTS_QUERY], + injections_query: &[], }, LanguageEntry { name: "zig", extensions: &["zig", "zon"], loader: || tree_sitter_zig::LANGUAGE.into(), highlights_query: &[tree_sitter_zig::HIGHLIGHTS_QUERY], + injections_query: &[], }, ]; @@ -587,6 +1046,11 @@ pub struct SyntaxRegistry { /// compilation failure (e.g. grammar / query ABI skew) is /// cached as `Err(message)` so we don't burn cycles re-trying. queries: RefCell, String>>>, + /// Fence-name → canonical-language alias map (framing Q#IJ4). Seeded + /// with [`default_injection_aliases`]; Lua adds to it through + /// [`Self::register_injection_alias`]. Snapshotted into each + /// [`ParseRequest`] at dispatch so the worker reads a `Send` copy. + injection_aliases: RefCell>, /// Active theme (T M4.3). Shared with every /// [`crate::highlight::SyntaxHighlightView`] attached through /// this registry --- editing the theme through Lua updates all @@ -613,6 +1077,7 @@ impl SyntaxRegistry { parse_jobs: RefCell::new(HashMap::new()), extra_extensions: RefCell::new(HashMap::new()), queries: RefCell::new(HashMap::new()), + injection_aliases: RefCell::new(default_injection_aliases()), theme: Arc::new(Mutex::new(Theme::default_dark())), } } @@ -777,6 +1242,47 @@ impl SyntaxRegistry { .insert(lang_name.to_owned(), compiled); result } + + /// Add or override a fence-name → language alias (framing Q#IJ4). The + /// alias key is case-folded to match the resolver. Called from Lua via + /// `pmacs.parse.injection_aliases`. + pub fn register_injection_alias(&self, alias: impl Into, lang: impl Into) { + self.injection_aliases + .borrow_mut() + .insert(alias.into().to_ascii_lowercase(), lang.into()); + } + + /// A `Send` snapshot of the alias map for a [`ParseRequest`] (Q#IJ4). + #[must_use] + pub fn injection_alias_snapshot(&self) -> Arc> { + Arc::new(self.injection_aliases.borrow().clone()) + } + + /// Stage 2 of the injection handoff (framing Q#IJ2): fill each layer's + /// `highlight_query` from this registry's cache and return the resolved + /// bundle. The worker leaves the queries `None`; this runs on the main + /// thread at settle where the `Rc` query cache lives. Tree clones are + /// cheap (`ts_tree_copy` shares subtrees), so rebuilding the bundle is + /// near-free. + #[must_use] + pub fn resolve_layer_queries(&self, raw: &ParseTreeBundle) -> Arc { + let layers = raw + .layers + .iter() + .map(|l| Layer { + language_name: l.language_name.clone(), + tree: l.tree.clone(), + depth: l.depth, + highlight_query: self.highlights_query(&l.language_name), + }) + .collect(); + Arc::new(ParseTreeBundle { + layers, + source: raw.source.clone(), + language_name: raw.language_name.clone(), + parse_duration: raw.parse_duration, + }) + } } impl Default for SyntaxRegistry { @@ -876,14 +1382,33 @@ pub fn compute_highlight_spans_in_range( query: &tree_sitter::Query, bundle: &ParseTreeBundle, byte_range: Option>, +) -> Vec { + compute_highlight_spans_for( + query, + bundle.root_tree(), + bundle.source.as_ref(), + byte_range, + ) +} + +/// Like [`compute_highlight_spans_in_range`] but over an explicit +/// `(tree, source)` — the per-layer form the producers call for each +/// injection layer (framing Q#IJ7). `source` is the whole buffer; a +/// child layer's tree carries absolute offsets into it, so the same +/// capture walk works unchanged. +#[must_use] +pub fn compute_highlight_spans_for( + query: &tree_sitter::Query, + tree: &tree_sitter::Tree, + source: &[u8], + byte_range: Option>, ) -> Vec { let mut spans = Vec::new(); let mut cursor = tree_sitter::QueryCursor::new(); if let Some(range) = byte_range { cursor.set_byte_range(range); } - let source: &[u8] = bundle.source.as_ref(); - let root = bundle.tree.root_node(); + let root = tree.root_node(); let mut iter = cursor.captures(query, root, source); while let Some((qmatch, capture_idx)) = iter.next() { // Fail-closed on the locals property predicate. The capture @@ -945,6 +1470,283 @@ mod tests { bundle } + /// Parse `src` as `lang` through `reg` with injection layers expanded + /// (worker) and each layer's highlight query resolved (settle) — the + /// full framing Q#IJ2 handoff. Returns the resolved bundle. + fn parse_layered(reg: &SyntaxRegistry, lang: &str, src: &[u8]) -> Arc { + let language = reg.language(lang).expect("grammar loads"); + let mut buf = fresh_buffer("doc"); + buf.apply_edit(EditOp::Insert { pos: 0, bytes: src }) + .unwrap(); + let view = ParseView::new(&buf, language, lang.to_owned()); + let handle = view.handle(); + let _vid = buf.attach_view(Box::new(view)); + let mut req = handle.make_request(); + req.injection_aliases = reg.injection_alias_snapshot(); + let bundle = run_parse(req).expect("parse succeeds"); + reg.resolve_layer_queries(&bundle) + } + + #[test] + fn injection_query_block_compiles() { + // ABI guard (framing acceptance #1): the markdown block + inline + // injection queries must compile against their grammars. A crate + // bump that drifted query and grammar apart surfaces here. + let mut cache = HashMap::new(); + assert!( + injection_query_cached(&mut cache, "markdown").is_some(), + "markdown ships a compilable injection query" + ); + assert!( + injection_query_cached(&mut cache, "markdown_inline").is_some(), + "markdown_inline ships a compilable injection query" + ); + // A language with no injections resolves to None, not an error. + assert!(injection_query_cached(&mut cache, "toml").is_none()); + } + + #[test] + fn markdown_fence_builds_rust_child_layer() { + // Framing acceptance #2. + let reg = SyntaxRegistry::new(); + let src = b"# Title\n\n```rust\nfn demo() { let x = 1; }\n```\n\nText.\n"; + let bundle = parse_layered(®, "markdown", src); + assert!( + bundle.layers.len() >= 2, + "fenced markdown is layered; got {} layer(s)", + bundle.layers.len() + ); + assert_eq!( + bundle.layers[0].language_name, "markdown", + "root is markdown" + ); + let rust = bundle + .layers + .iter() + .find(|l| l.language_name == "rust") + .expect("a rust child layer for the ```rust fence"); + assert_eq!( + rust.tree.root_node().kind(), + "source_file", + "rust root kind" + ); + assert_eq!(rust.depth, 1, "the fence child is at depth 1"); + } + + #[test] + fn child_layer_offsets_are_absolute() { + // Framing acceptance #3 / mechanic #1: a node inside the fence has + // byte offsets absolute into the FULL markdown source. + let reg = SyntaxRegistry::new(); + let prefix = "# Title\n\n```rust\n"; + let src = format!("{prefix}fn demo() {{}}\n```\n"); + let bundle = parse_layered(®, "markdown", src.as_bytes()); + let rust = bundle + .layers + .iter() + .find(|l| l.language_name == "rust") + .expect("rust child layer"); + let fi = rust.tree.root_node().child(0).expect("function_item"); + assert!( + fi.start_byte() >= prefix.len(), + "child offset {} is absolute (>= prefix len {})", + fi.start_byte(), + prefix.len() + ); + let text = &bundle.source[fi.start_byte()..fi.end_byte()]; + assert!( + std::str::from_utf8(text).unwrap().contains("fn demo"), + "absolute offsets index the rust code within the full source" + ); + } + + #[test] + fn dynamic_alias_resolves_and_unknown_skips() { + // Framing acceptance #4: case-folded alias resolution + graceful + // skip of an unknown fence language. + let reg = SyntaxRegistry::new(); + for (fence, lang) in [("py", "python"), ("rs", "rust"), ("JS", "javascript")] { + let src = format!("```{fence}\nvalue\n```\n"); + let bundle = parse_layered(®, "markdown", src.as_bytes()); + assert!( + bundle.layers.iter().any(|l| l.language_name == lang), + "fence ```{fence} resolves to {lang}" + ); + } + let bundle = parse_layered(®, "markdown", b"```nonsense\nvalue\n```\n"); + assert!( + !bundle.layers.iter().any(|l| l.language_name == "nonsense"), + "unknown fence language produces no child layer" + ); + assert_eq!(bundle.layers[0].language_name, "markdown", "root intact"); + assert_eq!(bundle.root_tree().root_node().kind(), "document"); + } + + #[test] + fn registry_alias_override_resolves_via_snapshot() { + // Framing acceptance #5 (Rust-level bridge): a registered alias is + // snapshotted into the request and resolved by the worker. The full + // Lua-async path is covered in `tests/injection_acceptance.rs`. + let reg = SyntaxRegistry::new(); + reg.register_injection_alias("MyLang", "rust"); // case-folded to `mylang` + let bundle = parse_layered(®, "markdown", b"```mylang\nfn f() {}\n```\n"); + assert!( + bundle.layers.iter().any(|l| l.language_name == "rust"), + "a registered alias resolves the fence to its target grammar" + ); + } + + #[test] + fn inline_layer_multi_range_link_and_emphasis() { + // Framing acceptance #6: a paragraph with a link AND emphasis + // becomes a markdown_inline layer whose included ranges exclude the + // block grammar's named children — the inline grammar still parses + // the emphasis run in a surviving text range. + let reg = SyntaxRegistry::new(); + let src = b"See [the docs](http://example.com) and *emphasis* here.\n"; + let bundle = parse_layered(®, "markdown", src); + let inline = bundle + .layers + .iter() + .find(|l| l.language_name == "markdown_inline") + .expect("inline paragraph becomes a markdown_inline layer"); + assert_eq!( + inline.depth, 1, + "inline is a depth-1 injection of the block grammar" + ); + let sexp = inline.tree.root_node().to_sexp(); + assert!( + sexp.contains("emphasis"), + "the inline layer parsed the *emphasis* run: {sexp}" + ); + let query = inline + .highlight_query + .as_ref() + .expect("inline highlights resolved at settle"); + let spans = compute_highlight_spans_for(query, &inline.tree, &bundle.source, None); + assert!( + !spans.is_empty(), + "the inline layer produces highlight spans" + ); + } + + #[test] + fn recursion_bounds_terminate() { + // Framing acceptance #7: rust self-injects into macro token-trees. + // Nested injections must terminate (depth cap + visited guard), + // never loop — if the guard failed this test would hang. + let reg = SyntaxRegistry::new(); + let src = + b"macro_rules! m { () => { println!(\"{}\", vec![1, 2, 3]); }; }\nfn f() { m!(); }\n"; + let bundle = parse_layered(®, "rust", src); + assert!( + bundle.layers.len() <= MAX_INJECTION_LAYERS, + "layer count within the backstop" + ); + let max_depth = bundle.layers.iter().map(|l| l.depth).max().unwrap_or(0); + assert!( + max_depth <= MAX_INJECTION_DEPTH, + "max depth {max_depth} within cap {MAX_INJECTION_DEPTH}" + ); + assert_eq!(bundle.layers[0].language_name, "rust", "root is rust"); + assert!( + bundle.layers.iter().all(|l| l.language_name == "rust"), + "all layers are rust (self-injection)" + ); + } + + #[test] + fn non_injecting_buffer_single_layer() { + // Framing acceptance #13: a plain rust file with no macros produces + // exactly one layer — no behavior change for non-injecting content. + let reg = SyntaxRegistry::new(); + let bundle = parse_layered(®, "rust", b"fn main() { let x = 1; }\n"); + assert_eq!( + bundle.layers.len(), + 1, + "no injections yields the single root layer" + ); + assert_eq!(bundle.layers[0].depth, 0); + } + + #[test] + fn incremental_edit_reflects_in_child_and_new_fence_adds_layer() { + // Framing acceptance #11: editing inside a fence reflects in the + // child layer after reparse; a NEW fence adds a layer. + let reg = SyntaxRegistry::new(); + let language = reg.language("markdown").expect("markdown"); + let mut buf = fresh_buffer("doc"); + buf.apply_edit(EditOp::Insert { + pos: 0, + bytes: b"```rust\nfn a() {}\n```\n", + }) + .unwrap(); + let view = ParseView::new(&buf, language, "markdown".to_owned()); + let handle = view.handle(); + let _vid = buf.attach_view(Box::new(view)); + + let reparse = |handle: &ParseViewHandle| -> Arc { + let mut req = handle.make_request(); + req.injection_aliases = reg.injection_alias_snapshot(); + let bundle = reg.resolve_layer_queries(&run_parse(req).expect("parse")); + handle.install(bundle.clone()); + bundle + }; + let count_fns = |b: &ParseTreeBundle| -> usize { + b.layers + .iter() + .find(|l| l.language_name == "rust") + .map_or(0, |l| { + l.tree + .root_node() + .to_sexp() + .matches("function_item") + .count() + }) + }; + + let b0 = reparse(&handle); + assert_eq!( + b0.layers + .iter() + .filter(|l| l.language_name == "rust") + .count(), + 1, + "initial: one rust fence layer" + ); + assert_eq!(count_fns(&b0), 1, "initial rust child has one function"); + + // Edit inside the fence: add a second function before the closer. + let src = handle.source_snapshot(); + let at = src + .windows(4) + .position(|w| w == b"\n```") + .expect("closing fence"); + buf.apply_edit(EditOp::Insert { + pos: at as u64, + bytes: b"\nfn b() {}", + }) + .unwrap(); + let b1 = reparse(&handle); + assert!( + count_fns(&b1) >= 2, + "an edit inside the fence is reflected in the rust child layer" + ); + + // Append a NEW python fence → a new layer appears. + let end = buf.len(); + buf.apply_edit(EditOp::Insert { + pos: end, + bytes: b"\n```py\nx = 1\n```\n", + }) + .unwrap(); + let b2 = reparse(&handle); + assert!( + b2.layers.iter().any(|l| l.language_name == "python"), + "a newly-added fence adds its child layer" + ); + } + #[test] fn builtin_languages_include_c_and_cpp() { // M_B3 regression guard: a future refactor must not silently @@ -1060,12 +1862,12 @@ mod tests { let _vid = buf.attach_view(Box::new(view)); let bundle = parse_synchronously(&handle); assert_eq!( - bundle.tree.root_node().kind(), + bundle.root_tree().root_node().kind(), "translation_unit", "CUDA grammar (C-derived) roots at translation_unit" ); assert!( - !bundle.tree.root_node().has_error(), + !bundle.root_tree().root_node().has_error(), "CUDA grammar parses the `<<<...>>>` kernel launch without error" ); } @@ -1128,12 +1930,12 @@ mod tests { let _vid = buf.attach_view(Box::new(view)); let bundle = parse_synchronously(&handle); assert_eq!( - bundle.tree.root_node().kind(), + bundle.root_tree().root_node().kind(), "program", "bash grammar roots at `program`" ); assert!( - !bundle.tree.root_node().has_error(), + !bundle.root_tree().root_node().has_error(), "bash grammar parses a representative script without error" ); } @@ -1232,12 +2034,12 @@ mod tests { let _vid = buf.attach_view(Box::new(view)); let bundle = parse_synchronously(&handle); assert_eq!( - bundle.tree.root_node().kind(), + bundle.root_tree().root_node().kind(), *root_kind, "`{lang}` roots at `{root_kind}`" ); assert!( - !bundle.tree.root_node().has_error(), + !bundle.root_tree().root_node().has_error(), "`{lang}` parses its snippet without error" ); } @@ -1319,12 +2121,12 @@ mod tests { let _vid = buf.attach_view(Box::new(view)); let bundle = parse_synchronously(&handle); assert_eq!( - bundle.tree.root_node().kind(), + bundle.root_tree().root_node().kind(), *root_kind, "`{lang}` roots at `{root_kind}`" ); assert!( - !bundle.tree.root_node().has_error(), + !bundle.root_tree().root_node().has_error(), "`{lang}` parses its snippet without error" ); } @@ -1486,7 +2288,7 @@ mod tests { // Cold parse. let bundle = parse_synchronously(&handle); - assert_eq!(bundle.tree.root_node().kind(), "source_file"); + assert_eq!(bundle.root_tree().root_node().kind(), "source_file"); assert_eq!(handle.pending_edit_count(), 0); // One incremental edit, then re-parse with the new source + @@ -1498,7 +2300,7 @@ mod tests { .unwrap(); assert_eq!(handle.pending_edit_count(), 1); let bundle = parse_synchronously(&handle); - assert_eq!(bundle.tree.root_node().kind(), "source_file"); + assert_eq!(bundle.root_tree().root_node().kind(), "source_file"); assert_eq!( bundle.source.as_ref(), b"fn main() { let _ = 1;}\n", diff --git a/tests/injection_acceptance.rs b/tests/injection_acceptance.rs new file mode 100644 index 0000000..a5129e6 --- /dev/null +++ b/tests/injection_acceptance.rs @@ -0,0 +1,165 @@ +// injection_acceptance.rs --- multi-language injection acceptance gates. + +//! Multi-language injection acceptance gates that need the public API +//! surface (the Lua async path and a settle-time perf budget). The +//! layer-structure, alias-resolution, producer, and GPU gates live next to +//! the code they exercise (`syntax.rs` / `semantic_render.rs` / +//! `highlight.rs` / `pmacs-gpu`), where `run_parse`, `scoped_style_spans`, +//! and `source_color_at` are reachable. See the framing acceptance list in +//! `docs/multi-language-injections-framing.md`. + +use std::fmt::Write as _; +use std::time::{Duration, Instant}; + +use pmacs::buffer::{Buffer, BufferId, EditOp}; +use pmacs::editor::EditorState; +use pmacs::lua_bindings::BufferIdLua; +use pmacs::syntax::{self, ParseView, SyntaxRegistry}; + +/// Drive `tick_async` until `predicate` holds or a deadline passes. +fn pump_async bool>(state: &mut EditorState, predicate: F) { + let deadline = Instant::now() + Duration::from_secs(3); + while !predicate(state) { + assert!(Instant::now() < deadline, "async pump deadline exceeded"); + state.tick_async(); + std::thread::sleep(Duration::from_millis(2)); + } +} + +/// Framing acceptance #5 (full Lua async path): an alias added from Lua via +/// `pmacs.parse.injection_aliases` must reach the parse worker through the +/// dispatch snapshot, so an *asynchronously* dispatched injected parse +/// resolves a fence named by the new alias. Testing only the static +/// resolver would leave the Lua write-through + snapshot bridge unproven. +#[test] +fn lua_alias_override_resolves_on_async_parse() { + let mut state = EditorState::new(); + + // Add a bespoke fence alias from Lua (write-through to the registry). + state + .lua_host + .lua() + .load(r#"pmacs.parse.injection_aliases.mydsl = "rust""#) + .exec() + .expect("set injection alias from Lua"); + + // A markdown buffer whose fence uses the new alias. + let src = b"# Doc\n\n```mydsl\nfn injected() { let x = 1; }\n```\n"; + let buf_id = state + .lua_host + .registry() + .borrow_mut() + .create_from_bytes("doc.md".to_owned(), src); + state + .lua_host + .lua() + .globals() + .set("BUF", BufferIdLua(buf_id)) + .expect("bind BUF"); + + // Dispatch asynchronously (the wrapped `_dispatch` records the job; the + // per-tick settle path installs the resolved bundle). + state + .lua_host + .lua() + .load("pmacs.parse._dispatch(BUF, 'markdown')") + .exec() + .expect("async dispatch"); + + pump_async(&mut state, |s| { + s.syntax_registry + .view(buf_id) + .and_then(|h| h.current()) + .is_some() + }); + + let bundle = state + .syntax_registry + .view(buf_id) + .and_then(|h| h.current()) + .expect("settled bundle"); + assert!( + bundle.layers.iter().any(|l| l.language_name == "rust"), + "the Lua-set `mydsl` alias resolved the fence to a rust child layer; \ + layers: {:?}", + bundle + .layers + .iter() + .map(|l| l.language_name.as_str()) + .collect::>() + ); +} + +/// Framing acceptance #12: a large all-inline markdown buffer settles +/// (root + one cold inline layer per paragraph) within a comfortable +/// budget, and the FINAL paragraph still receives an inline layer — the +/// tail is not silently dropped. This is the measured guard that keeps +/// child-incrementality (Q#IJ8) out of v1. +#[test] +fn many_paragraph_settle_under_budget_with_tail_covered() { + let reg = SyntaxRegistry::new(); + let n = 200usize; + let mut src = String::new(); + for i in 0..n { + // A blank line separates paragraphs; each carries emphasis + a link + // so the block grammar injects a markdown_inline layer for it. + writeln!( + src, + "Paragraph {i} with *emphasis* and a [link](http://x).\n" + ) + .expect("write"); + } + // A distinctly-marked final paragraph. + let marker = "FINALPARAGRAPH"; + writeln!(src, "{marker} with *stress*.\n").expect("write"); + + let language = reg.language("markdown").expect("markdown grammar"); + let mut buf = Buffer::new(BufferId::next(), "big.md"); + buf.apply_edit(EditOp::Insert { + pos: 0, + bytes: src.as_bytes(), + }) + .expect("seed markdown"); + let view = ParseView::new(&buf, language, "markdown".to_owned()); + let handle = view.handle(); + let _vid = buf.attach_view(Box::new(view)); + let mut req = handle.make_request(); + req.injection_aliases = reg.injection_alias_snapshot(); + + let start = Instant::now(); + let bundle = syntax::run_parse(req).expect("layered markdown parse"); + let elapsed = start.elapsed(); + + // Catastrophic-regression guard: cold-parsing ~200 tiny inline layers + // is milliseconds of work; a generous ceiling avoids debug/CI flakiness + // while still catching a blow-up (e.g. accidental O(n^2) layer work). + assert!( + elapsed < Duration::from_secs(2), + "many-paragraph settle took {elapsed:?}, exceeds the budget" + ); + + let inline_count = bundle + .layers + .iter() + .filter(|l| l.language_name == "markdown_inline") + .count(); + assert!( + inline_count >= 100, + "most paragraphs produce an inline layer; got {inline_count}" + ); + + // Tail coverage: an inline layer's tree spans the final paragraph. + let marker_off = src.find(marker).expect("marker present"); + let tail_covered = bundle + .layers + .iter() + .filter(|l| l.language_name == "markdown_inline") + .any(|l| { + let r = l.tree.root_node(); + (r.start_byte()..r.end_byte()).contains(&marker_off) + }); + assert!( + tail_covered, + "the final paragraph still receives an inline layer (no tail loss)" + ); +} diff --git a/tests/m4_acceptance.rs b/tests/m4_acceptance.rs index f922253..59dde84 100644 --- a/tests/m4_acceptance.rs +++ b/tests/m4_acceptance.rs @@ -125,9 +125,10 @@ fn m4_1_initial_parse_of_5000_line_file_under_100ms() { language_name: "rust".to_owned(), prior_tree: None, edits: Vec::new(), + injection_aliases: Arc::new(std::collections::HashMap::new()), }; let bundle = syntax::run_parse(req).expect("parse succeeds"); - assert_eq!(bundle.tree.root_node().kind(), "source_file"); + assert_eq!(bundle.root_tree().root_node().kind(), "source_file"); assert!( bundle.parse_duration < Duration::from_millis(100), "5000-line cold parse took {:?}, exceeds 100 ms budget", @@ -204,7 +205,7 @@ fn m4_1_incremental_parse_under_5ms() { let req = handle.make_request(); let bundle = syntax::run_parse(req).expect("incremental parse"); - assert_eq!(bundle.tree.root_node().kind(), "source_file"); + assert_eq!(bundle.root_tree().root_node().kind(), "source_file"); assert!( bundle.parse_duration < Duration::from_millis(5), "incremental parse took {:?}, exceeds 5 ms budget", @@ -226,6 +227,7 @@ fn m4_1_dispatch_parse_round_trips_via_runtime() { language_name: "rust".to_owned(), prior_tree: None, edits: Vec::new(), + injection_aliases: Arc::new(std::collections::HashMap::new()), }; let id = rt.dispatch_parse(req, None); let deadline = Instant::now() + Duration::from_secs(5); @@ -243,7 +245,7 @@ fn m4_1_dispatch_parse_round_trips_via_runtime() { } other => panic!("unexpected outcome: {other:?}"), } - assert_eq!(bundle.tree.root_node().kind(), "source_file"); + assert_eq!(bundle.root_tree().root_node().kind(), "source_file"); assert_eq!(bundle.language_name, "rust"); } @@ -542,6 +544,7 @@ fn m4_3_open_rust_file_highlights_under_100ms() { language_name: "rust".to_owned(), prior_tree: None, edits: Vec::new(), + injection_aliases: Arc::new(std::collections::HashMap::new()), }; let bundle = syntax::run_parse(req).expect("parse succeeds"); let spans = syntax::compute_highlight_spans(&query, &bundle);