From 47ffe5dcffac48a184b8d2603b5d7e20ca33f973 Mon Sep 17 00:00:00 2001 From: Levi Neuwirth Date: Wed, 22 Jul 2026 12:28:31 -0400 Subject: [PATCH] syntax: process bundled locals queries Compile grammar locals metadata, resolve lexical definitions and references once per settled layer, and apply local property predicates in both highlight producers. Restore non-shadowed JavaScript builtins while suppressing local shadows, with lexical, viewport, render, and edit-freshness regressions. --- src/highlight.rs | 8 +- src/semantic_render.rs | 80 +++++- src/syntax.rs | 609 ++++++++++++++++++++++++++++++++++++----- tests/m4_acceptance.rs | 92 +++++++ 4 files changed, 707 insertions(+), 82 deletions(-) diff --git a/src/highlight.rs b/src/highlight.rs index 487afb9..18740ca 100644 --- a/src/highlight.rs +++ b/src/highlight.rs @@ -374,7 +374,13 @@ impl SyntaxHighlightView { let Some(query) = layer.highlight_query.as_ref() else { continue; }; - let spans = compute_highlight_spans_for(query, &layer.tree, source, None); + let spans = compute_highlight_spans_for( + query, + &layer.tree, + source, + layer.local_facts.as_deref(), + None, + ); if spans.is_empty() { continue; } diff --git a/src/semantic_render.rs b/src/semantic_render.rs index 2d3ec0a..3ca2aa7 100644 --- a/src/semantic_render.rs +++ b/src/semantic_render.rs @@ -1964,6 +1964,7 @@ fn scoped_style_spans(state: &EditorState, vp: &DeclaredViewport) -> Vec crate::syntax::ParseViewHandle { let language = state .syntax_registry - .language("rust") - .expect("rust language"); + .language(language_name) + .unwrap_or_else(|| panic!("{language_name} language")); let mut core = state.core.borrow_mut(); let registry_handle = core.registry.clone(); let mut registry = registry_handle.borrow_mut(); @@ -3936,23 +3939,31 @@ mod tests { pos: 0, bytes: text, }) - .expect("seed rust text"); + .expect("seed syntax text"); } - let parse_view = crate::syntax::ParseView::new(buf, language, "rust".to_owned()); + let parse_view = crate::syntax::ParseView::new(buf, language, language_name.to_owned()); let handle = parse_view.handle(); let req = handle.make_request(); - let bundle = crate::syntax::run_parse(req).expect("initial rust parse"); - // Mirror the production settle path: resolve each layer's highlight - // query before install so the producer can style it (framing Q#IJ2). + let bundle = crate::syntax::run_parse(req).expect("initial syntax parse"); + // Mirror the production settle path: queries and lexical facts travel + // with the same bundle the producer reads. handle.install(state.syntax_registry.resolve_layer_queries(&bundle)); buf.attach_view(Box::new(parse_view)); drop(registry); - core.set_buffer_path(buffer_id, Some(std::path::PathBuf::from("/tmp/x.rs"))); + core.set_buffer_path(buffer_id, Some(std::path::PathBuf::from(path))); drop(core); state.syntax_registry.attach_view(buffer_id, handle.clone()); handle } + fn seed_rust_parse_view( + state: &EditorState, + buffer_id: BufferId, + text: &[u8], + ) -> crate::syntax::ParseViewHandle { + seed_parse_view(state, buffer_id, text, "rust", "/tmp/x.rs") + } + fn seed_markdown_parse_view( state: &EditorState, buffer_id: BufferId, @@ -4037,6 +4048,57 @@ mod tests { ); } + #[test] + fn viewport_style_producer_uses_settled_local_facts() { + let state = empty_state(); + let buffer_id = active_buffer(&state); + let source = b"console;\nfunction f(console) { console; }\n"; + seed_parse_view(&state, buffer_id, source, "javascript", "/tmp/locals.js"); + state.syntax_registry.theme().lock().expect("theme").insert( + "variable.builtin", + Style { + fg: crate::cell::Color::Indexed(6), + ..Style::default() + }, + ); + + let style_at = |visible: ByteRange, offset: u64| { + scoped_style_spans( + &state, + &DeclaredViewport { + buffer_id, + visible, + frontend_generation: 0, + }, + ) + .into_iter() + .find(|span| span.range.start <= offset && offset < span.range.end) + .map_or_else(Style::default, |span| span.style) + }; + + assert_eq!( + style_at(ByteRange { start: 0, end: 7 }, 0).fg, + crate::cell::Color::Indexed(6), + "unresolved outer `console` receives the builtin capture" + ); + let inner = source + .windows("console".len()) + .rposition(|window| window == b"console") + .expect("inner console") as u64; + assert_ne!( + style_at( + ByteRange { + start: inner, + end: inner + "console".len() as u64, + }, + inner, + ) + .fg, + crate::cell::Color::Indexed(6), + "viewport-only highlighting still sees the parameter definition outside the viewport" + ); + } + #[test] fn full_buffer_summary_flatten_scales_on_large_grammar_file() { // Perf gate (round-1 finding 1): the file-style summary runs the diff --git a/src/syntax.rs b/src/syntax.rs index cdd6d56..2538641 100644 --- a/src/syntax.rs +++ b/src/syntax.rs @@ -115,6 +115,14 @@ pub struct ParseTreeBundle { pub injection_capped: bool, } +/// Lexically-local identifier ranges derived from a grammar's bundled +/// `locals.scm` query. Ranges are sorted and deduplicated so highlight +/// predicate checks are allocation-free binary searches. +#[derive(Debug, Default)] +pub struct LocalFacts { + ranges: Box<[(u32, u32)]>, +} + /// One injection layer within a [`ParseTreeBundle`] (framing Q#IJ1). A /// layer pairs a parse tree with the language that produced it and the /// injection-nesting depth (root = 0). `highlight_query` is resolved on @@ -133,6 +141,10 @@ pub struct Layer { /// `None` when the language ships no highlights, or on the worker /// (pre-settle). Producers read it to style this layer. pub highlight_query: Option>, + /// Lexically-local definitions and resolved references for this tree. + /// Present only when the highlight query asks about the `local` + /// property; computed once when the bundle settles. + pub local_facts: Option>, } impl ParseTreeBundle { @@ -184,6 +196,7 @@ pub fn run_parse(req: ParseRequest) -> Result { tree: root_tree, depth: 0, highlight_query: None, + local_facts: None, }]; let injection_capped = build_injection_layers(&mut layers, req.source.as_ref(), &req.injection_aliases); @@ -309,6 +322,7 @@ fn build_injection_layers( tree, depth: depth + 1, highlight_query: None, + local_facts: None, }, ranges, )); @@ -773,6 +787,10 @@ pub struct LanguageEntry { /// slice (or all-empty fragments) means no highlights: the view /// runs but emits nothing. pub highlights_query: &'static [&'static str], + /// Bundled `locals.scm` query fragments, composed base-first like + /// [`Self::highlights_query`]. The query supplies lexical scopes, + /// definitions, values, and references for `local` property predicates. + pub locals_query: &'static [&'static str], /// Bundled `injections.scm` fragments (framing Q#IJ2), joined with a /// newline and compiled on the parse worker to find embedded-language /// regions. Empty for the many grammars that ship none (or don't @@ -801,6 +819,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["rs"], loader: || tree_sitter_rust::LANGUAGE.into(), highlights_query: &[tree_sitter_rust::HIGHLIGHTS_QUERY], + locals_query: &[], injections_query: &[tree_sitter_rust::INJECTIONS_QUERY], }, LanguageEntry { @@ -808,6 +827,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["lua"], loader: || tree_sitter_lua::LANGUAGE.into(), highlights_query: &[tree_sitter_lua::HIGHLIGHTS_QUERY], + locals_query: &[tree_sitter_lua::LOCALS_QUERY], injections_query: &[], }, // T M9.7: markdown block grammar (`tree_sitter_md::LANGUAGE`) — headers, @@ -824,6 +844,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["md", "markdown"], loader: || tree_sitter_md::LANGUAGE.into(), highlights_query: &[tree_sitter_md::HIGHLIGHT_QUERY_BLOCK], + locals_query: &[], injections_query: &[tree_sitter_md::INJECTION_QUERY_BLOCK], }, // markdown_inline (framing Q#IJ10) — the inline grammar the block @@ -837,6 +858,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &[], loader: || tree_sitter_md::INLINE_LANGUAGE.into(), highlights_query: &[tree_sitter_md::HIGHLIGHT_QUERY_INLINE], + locals_query: &[], injections_query: &[tree_sitter_md::INJECTION_QUERY_INLINE], }, // T M_B3 — C / C++. Lexical highlighting (keywords / strings / @@ -859,6 +881,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["c", "h"], loader: || tree_sitter_c::LANGUAGE.into(), highlights_query: &[tree_sitter_c::HIGHLIGHT_QUERY], + locals_query: &[], injections_query: &[], }, LanguageEntry { @@ -866,6 +889,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["cpp", "cc", "cxx", "hpp", "hh", "hxx", "ipp", "inl", "cppm"], loader: || tree_sitter_cpp::LANGUAGE.into(), highlights_query: &[tree_sitter_cpp::HIGHLIGHT_QUERY], + locals_query: &[], injections_query: &[], }, // CUDA (`.cu` source, `.cuh` header). A dedicated grammar rather @@ -897,6 +921,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ tree_sitter_cpp::HIGHLIGHT_QUERY, tree_sitter_cuda::HIGHLIGHTS_QUERY, ], + locals_query: &[], injections_query: &[], }, // Shell / bash. Lexical highlighting for the shell family; the LSP @@ -915,6 +940,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["sh", "bash", "zsh", "ksh", "ash", "bats"], loader: || tree_sitter_bash::LANGUAGE.into(), highlights_query: &[tree_sitter_bash::HIGHLIGHT_QUERY], + locals_query: &[], injections_query: &[], }, // Filename-identified languages. These files usually have no useful @@ -931,6 +957,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["dockerfile", "containerfile"], loader: || tree_sitter_containerfile::LANGUAGE.into(), highlights_query: &[tree_sitter_containerfile::HIGHLIGHTS_QUERY], + locals_query: &[], injections_query: &[], }, LanguageEntry { @@ -938,6 +965,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["mk", "make"], loader: || tree_sitter_make::LANGUAGE.into(), highlights_query: &[tree_sitter_make::HIGHLIGHTS_QUERY], + locals_query: &[], injections_query: &[], }, LanguageEntry { @@ -945,6 +973,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["cmake"], loader: || tree_sitter_cmake::LANGUAGE.into(), highlights_query: &[tree_sitter_cmake::HIGHLIGHTS_QUERY], + locals_query: &[], injections_query: &[], }, // Grammar-gap languages — these already had LSP configs but no @@ -958,6 +987,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["py", "pyi"], loader: || tree_sitter_python::LANGUAGE.into(), highlights_query: &[tree_sitter_python::HIGHLIGHTS_QUERY], + locals_query: &[], injections_query: &[], }, LanguageEntry { @@ -965,6 +995,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["go"], loader: || tree_sitter_go::LANGUAGE.into(), highlights_query: &[tree_sitter_go::HIGHLIGHTS_QUERY], + locals_query: &[], injections_query: &[], }, // JavaScript / TypeScript. One `tree-sitter-javascript` grammar parses @@ -980,6 +1011,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["js", "mjs", "cjs"], loader: || tree_sitter_javascript::LANGUAGE.into(), highlights_query: &[tree_sitter_javascript::HIGHLIGHT_QUERY], + locals_query: &[tree_sitter_javascript::LOCALS_QUERY], injections_query: &[], }, LanguageEntry { @@ -990,6 +1022,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ tree_sitter_javascript::HIGHLIGHT_QUERY, tree_sitter_javascript::JSX_HIGHLIGHT_QUERY, ], + locals_query: &[tree_sitter_javascript::LOCALS_QUERY], injections_query: &[], }, LanguageEntry { @@ -1000,6 +1033,10 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ tree_sitter_javascript::HIGHLIGHT_QUERY, tree_sitter_typescript::HIGHLIGHTS_QUERY, ], + locals_query: &[ + tree_sitter_javascript::LOCALS_QUERY, + tree_sitter_typescript::LOCALS_QUERY, + ], injections_query: &[], }, LanguageEntry { @@ -1011,6 +1048,10 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ tree_sitter_javascript::JSX_HIGHLIGHT_QUERY, tree_sitter_typescript::HIGHLIGHTS_QUERY, ], + locals_query: &[ + tree_sitter_javascript::LOCALS_QUERY, + tree_sitter_typescript::LOCALS_QUERY, + ], injections_query: &[], }, LanguageEntry { @@ -1018,6 +1059,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["toml"], loader: || tree_sitter_toml_ng::LANGUAGE.into(), highlights_query: &[tree_sitter_toml_ng::HIGHLIGHTS_QUERY], + locals_query: &[], injections_query: &[], }, LanguageEntry { @@ -1025,6 +1067,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["zig", "zon"], loader: || tree_sitter_zig::LANGUAGE.into(), highlights_query: &[tree_sitter_zig::HIGHLIGHTS_QUERY], + locals_query: &[], injections_query: &[], }, // JSON + YAML — config formats, both self-contained highlights and no @@ -1039,6 +1082,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["json"], loader: || tree_sitter_json::LANGUAGE.into(), highlights_query: &[tree_sitter_json::HIGHLIGHTS_QUERY], + locals_query: &[], injections_query: &[], }, LanguageEntry { @@ -1046,6 +1090,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[ extensions: &["yaml", "yml"], loader: || tree_sitter_yaml::LANGUAGE.into(), highlights_query: &[tree_sitter_yaml::HIGHLIGHTS_QUERY], + locals_query: &[], injections_query: &[], }, ]; @@ -1076,6 +1121,9 @@ pub struct SyntaxRegistry { /// compilation failure (e.g. grammar / query ABI skew) is /// cached as `Err(message)` so we don't burn cycles re-trying. queries: RefCell, String>>>, + /// Compiled `locals.scm` query per language. Like `queries`, both + /// compilation failures and absent query sources are cached. + local_queries: RefCell, String>>>, /// Fence-name → canonical-language alias map (framing Q#IJ4). Seeded /// with [`default_injection_aliases`]; Lua adds to it through /// [`Self::register_injection_alias`]. Snapshotted into each @@ -1107,6 +1155,7 @@ impl SyntaxRegistry { parse_jobs: RefCell::new(HashMap::new()), extra_extensions: RefCell::new(HashMap::new()), queries: RefCell::new(HashMap::new()), + local_queries: RefCell::new(HashMap::new()), injection_aliases: RefCell::new(default_injection_aliases()), theme: Arc::new(Mutex::new(Theme::default_dark())), } @@ -1273,6 +1322,32 @@ impl SyntaxRegistry { result } + /// Lazy-compile and cache the bundled `locals.scm` query for + /// `lang_name`. Empty sources and compilation failures are cached. + #[must_use] + pub fn locals_query(&self, lang_name: &str) -> Option> { + if let Some(slot) = self.local_queries.borrow().get(lang_name) { + return slot.as_ref().ok().cloned(); + } + let language = self.language(lang_name)?; + let entry = BUILTIN_LANGUAGES.iter().find(|e| e.name == lang_name); + let source = entry.map_or_else(String::new, |e| e.locals_query.join("\n")); + if source.trim().is_empty() { + self.local_queries + .borrow_mut() + .insert(lang_name.to_owned(), Err("no locals query".to_owned())); + return None; + } + let compiled = tree_sitter::Query::new(&language, &source) + .map(Arc::new) + .map_err(|e| format!("compile {lang_name} locals: {e:?}")); + let result = compiled.as_ref().ok().cloned(); + self.local_queries + .borrow_mut() + .insert(lang_name.to_owned(), compiled); + result + } + /// Add or override a fence-name → language alias (framing Q#IJ4). The /// alias key is case-folded to match the resolver. Called from Lua via /// `pmacs.parse.injection_aliases`. @@ -1299,11 +1374,26 @@ impl SyntaxRegistry { let layers = raw .layers .iter() - .map(|l| Layer { - language_name: l.language_name.clone(), - tree: l.tree.clone(), - depth: l.depth, - highlight_query: self.highlights_query(&l.language_name), + .map(|layer| { + let highlight_query = self.highlights_query(&layer.language_name); + let local_facts = highlight_query + .as_deref() + .filter(|query| query_uses_local_predicates(query)) + .and_then(|_| self.locals_query(&layer.language_name)) + .map(|query| { + Arc::new(compute_local_facts( + &query, + &layer.tree, + raw.source.as_ref(), + )) + }); + Layer { + language_name: layer.language_name.clone(), + tree: layer.tree.clone(), + depth: layer.depth, + highlight_query, + local_facts, + } }) .collect(); Arc::new(ParseTreeBundle { @@ -1316,6 +1406,148 @@ impl SyntaxRegistry { } } +fn query_uses_local_predicates(query: &tree_sitter::Query) -> bool { + (0..query.pattern_count()).any(|pattern| { + query + .property_predicates(pattern) + .iter() + .any(|(property, _)| property.key.as_ref() == "local") + }) +} + +#[derive(Debug)] +struct LocalDefinition { + name_range: std::ops::Range, + value_range: std::ops::Range, +} + +#[derive(Debug)] +struct LocalScope { + inherits: bool, + range: std::ops::Range, + definitions: Vec, +} + +impl LocalFacts { + fn contains(&self, start_byte: usize, end_byte: usize) -> bool { + let (Ok(start_byte), Ok(end_byte)) = (u32::try_from(start_byte), u32::try_from(end_byte)) + else { + return false; + }; + self.ranges.binary_search(&(start_byte, end_byte)).is_ok() + } +} + +/// Resolve lexical definitions and references according to Tree-sitter's +/// standard `locals.scm` capture conventions. +fn compute_local_facts( + query: &tree_sitter::Query, + tree: &tree_sitter::Tree, + source: &[u8], +) -> LocalFacts { + let scope_capture = query.capture_index_for_name("local.scope"); + let definition_capture = query.capture_index_for_name("local.definition"); + let value_capture = query.capture_index_for_name("local.definition-value"); + let reference_capture = query.capture_index_for_name("local.reference"); + + let mut scopes = vec![LocalScope { + inherits: false, + range: 0..source.len(), + definitions: Vec::new(), + }]; + let mut ranges = Vec::new(); + let mut cursor = tree_sitter::QueryCursor::new(); + let mut captures = cursor.captures(query, tree.root_node(), source); + + while let Some((query_match, capture_index)) = captures.next() { + let capture = query_match.captures[*capture_index]; + let node_range = capture.node.byte_range(); + while scopes.len() > 1 + && node_range.start > scopes.last().expect("root scope exists").range.end + { + scopes.pop(); + } + + if Some(capture.index) == scope_capture { + let mut inherits = true; + for property in query.property_settings(query_match.pattern_index) { + if property.key.as_ref() == "local.scope-inherits" { + inherits = property + .value + .as_deref() + .is_none_or(|value| value == "true"); + } + } + scopes.push(LocalScope { + inherits, + range: node_range, + definitions: Vec::new(), + }); + continue; + } + + if Some(capture.index) == definition_capture { + let Some(_) = source.get(node_range.clone()) else { + continue; + }; + let value_range = query_match + .captures + .iter() + .find(|candidate| Some(candidate.index) == value_capture) + .map_or(0..0, |candidate| candidate.node.byte_range()); + scopes + .last_mut() + .expect("root scope exists") + .definitions + .push(LocalDefinition { + name_range: node_range.clone(), + value_range, + }); + if let (Ok(start), Ok(end)) = ( + u32::try_from(node_range.start), + u32::try_from(node_range.end), + ) { + ranges.push((start, end)); + } + continue; + } + + if Some(capture.index) != reference_capture { + continue; + } + let Some(name) = source.get(node_range.clone()) else { + continue; + }; + let mut resolved = false; + for scope in scopes.iter().rev() { + if scope.definitions.iter().rev().any(|definition| { + node_range.start >= definition.value_range.end + && source.get(definition.name_range.clone()) == Some(name) + }) { + resolved = true; + break; + } + if !scope.inherits { + break; + } + } + if resolved + && let (Ok(start), Ok(end)) = ( + u32::try_from(node_range.start), + u32::try_from(node_range.end), + ) + { + ranges.push((start, end)); + } + } + + ranges.sort_unstable(); + ranges.dedup(); + LocalFacts { + ranges: ranges.into_boxed_slice(), + } +} + impl Default for SyntaxRegistry { fn default() -> Self { Self::new() @@ -1418,20 +1650,22 @@ pub fn compute_highlight_spans_in_range( query, bundle.root_tree(), bundle.source.as_ref(), + bundle.layers[0].local_facts.as_deref(), byte_range, ) } /// Like [`compute_highlight_spans_in_range`] but over an explicit -/// `(tree, source)` — the per-layer form the producers call for each -/// injection layer (framing Q#IJ7). `source` is the whole buffer; a -/// child layer's tree carries absolute offsets into it, so the same +/// `(tree, source, local_facts)` layer tuple — the form producers call for +/// each injection layer (framing Q#IJ7 and Q#LQ5). `source` is the whole +/// buffer; a child layer's tree carries absolute offsets into it, so the same /// capture walk works unchanged. #[must_use] pub fn compute_highlight_spans_for( query: &tree_sitter::Query, tree: &tree_sitter::Tree, source: &[u8], + local_facts: Option<&LocalFacts>, byte_range: Option>, ) -> Vec { let mut spans = Vec::new(); @@ -1441,30 +1675,33 @@ pub fn compute_highlight_spans_for( } let root = tree.root_node(); let mut iter = cursor.captures(query, root, source); - while let Some((qmatch, capture_idx)) = iter.next() { - // Fail-closed on the locals property predicate. The capture - // iterator already applies text predicates (`#eq?`/`#match?`/ - // `#any-of?`), but `#is? local` / `#is-not? local` are *property* - // predicates (`Query::property_predicates`) that need a scope map - // built from the grammar's LOCALS_QUERY, which pmacs does not run. - // Applying such a capture regardless mis-styles shadowed locals — - // e.g. a local `console`/`require` in JS/TS would still capture as - // `@variable.builtin`/`@function.builtin`. Until locals processing - // exists, drop captures whose pattern carries one; the identifier - // falls back to its non-builtin capture. `#set!` (property - // *settings*) is a different API and is not consulted here. - if query - .property_predicates(qmatch.pattern_index) + while let Some((query_match, capture_index)) = iter.next() { + let capture = query_match.captures[*capture_index]; + let local_predicates_match = query + .property_predicates(query_match.pattern_index) .iter() - .any(|(prop, _)| &*prop.key == "local") - { + .filter(|(property, _)| property.key.as_ref() == "local") + .all(|(property, positive)| { + let node = property.capture_id.map_or(Some(capture.node), |target| { + query_match + .captures + .iter() + .find(|candidate| candidate.index as usize == target) + .map(|candidate| candidate.node) + }); + let is_local = node.is_some_and(|node| { + local_facts + .is_some_and(|facts| facts.contains(node.start_byte(), node.end_byte())) + }); + is_local == *positive + }); + if !local_predicates_match { continue; } - let cap = qmatch.captures[*capture_idx]; spans.push(HighlightSpan { - start_byte: cap.node.start_byte() as u32, - end_byte: cap.node.end_byte() as u32, - capture_index: cap.index, + start_byte: capture.node.start_byte() as u32, + end_byte: capture.node.end_byte() as u32, + capture_index: capture.index, }); } // Wider-first ordering at equal start: later writes (the @@ -1684,7 +1921,13 @@ mod tests { .highlight_query .as_ref() .expect("inline highlights resolved at settle"); - let spans = compute_highlight_spans_for(hquery, &inline.tree, &bundle.source, None); + let spans = compute_highlight_spans_for( + hquery, + &inline.tree, + &bundle.source, + inline.local_facts.as_deref(), + None, + ); assert!( !spans.is_empty(), "the inline layer produces highlight spans across both ranges" @@ -2221,7 +2464,13 @@ mod tests { .highlight_query .as_ref() .expect("yaml highlights resolved"); - let spans = compute_highlight_spans_for(query, &yaml.tree, &bundle.source, None); + let spans = compute_highlight_spans_for( + query, + &yaml.tree, + &bundle.source, + yaml.local_facts.as_deref(), + None, + ); assert!(!spans.is_empty(), "the yaml frontmatter layer highlights"); } @@ -2418,54 +2667,270 @@ mod tests { } #[test] - fn javascript_shadowed_builtin_is_not_mislabeled() { - // `#is-not? local` (JS/TS use it for console/require/etc.) needs a - // scope map from the LOCALS_QUERY we don't run, so - // `compute_highlight_spans` drops captures guarded by it. Here - // `console` is a LOCAL declaration — it must not surface as a - // `*.builtin` capture (which is what a naive run of the shared JS - // query would produce). - let reg = SyntaxRegistry::new(); - let language = reg.language("javascript").expect("javascript loads"); - let query = reg - .highlights_query("javascript") + fn javascript_local_predicates_distinguish_lexical_scope() { + let registry = SyntaxRegistry::new(); + let source = b"console.log('outer');\n\ + require('outer');\n\ + function f(console, require) {\n\ + console.log('inner');\n\ + require('inner');\n\ + }\n\ + window.alert('outer');\n"; + let bundle = parse_layered(®istry, "javascript", source); + let layer = &bundle.layers[0]; + let query = layer + .highlight_query + .as_deref() .expect("javascript highlights compile"); - let mut buf = fresh_buffer("shadow.js"); - buf.apply_edit(EditOp::Insert { - pos: 0, - bytes: b"const console = 5;\nconsole;\n", - }) - .unwrap(); - let view = ParseView::new(&buf, language, "javascript".to_owned()); - let handle = view.handle(); - let _vid = buf.attach_view(Box::new(view)); - let bundle = parse_synchronously(&handle); - let spans = compute_highlight_spans(&query, &bundle); - assert!(!spans.is_empty(), "the JS query produced highlight spans"); - let names = query.capture_names(); - let builtin: Vec<&str> = spans - .iter() - .map(|s| names[s.capture_index as usize]) - .filter(|n| n.contains("builtin")) - .collect(); assert!( - builtin.is_empty(), - "a locally-shadowed `console` must not get a *.builtin capture; got {builtin:?}" + layer.local_facts.is_some(), + "a local-sensitive highlight query must settle lexical facts" ); - // ...and dropping the builtin pattern must not strip *all* styling: - // each `console` occurrence still keeps its ordinary `@variable` - // capture (the fallback), so the loss is only the `.builtin` refine. - let src = "const console = 5;\nconsole;\n"; - for (pos, _) in src.match_indices("console") { - let (start, end) = (pos as u32, (pos + "console".len()) as u32); - let caps: Vec<&str> = spans + let spans = compute_highlight_spans(query, &bundle); + let names = query.capture_names(); + let captures_at = |start: usize, len: usize| -> Vec<&str> { + spans .iter() - .filter(|s| s.start_byte == start && s.end_byte == end) - .map(|s| names[s.capture_index as usize]) + .filter(|span| { + span.start_byte == start as u32 && span.end_byte == (start + len) as u32 + }) + .map(|span| names[span.capture_index as usize]) + .collect() + }; + + for identifier in ["console", "require"] { + let positions: Vec = std::str::from_utf8(source) + .expect("fixture is UTF-8") + .match_indices(identifier) + .map(|(position, _)| position) .collect(); + assert_eq!(positions.len(), 3, "fixture has three `{identifier}` uses"); assert!( - caps.iter().any(|n| n.starts_with("variable")), - "`console` at byte {pos} keeps a variable capture; got {caps:?}" + captures_at(positions[0], identifier.len()) + .iter() + .any(|name| name.ends_with(".builtin")), + "unshadowed outer `{identifier}` keeps its builtin refinement" + ); + for position in &positions[1..] { + let captures = captures_at(*position, identifier.len()); + assert!( + !captures.iter().any(|name| name.ends_with(".builtin")), + "local `{identifier}` at byte {position} is not builtin: {captures:?}" + ); + assert!( + captures.iter().any(|name| name.starts_with("variable")), + "local `{identifier}` keeps an ordinary variable capture: {captures:?}" + ); + } + } + + let window = std::str::from_utf8(source) + .expect("fixture is UTF-8") + .find("window") + .expect("window fixture"); + assert!( + captures_at(window, "window".len()) + .iter() + .any(|name| name == &"variable.builtin"), + "an unresolved builtin after the function remains builtin" + ); + } + + #[test] + fn positive_and_capture_qualified_local_predicates_use_resolved_facts() { + let registry = SyntaxRegistry::new(); + let source = b"let f = () => {};\nf();\ng();\n"; + let bundle = parse_layered(®istry, "javascript", source); + let language = registry.language("javascript").expect("javascript loads"); + let facts = bundle.layers[0] + .local_facts + .as_deref() + .expect("javascript local facts settle"); + + let positive = tree_sitter::Query::new(&language, "((identifier) @local-id (#is? local))") + .expect("positive local predicate compiles"); + let positive_spans = + compute_highlight_spans_for(&positive, bundle.root_tree(), source, Some(facts), None); + let f_positions: Vec = std::str::from_utf8(source) + .expect("fixture is UTF-8") + .match_indices('f') + .map(|(position, _)| position) + .collect(); + assert_eq!(f_positions.len(), 2); + for position in f_positions { + assert!( + positive_spans.iter().any(|span| { + span.start_byte == position as u32 && span.end_byte == (position + 1) as u32 + }), + "definition/reference `f` at byte {position} is local" + ); + } + let g_position = std::str::from_utf8(source) + .expect("fixture is UTF-8") + .find("g()") + .expect("g call"); + assert!( + positive_spans + .iter() + .all(|span| span.start_byte != g_position as u32), + "unresolved `g` does not satisfy #is? local" + ); + + let qualified = tree_sitter::Query::new( + &language, + "((call_expression function: (identifier) @callee) @call \ + (#is? @callee local))", + ) + .expect("capture-qualified local predicate compiles"); + let qualified_spans = + compute_highlight_spans_for(&qualified, bundle.root_tree(), source, Some(facts), None); + let qualified_names = qualified.capture_names(); + assert!( + qualified_spans.iter().any(|span| { + qualified_names[span.capture_index as usize] == "call" + && span.start_byte + == source + .windows(4) + .position(|window| window == b"f();") + .expect("f call") as u32 + }), + "the call whose @callee is local satisfies the qualified predicate" + ); + assert!( + qualified_spans + .iter() + .all(|span| span.start_byte != g_position as u32), + "the call whose @callee is unresolved fails the qualified predicate" + ); + } + + #[test] + fn local_definition_value_and_scope_inheritance_control_resolution() { + let registry = SyntaxRegistry::new(); + let language = registry.language("javascript").expect("javascript loads"); + + let value_source = b"let x = x;\nx;\n"; + let value_tree = { + let mut parser = tree_sitter::Parser::new(); + parser + .set_language(&language) + .expect("set javascript language"); + parser + .parse(value_source, None) + .expect("parse value fixture") + }; + let value_locals = tree_sitter::Query::new( + &language, + "(variable_declarator \ + name: (identifier) @local.definition \ + value: (identifier) @local.definition-value) \ + (identifier) @local.reference", + ) + .expect("definition-value locals query compiles"); + let value_facts = compute_local_facts(&value_locals, &value_tree, value_source); + let x_positions: Vec = std::str::from_utf8(value_source) + .expect("fixture is UTF-8") + .match_indices('x') + .map(|(position, _)| position) + .collect(); + assert_eq!(x_positions.len(), 3); + assert!(value_facts.contains(x_positions[0], x_positions[0] + 1)); + assert!( + !value_facts.contains(x_positions[1], x_positions[1] + 1), + "a definition is not visible inside its own value" + ); + assert!(value_facts.contains(x_positions[2], x_positions[2] + 1)); + + let scope_source = b"let x = 1;\nfunction f() { x; }\nx;\n"; + let scope_tree = { + let mut parser = tree_sitter::Parser::new(); + parser + .set_language(&language) + .expect("set javascript language"); + parser + .parse(scope_source, None) + .expect("parse scope fixture") + }; + let scope_locals = tree_sitter::Query::new( + &language, + "((function_declaration) @local.scope \ + (#set! local.scope-inherits false)) \ + (variable_declarator name: (identifier) @local.definition) \ + (identifier) @local.reference", + ) + .expect("non-inheriting locals query compiles"); + let scope_facts = compute_local_facts(&scope_locals, &scope_tree, scope_source); + let x_positions: Vec = std::str::from_utf8(scope_source) + .expect("fixture is UTF-8") + .match_indices('x') + .map(|(position, _)| position) + .collect(); + assert_eq!(x_positions.len(), 3); + assert!(scope_facts.contains(x_positions[0], x_positions[0] + 1)); + assert!( + !scope_facts.contains(x_positions[1], x_positions[1] + 1), + "a non-inheriting scope cannot see the outer `x`" + ); + assert!( + scope_facts.contains(x_positions[2], x_positions[2] + 1), + "leaving the scope restores outer resolution" + ); + } + + #[test] + fn typescript_locals_compose_javascript_scopes_and_parameter_delta() { + let registry = SyntaxRegistry::new(); + for (language_name, source) in [ + ( + "typescript", + &b"function f(console: string) { console.log('x'); }\n\ + window.alert('x');\n"[..], + ), + ( + "typescriptreact", + &b"function F(console: string) { return
{console}
; }\n\ + window.alert('x');\n"[..], + ), + ] { + let locals = registry + .locals_query(language_name) + .unwrap_or_else(|| panic!("{language_name} locals compile")); + assert!( + locals.capture_index_for_name("local.scope").is_some() + && locals.capture_index_for_name("local.definition").is_some() + && locals.capture_index_for_name("local.reference").is_some(), + "{language_name} includes JavaScript's scopes and references" + ); + + let bundle = parse_layered(®istry, language_name, source); + let layer = &bundle.layers[0]; + let query = layer + .highlight_query + .as_deref() + .expect("highlights compile"); + let spans = compute_highlight_spans(query, &bundle); + let names = query.capture_names(); + let text = std::str::from_utf8(source).expect("fixture is UTF-8"); + for (position, _) in text.match_indices("console") { + assert!( + spans + .iter() + .filter(|span| { + span.start_byte == position as u32 + && span.end_byte == (position + "console".len()) as u32 + }) + .all(|span| !names[span.capture_index as usize].ends_with(".builtin")), + "{language_name} parameter/reference `console` is local" + ); + } + let window = text.find("window").expect("window fixture"); + assert!( + spans.iter().any(|span| { + span.start_byte == window as u32 + && span.end_byte == (window + "window".len()) as u32 + && names[span.capture_index as usize] == "variable.builtin" + }), + "{language_name} unresolved `window` remains builtin" ); } } diff --git a/tests/m4_acceptance.rs b/tests/m4_acceptance.rs index 753aeec..69c8500 100644 --- a/tests/m4_acceptance.rs +++ b/tests/m4_acceptance.rs @@ -771,6 +771,98 @@ fn m4_3_theming_via_lua_color_scheme() { ); } +/// Locals-query acceptance: the shipped JavaScript grammar must distinguish an +/// unresolved builtin from a lexically-shadowed parameter, then replace the +/// classification with the fresh parse bundle after an edit removes the +/// shadow. The theme maps only `variable.builtin`, making the classification +/// observable in rendered cells rather than through an internal scope map. +#[test] +fn m4_locals_query_shadowing_and_edit_freshness() { + use pmacs::cell::Color; + + const COLS: usize = 40; + const BUILTIN: Color = Color::Indexed(6); + let initial = "console;\nfunction f(console) {\n console;\n}\n"; + let edited = "console;\nfunction f(logger) {\n console;\n}\n"; + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("locals.js"); + std::fs::write(&path, initial.as_bytes()).expect("write JavaScript fixture"); + + let mut state = open_and_wait_for_parse(path); + state + .lua_host + .lua() + .load( + r#" + pmacs.theme.set { + ["variable.builtin"] = { fg = 6, bold = true }, + } + "#, + ) + .exec() + .expect("apply builtin-only theme"); + + let before = render_active_window_to_grid(&mut state, 5, COLS as u32); + for (col, cell) in before.iter().take(7).enumerate() { + assert_eq!( + cell.style.fg, BUILTIN, + "unshadowed row-0 `console` byte {col} is builtin" + ); + assert!( + cell.style.bold, + "builtin-only theme reaches row-0 `console` byte {col}" + ); + } + for col in 11..18 { + assert_ne!( + before[COLS + col].style.fg, + BUILTIN, + "parameter `console` byte {col} is lexically local" + ); + } + for col in 2..9 { + assert_ne!( + before[2 * COLS + col].style.fg, + BUILTIN, + "reference to shadowing parameter at row 2, col {col} is not builtin" + ); + } + + state + .lua_host + .lua() + .load( + r" + local buf = pmacs.window.buffer() + buf:replace(20, 27, 'logger') + pmacs.parse._dispatch(buf, 'javascript') + ", + ) + .exec() + .expect("rename shadowing parameter"); + pump_async(&mut state, |s| { + current_tree_text(s).as_deref() == Some(edited) + }); + assert_eq!( + current_tree_text(&state).as_deref(), + Some(edited), + "the edited JavaScript parse settled" + ); + + let after = render_active_window_to_grid(&mut state, 5, COLS as u32); + for col in 2..9 { + assert_eq!( + after[2 * COLS + col].style.fg, + BUILTIN, + "fresh local facts restore builtin styling at row 2, col {col}" + ); + assert!( + after[2 * COLS + col].style.bold, + "fresh builtin capture reaches the rendered cell at row 2, col {col}" + ); + } +} + /// Sanity: the bundled `default_dark` theme produces a non-empty /// capture map and resolves common captures to non-default styles. /// Catches accidental regressions to a literally empty theme that