syntax: process bundled locals queries
Compile grammar locals metadata, resolve lexical definitions and references once per settled layer, and apply local property predicates in both highlight producers. Restore non-shadowed JavaScript builtins while suppressing local shadows, with lexical, viewport, render, and edit-freshness regressions.
This commit is contained in:
parent
42a28b90c7
commit
47ffe5dcff
|
|
@ -374,7 +374,13 @@ impl SyntaxHighlightView {
|
||||||
let Some(query) = layer.highlight_query.as_ref() else {
|
let Some(query) = layer.highlight_query.as_ref() else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
let spans = compute_highlight_spans_for(query, &layer.tree, source, None);
|
let spans = compute_highlight_spans_for(
|
||||||
|
query,
|
||||||
|
&layer.tree,
|
||||||
|
source,
|
||||||
|
layer.local_facts.as_deref(),
|
||||||
|
None,
|
||||||
|
);
|
||||||
if spans.is_empty() {
|
if spans.is_empty() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1964,6 +1964,7 @@ fn scoped_style_spans(state: &EditorState, vp: &DeclaredViewport) -> Vec<StyleSp
|
||||||
query,
|
query,
|
||||||
&layer.tree,
|
&layer.tree,
|
||||||
source,
|
source,
|
||||||
|
layer.local_facts.as_deref(),
|
||||||
Some(vis_start as usize..vis_end as usize),
|
Some(vis_start as usize..vis_end as usize),
|
||||||
);
|
);
|
||||||
for (order, hs) in highlights.iter().enumerate() {
|
for (order, hs) in highlights.iter().enumerate() {
|
||||||
|
|
@ -3918,15 +3919,17 @@ mod tests {
|
||||||
sid
|
sid
|
||||||
}
|
}
|
||||||
|
|
||||||
fn seed_rust_parse_view(
|
fn seed_parse_view(
|
||||||
state: &EditorState,
|
state: &EditorState,
|
||||||
buffer_id: BufferId,
|
buffer_id: BufferId,
|
||||||
text: &[u8],
|
text: &[u8],
|
||||||
|
language_name: &str,
|
||||||
|
path: &str,
|
||||||
) -> crate::syntax::ParseViewHandle {
|
) -> crate::syntax::ParseViewHandle {
|
||||||
let language = state
|
let language = state
|
||||||
.syntax_registry
|
.syntax_registry
|
||||||
.language("rust")
|
.language(language_name)
|
||||||
.expect("rust language");
|
.unwrap_or_else(|| panic!("{language_name} language"));
|
||||||
let mut core = state.core.borrow_mut();
|
let mut core = state.core.borrow_mut();
|
||||||
let registry_handle = core.registry.clone();
|
let registry_handle = core.registry.clone();
|
||||||
let mut registry = registry_handle.borrow_mut();
|
let mut registry = registry_handle.borrow_mut();
|
||||||
|
|
@ -3936,23 +3939,31 @@ mod tests {
|
||||||
pos: 0,
|
pos: 0,
|
||||||
bytes: text,
|
bytes: text,
|
||||||
})
|
})
|
||||||
.expect("seed rust text");
|
.expect("seed syntax text");
|
||||||
}
|
}
|
||||||
let parse_view = crate::syntax::ParseView::new(buf, language, "rust".to_owned());
|
let parse_view = crate::syntax::ParseView::new(buf, language, language_name.to_owned());
|
||||||
let handle = parse_view.handle();
|
let handle = parse_view.handle();
|
||||||
let req = handle.make_request();
|
let req = handle.make_request();
|
||||||
let bundle = crate::syntax::run_parse(req).expect("initial rust parse");
|
let bundle = crate::syntax::run_parse(req).expect("initial syntax parse");
|
||||||
// Mirror the production settle path: resolve each layer's highlight
|
// Mirror the production settle path: queries and lexical facts travel
|
||||||
// query before install so the producer can style it (framing Q#IJ2).
|
// with the same bundle the producer reads.
|
||||||
handle.install(state.syntax_registry.resolve_layer_queries(&bundle));
|
handle.install(state.syntax_registry.resolve_layer_queries(&bundle));
|
||||||
buf.attach_view(Box::new(parse_view));
|
buf.attach_view(Box::new(parse_view));
|
||||||
drop(registry);
|
drop(registry);
|
||||||
core.set_buffer_path(buffer_id, Some(std::path::PathBuf::from("/tmp/x.rs")));
|
core.set_buffer_path(buffer_id, Some(std::path::PathBuf::from(path)));
|
||||||
drop(core);
|
drop(core);
|
||||||
state.syntax_registry.attach_view(buffer_id, handle.clone());
|
state.syntax_registry.attach_view(buffer_id, handle.clone());
|
||||||
handle
|
handle
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn seed_rust_parse_view(
|
||||||
|
state: &EditorState,
|
||||||
|
buffer_id: BufferId,
|
||||||
|
text: &[u8],
|
||||||
|
) -> crate::syntax::ParseViewHandle {
|
||||||
|
seed_parse_view(state, buffer_id, text, "rust", "/tmp/x.rs")
|
||||||
|
}
|
||||||
|
|
||||||
fn seed_markdown_parse_view(
|
fn seed_markdown_parse_view(
|
||||||
state: &EditorState,
|
state: &EditorState,
|
||||||
buffer_id: BufferId,
|
buffer_id: BufferId,
|
||||||
|
|
@ -4037,6 +4048,57 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn viewport_style_producer_uses_settled_local_facts() {
|
||||||
|
let state = empty_state();
|
||||||
|
let buffer_id = active_buffer(&state);
|
||||||
|
let source = b"console;\nfunction f(console) { console; }\n";
|
||||||
|
seed_parse_view(&state, buffer_id, source, "javascript", "/tmp/locals.js");
|
||||||
|
state.syntax_registry.theme().lock().expect("theme").insert(
|
||||||
|
"variable.builtin",
|
||||||
|
Style {
|
||||||
|
fg: crate::cell::Color::Indexed(6),
|
||||||
|
..Style::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
let style_at = |visible: ByteRange, offset: u64| {
|
||||||
|
scoped_style_spans(
|
||||||
|
&state,
|
||||||
|
&DeclaredViewport {
|
||||||
|
buffer_id,
|
||||||
|
visible,
|
||||||
|
frontend_generation: 0,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.into_iter()
|
||||||
|
.find(|span| span.range.start <= offset && offset < span.range.end)
|
||||||
|
.map_or_else(Style::default, |span| span.style)
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
style_at(ByteRange { start: 0, end: 7 }, 0).fg,
|
||||||
|
crate::cell::Color::Indexed(6),
|
||||||
|
"unresolved outer `console` receives the builtin capture"
|
||||||
|
);
|
||||||
|
let inner = source
|
||||||
|
.windows("console".len())
|
||||||
|
.rposition(|window| window == b"console")
|
||||||
|
.expect("inner console") as u64;
|
||||||
|
assert_ne!(
|
||||||
|
style_at(
|
||||||
|
ByteRange {
|
||||||
|
start: inner,
|
||||||
|
end: inner + "console".len() as u64,
|
||||||
|
},
|
||||||
|
inner,
|
||||||
|
)
|
||||||
|
.fg,
|
||||||
|
crate::cell::Color::Indexed(6),
|
||||||
|
"viewport-only highlighting still sees the parameter definition outside the viewport"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn full_buffer_summary_flatten_scales_on_large_grammar_file() {
|
fn full_buffer_summary_flatten_scales_on_large_grammar_file() {
|
||||||
// Perf gate (round-1 finding 1): the file-style summary runs the
|
// Perf gate (round-1 finding 1): the file-style summary runs the
|
||||||
|
|
|
||||||
609
src/syntax.rs
609
src/syntax.rs
|
|
@ -115,6 +115,14 @@ pub struct ParseTreeBundle {
|
||||||
pub injection_capped: bool,
|
pub injection_capped: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Lexically-local identifier ranges derived from a grammar's bundled
|
||||||
|
/// `locals.scm` query. Ranges are sorted and deduplicated so highlight
|
||||||
|
/// predicate checks are allocation-free binary searches.
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub struct LocalFacts {
|
||||||
|
ranges: Box<[(u32, u32)]>,
|
||||||
|
}
|
||||||
|
|
||||||
/// One injection layer within a [`ParseTreeBundle`] (framing Q#IJ1). A
|
/// One injection layer within a [`ParseTreeBundle`] (framing Q#IJ1). A
|
||||||
/// layer pairs a parse tree with the language that produced it and the
|
/// layer pairs a parse tree with the language that produced it and the
|
||||||
/// injection-nesting depth (root = 0). `highlight_query` is resolved on
|
/// injection-nesting depth (root = 0). `highlight_query` is resolved on
|
||||||
|
|
@ -133,6 +141,10 @@ pub struct Layer {
|
||||||
/// `None` when the language ships no highlights, or on the worker
|
/// `None` when the language ships no highlights, or on the worker
|
||||||
/// (pre-settle). Producers read it to style this layer.
|
/// (pre-settle). Producers read it to style this layer.
|
||||||
pub highlight_query: Option<Arc<tree_sitter::Query>>,
|
pub highlight_query: Option<Arc<tree_sitter::Query>>,
|
||||||
|
/// Lexically-local definitions and resolved references for this tree.
|
||||||
|
/// Present only when the highlight query asks about the `local`
|
||||||
|
/// property; computed once when the bundle settles.
|
||||||
|
pub local_facts: Option<Arc<LocalFacts>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ParseTreeBundle {
|
impl ParseTreeBundle {
|
||||||
|
|
@ -184,6 +196,7 @@ pub fn run_parse(req: ParseRequest) -> Result<ParseTreeBundle, String> {
|
||||||
tree: root_tree,
|
tree: root_tree,
|
||||||
depth: 0,
|
depth: 0,
|
||||||
highlight_query: None,
|
highlight_query: None,
|
||||||
|
local_facts: None,
|
||||||
}];
|
}];
|
||||||
let injection_capped =
|
let injection_capped =
|
||||||
build_injection_layers(&mut layers, req.source.as_ref(), &req.injection_aliases);
|
build_injection_layers(&mut layers, req.source.as_ref(), &req.injection_aliases);
|
||||||
|
|
@ -309,6 +322,7 @@ fn build_injection_layers(
|
||||||
tree,
|
tree,
|
||||||
depth: depth + 1,
|
depth: depth + 1,
|
||||||
highlight_query: None,
|
highlight_query: None,
|
||||||
|
local_facts: None,
|
||||||
},
|
},
|
||||||
ranges,
|
ranges,
|
||||||
));
|
));
|
||||||
|
|
@ -773,6 +787,10 @@ pub struct LanguageEntry {
|
||||||
/// slice (or all-empty fragments) means no highlights: the view
|
/// slice (or all-empty fragments) means no highlights: the view
|
||||||
/// runs but emits nothing.
|
/// runs but emits nothing.
|
||||||
pub highlights_query: &'static [&'static str],
|
pub highlights_query: &'static [&'static str],
|
||||||
|
/// Bundled `locals.scm` query fragments, composed base-first like
|
||||||
|
/// [`Self::highlights_query`]. The query supplies lexical scopes,
|
||||||
|
/// definitions, values, and references for `local` property predicates.
|
||||||
|
pub locals_query: &'static [&'static str],
|
||||||
/// Bundled `injections.scm` fragments (framing Q#IJ2), joined with a
|
/// Bundled `injections.scm` fragments (framing Q#IJ2), joined with a
|
||||||
/// newline and compiled on the parse worker to find embedded-language
|
/// newline and compiled on the parse worker to find embedded-language
|
||||||
/// regions. Empty for the many grammars that ship none (or don't
|
/// regions. Empty for the many grammars that ship none (or don't
|
||||||
|
|
@ -801,6 +819,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
extensions: &["rs"],
|
extensions: &["rs"],
|
||||||
loader: || tree_sitter_rust::LANGUAGE.into(),
|
loader: || tree_sitter_rust::LANGUAGE.into(),
|
||||||
highlights_query: &[tree_sitter_rust::HIGHLIGHTS_QUERY],
|
highlights_query: &[tree_sitter_rust::HIGHLIGHTS_QUERY],
|
||||||
|
locals_query: &[],
|
||||||
injections_query: &[tree_sitter_rust::INJECTIONS_QUERY],
|
injections_query: &[tree_sitter_rust::INJECTIONS_QUERY],
|
||||||
},
|
},
|
||||||
LanguageEntry {
|
LanguageEntry {
|
||||||
|
|
@ -808,6 +827,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
extensions: &["lua"],
|
extensions: &["lua"],
|
||||||
loader: || tree_sitter_lua::LANGUAGE.into(),
|
loader: || tree_sitter_lua::LANGUAGE.into(),
|
||||||
highlights_query: &[tree_sitter_lua::HIGHLIGHTS_QUERY],
|
highlights_query: &[tree_sitter_lua::HIGHLIGHTS_QUERY],
|
||||||
|
locals_query: &[tree_sitter_lua::LOCALS_QUERY],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
// T M9.7: markdown block grammar (`tree_sitter_md::LANGUAGE`) — headers,
|
// T M9.7: markdown block grammar (`tree_sitter_md::LANGUAGE`) — headers,
|
||||||
|
|
@ -824,6 +844,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
extensions: &["md", "markdown"],
|
extensions: &["md", "markdown"],
|
||||||
loader: || tree_sitter_md::LANGUAGE.into(),
|
loader: || tree_sitter_md::LANGUAGE.into(),
|
||||||
highlights_query: &[tree_sitter_md::HIGHLIGHT_QUERY_BLOCK],
|
highlights_query: &[tree_sitter_md::HIGHLIGHT_QUERY_BLOCK],
|
||||||
|
locals_query: &[],
|
||||||
injections_query: &[tree_sitter_md::INJECTION_QUERY_BLOCK],
|
injections_query: &[tree_sitter_md::INJECTION_QUERY_BLOCK],
|
||||||
},
|
},
|
||||||
// markdown_inline (framing Q#IJ10) — the inline grammar the block
|
// markdown_inline (framing Q#IJ10) — the inline grammar the block
|
||||||
|
|
@ -837,6 +858,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
extensions: &[],
|
extensions: &[],
|
||||||
loader: || tree_sitter_md::INLINE_LANGUAGE.into(),
|
loader: || tree_sitter_md::INLINE_LANGUAGE.into(),
|
||||||
highlights_query: &[tree_sitter_md::HIGHLIGHT_QUERY_INLINE],
|
highlights_query: &[tree_sitter_md::HIGHLIGHT_QUERY_INLINE],
|
||||||
|
locals_query: &[],
|
||||||
injections_query: &[tree_sitter_md::INJECTION_QUERY_INLINE],
|
injections_query: &[tree_sitter_md::INJECTION_QUERY_INLINE],
|
||||||
},
|
},
|
||||||
// T M_B3 — C / C++. Lexical highlighting (keywords / strings /
|
// T M_B3 — C / C++. Lexical highlighting (keywords / strings /
|
||||||
|
|
@ -859,6 +881,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
extensions: &["c", "h"],
|
extensions: &["c", "h"],
|
||||||
loader: || tree_sitter_c::LANGUAGE.into(),
|
loader: || tree_sitter_c::LANGUAGE.into(),
|
||||||
highlights_query: &[tree_sitter_c::HIGHLIGHT_QUERY],
|
highlights_query: &[tree_sitter_c::HIGHLIGHT_QUERY],
|
||||||
|
locals_query: &[],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
LanguageEntry {
|
LanguageEntry {
|
||||||
|
|
@ -866,6 +889,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
extensions: &["cpp", "cc", "cxx", "hpp", "hh", "hxx", "ipp", "inl", "cppm"],
|
extensions: &["cpp", "cc", "cxx", "hpp", "hh", "hxx", "ipp", "inl", "cppm"],
|
||||||
loader: || tree_sitter_cpp::LANGUAGE.into(),
|
loader: || tree_sitter_cpp::LANGUAGE.into(),
|
||||||
highlights_query: &[tree_sitter_cpp::HIGHLIGHT_QUERY],
|
highlights_query: &[tree_sitter_cpp::HIGHLIGHT_QUERY],
|
||||||
|
locals_query: &[],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
// CUDA (`.cu` source, `.cuh` header). A dedicated grammar rather
|
// CUDA (`.cu` source, `.cuh` header). A dedicated grammar rather
|
||||||
|
|
@ -897,6 +921,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
tree_sitter_cpp::HIGHLIGHT_QUERY,
|
tree_sitter_cpp::HIGHLIGHT_QUERY,
|
||||||
tree_sitter_cuda::HIGHLIGHTS_QUERY,
|
tree_sitter_cuda::HIGHLIGHTS_QUERY,
|
||||||
],
|
],
|
||||||
|
locals_query: &[],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
// Shell / bash. Lexical highlighting for the shell family; the LSP
|
// Shell / bash. Lexical highlighting for the shell family; the LSP
|
||||||
|
|
@ -915,6 +940,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
extensions: &["sh", "bash", "zsh", "ksh", "ash", "bats"],
|
extensions: &["sh", "bash", "zsh", "ksh", "ash", "bats"],
|
||||||
loader: || tree_sitter_bash::LANGUAGE.into(),
|
loader: || tree_sitter_bash::LANGUAGE.into(),
|
||||||
highlights_query: &[tree_sitter_bash::HIGHLIGHT_QUERY],
|
highlights_query: &[tree_sitter_bash::HIGHLIGHT_QUERY],
|
||||||
|
locals_query: &[],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
// Filename-identified languages. These files usually have no useful
|
// Filename-identified languages. These files usually have no useful
|
||||||
|
|
@ -931,6 +957,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
extensions: &["dockerfile", "containerfile"],
|
extensions: &["dockerfile", "containerfile"],
|
||||||
loader: || tree_sitter_containerfile::LANGUAGE.into(),
|
loader: || tree_sitter_containerfile::LANGUAGE.into(),
|
||||||
highlights_query: &[tree_sitter_containerfile::HIGHLIGHTS_QUERY],
|
highlights_query: &[tree_sitter_containerfile::HIGHLIGHTS_QUERY],
|
||||||
|
locals_query: &[],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
LanguageEntry {
|
LanguageEntry {
|
||||||
|
|
@ -938,6 +965,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
extensions: &["mk", "make"],
|
extensions: &["mk", "make"],
|
||||||
loader: || tree_sitter_make::LANGUAGE.into(),
|
loader: || tree_sitter_make::LANGUAGE.into(),
|
||||||
highlights_query: &[tree_sitter_make::HIGHLIGHTS_QUERY],
|
highlights_query: &[tree_sitter_make::HIGHLIGHTS_QUERY],
|
||||||
|
locals_query: &[],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
LanguageEntry {
|
LanguageEntry {
|
||||||
|
|
@ -945,6 +973,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
extensions: &["cmake"],
|
extensions: &["cmake"],
|
||||||
loader: || tree_sitter_cmake::LANGUAGE.into(),
|
loader: || tree_sitter_cmake::LANGUAGE.into(),
|
||||||
highlights_query: &[tree_sitter_cmake::HIGHLIGHTS_QUERY],
|
highlights_query: &[tree_sitter_cmake::HIGHLIGHTS_QUERY],
|
||||||
|
locals_query: &[],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
// Grammar-gap languages — these already had LSP configs but no
|
// Grammar-gap languages — these already had LSP configs but no
|
||||||
|
|
@ -958,6 +987,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
extensions: &["py", "pyi"],
|
extensions: &["py", "pyi"],
|
||||||
loader: || tree_sitter_python::LANGUAGE.into(),
|
loader: || tree_sitter_python::LANGUAGE.into(),
|
||||||
highlights_query: &[tree_sitter_python::HIGHLIGHTS_QUERY],
|
highlights_query: &[tree_sitter_python::HIGHLIGHTS_QUERY],
|
||||||
|
locals_query: &[],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
LanguageEntry {
|
LanguageEntry {
|
||||||
|
|
@ -965,6 +995,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
extensions: &["go"],
|
extensions: &["go"],
|
||||||
loader: || tree_sitter_go::LANGUAGE.into(),
|
loader: || tree_sitter_go::LANGUAGE.into(),
|
||||||
highlights_query: &[tree_sitter_go::HIGHLIGHTS_QUERY],
|
highlights_query: &[tree_sitter_go::HIGHLIGHTS_QUERY],
|
||||||
|
locals_query: &[],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
// JavaScript / TypeScript. One `tree-sitter-javascript` grammar parses
|
// JavaScript / TypeScript. One `tree-sitter-javascript` grammar parses
|
||||||
|
|
@ -980,6 +1011,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
extensions: &["js", "mjs", "cjs"],
|
extensions: &["js", "mjs", "cjs"],
|
||||||
loader: || tree_sitter_javascript::LANGUAGE.into(),
|
loader: || tree_sitter_javascript::LANGUAGE.into(),
|
||||||
highlights_query: &[tree_sitter_javascript::HIGHLIGHT_QUERY],
|
highlights_query: &[tree_sitter_javascript::HIGHLIGHT_QUERY],
|
||||||
|
locals_query: &[tree_sitter_javascript::LOCALS_QUERY],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
LanguageEntry {
|
LanguageEntry {
|
||||||
|
|
@ -990,6 +1022,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
tree_sitter_javascript::HIGHLIGHT_QUERY,
|
tree_sitter_javascript::HIGHLIGHT_QUERY,
|
||||||
tree_sitter_javascript::JSX_HIGHLIGHT_QUERY,
|
tree_sitter_javascript::JSX_HIGHLIGHT_QUERY,
|
||||||
],
|
],
|
||||||
|
locals_query: &[tree_sitter_javascript::LOCALS_QUERY],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
LanguageEntry {
|
LanguageEntry {
|
||||||
|
|
@ -1000,6 +1033,10 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
tree_sitter_javascript::HIGHLIGHT_QUERY,
|
tree_sitter_javascript::HIGHLIGHT_QUERY,
|
||||||
tree_sitter_typescript::HIGHLIGHTS_QUERY,
|
tree_sitter_typescript::HIGHLIGHTS_QUERY,
|
||||||
],
|
],
|
||||||
|
locals_query: &[
|
||||||
|
tree_sitter_javascript::LOCALS_QUERY,
|
||||||
|
tree_sitter_typescript::LOCALS_QUERY,
|
||||||
|
],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
LanguageEntry {
|
LanguageEntry {
|
||||||
|
|
@ -1011,6 +1048,10 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
tree_sitter_javascript::JSX_HIGHLIGHT_QUERY,
|
tree_sitter_javascript::JSX_HIGHLIGHT_QUERY,
|
||||||
tree_sitter_typescript::HIGHLIGHTS_QUERY,
|
tree_sitter_typescript::HIGHLIGHTS_QUERY,
|
||||||
],
|
],
|
||||||
|
locals_query: &[
|
||||||
|
tree_sitter_javascript::LOCALS_QUERY,
|
||||||
|
tree_sitter_typescript::LOCALS_QUERY,
|
||||||
|
],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
LanguageEntry {
|
LanguageEntry {
|
||||||
|
|
@ -1018,6 +1059,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
extensions: &["toml"],
|
extensions: &["toml"],
|
||||||
loader: || tree_sitter_toml_ng::LANGUAGE.into(),
|
loader: || tree_sitter_toml_ng::LANGUAGE.into(),
|
||||||
highlights_query: &[tree_sitter_toml_ng::HIGHLIGHTS_QUERY],
|
highlights_query: &[tree_sitter_toml_ng::HIGHLIGHTS_QUERY],
|
||||||
|
locals_query: &[],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
LanguageEntry {
|
LanguageEntry {
|
||||||
|
|
@ -1025,6 +1067,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
extensions: &["zig", "zon"],
|
extensions: &["zig", "zon"],
|
||||||
loader: || tree_sitter_zig::LANGUAGE.into(),
|
loader: || tree_sitter_zig::LANGUAGE.into(),
|
||||||
highlights_query: &[tree_sitter_zig::HIGHLIGHTS_QUERY],
|
highlights_query: &[tree_sitter_zig::HIGHLIGHTS_QUERY],
|
||||||
|
locals_query: &[],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
// JSON + YAML — config formats, both self-contained highlights and no
|
// JSON + YAML — config formats, both self-contained highlights and no
|
||||||
|
|
@ -1039,6 +1082,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
extensions: &["json"],
|
extensions: &["json"],
|
||||||
loader: || tree_sitter_json::LANGUAGE.into(),
|
loader: || tree_sitter_json::LANGUAGE.into(),
|
||||||
highlights_query: &[tree_sitter_json::HIGHLIGHTS_QUERY],
|
highlights_query: &[tree_sitter_json::HIGHLIGHTS_QUERY],
|
||||||
|
locals_query: &[],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
LanguageEntry {
|
LanguageEntry {
|
||||||
|
|
@ -1046,6 +1090,7 @@ pub const BUILTIN_LANGUAGES: &[LanguageEntry] = &[
|
||||||
extensions: &["yaml", "yml"],
|
extensions: &["yaml", "yml"],
|
||||||
loader: || tree_sitter_yaml::LANGUAGE.into(),
|
loader: || tree_sitter_yaml::LANGUAGE.into(),
|
||||||
highlights_query: &[tree_sitter_yaml::HIGHLIGHTS_QUERY],
|
highlights_query: &[tree_sitter_yaml::HIGHLIGHTS_QUERY],
|
||||||
|
locals_query: &[],
|
||||||
injections_query: &[],
|
injections_query: &[],
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
@ -1076,6 +1121,9 @@ pub struct SyntaxRegistry {
|
||||||
/// compilation failure (e.g. grammar / query ABI skew) is
|
/// compilation failure (e.g. grammar / query ABI skew) is
|
||||||
/// cached as `Err(message)` so we don't burn cycles re-trying.
|
/// cached as `Err(message)` so we don't burn cycles re-trying.
|
||||||
queries: RefCell<HashMap<String, Result<Arc<tree_sitter::Query>, String>>>,
|
queries: RefCell<HashMap<String, Result<Arc<tree_sitter::Query>, String>>>,
|
||||||
|
/// Compiled `locals.scm` query per language. Like `queries`, both
|
||||||
|
/// compilation failures and absent query sources are cached.
|
||||||
|
local_queries: RefCell<HashMap<String, Result<Arc<tree_sitter::Query>, String>>>,
|
||||||
/// Fence-name → canonical-language alias map (framing Q#IJ4). Seeded
|
/// Fence-name → canonical-language alias map (framing Q#IJ4). Seeded
|
||||||
/// with [`default_injection_aliases`]; Lua adds to it through
|
/// with [`default_injection_aliases`]; Lua adds to it through
|
||||||
/// [`Self::register_injection_alias`]. Snapshotted into each
|
/// [`Self::register_injection_alias`]. Snapshotted into each
|
||||||
|
|
@ -1107,6 +1155,7 @@ impl SyntaxRegistry {
|
||||||
parse_jobs: RefCell::new(HashMap::new()),
|
parse_jobs: RefCell::new(HashMap::new()),
|
||||||
extra_extensions: RefCell::new(HashMap::new()),
|
extra_extensions: RefCell::new(HashMap::new()),
|
||||||
queries: RefCell::new(HashMap::new()),
|
queries: RefCell::new(HashMap::new()),
|
||||||
|
local_queries: RefCell::new(HashMap::new()),
|
||||||
injection_aliases: RefCell::new(default_injection_aliases()),
|
injection_aliases: RefCell::new(default_injection_aliases()),
|
||||||
theme: Arc::new(Mutex::new(Theme::default_dark())),
|
theme: Arc::new(Mutex::new(Theme::default_dark())),
|
||||||
}
|
}
|
||||||
|
|
@ -1273,6 +1322,32 @@ impl SyntaxRegistry {
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Lazy-compile and cache the bundled `locals.scm` query for
|
||||||
|
/// `lang_name`. Empty sources and compilation failures are cached.
|
||||||
|
#[must_use]
|
||||||
|
pub fn locals_query(&self, lang_name: &str) -> Option<Arc<tree_sitter::Query>> {
|
||||||
|
if let Some(slot) = self.local_queries.borrow().get(lang_name) {
|
||||||
|
return slot.as_ref().ok().cloned();
|
||||||
|
}
|
||||||
|
let language = self.language(lang_name)?;
|
||||||
|
let entry = BUILTIN_LANGUAGES.iter().find(|e| e.name == lang_name);
|
||||||
|
let source = entry.map_or_else(String::new, |e| e.locals_query.join("\n"));
|
||||||
|
if source.trim().is_empty() {
|
||||||
|
self.local_queries
|
||||||
|
.borrow_mut()
|
||||||
|
.insert(lang_name.to_owned(), Err("no locals query".to_owned()));
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let compiled = tree_sitter::Query::new(&language, &source)
|
||||||
|
.map(Arc::new)
|
||||||
|
.map_err(|e| format!("compile {lang_name} locals: {e:?}"));
|
||||||
|
let result = compiled.as_ref().ok().cloned();
|
||||||
|
self.local_queries
|
||||||
|
.borrow_mut()
|
||||||
|
.insert(lang_name.to_owned(), compiled);
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
/// Add or override a fence-name → language alias (framing Q#IJ4). The
|
/// Add or override a fence-name → language alias (framing Q#IJ4). The
|
||||||
/// alias key is case-folded to match the resolver. Called from Lua via
|
/// alias key is case-folded to match the resolver. Called from Lua via
|
||||||
/// `pmacs.parse.injection_aliases`.
|
/// `pmacs.parse.injection_aliases`.
|
||||||
|
|
@ -1299,11 +1374,26 @@ impl SyntaxRegistry {
|
||||||
let layers = raw
|
let layers = raw
|
||||||
.layers
|
.layers
|
||||||
.iter()
|
.iter()
|
||||||
.map(|l| Layer {
|
.map(|layer| {
|
||||||
language_name: l.language_name.clone(),
|
let highlight_query = self.highlights_query(&layer.language_name);
|
||||||
tree: l.tree.clone(),
|
let local_facts = highlight_query
|
||||||
depth: l.depth,
|
.as_deref()
|
||||||
highlight_query: self.highlights_query(&l.language_name),
|
.filter(|query| query_uses_local_predicates(query))
|
||||||
|
.and_then(|_| self.locals_query(&layer.language_name))
|
||||||
|
.map(|query| {
|
||||||
|
Arc::new(compute_local_facts(
|
||||||
|
&query,
|
||||||
|
&layer.tree,
|
||||||
|
raw.source.as_ref(),
|
||||||
|
))
|
||||||
|
});
|
||||||
|
Layer {
|
||||||
|
language_name: layer.language_name.clone(),
|
||||||
|
tree: layer.tree.clone(),
|
||||||
|
depth: layer.depth,
|
||||||
|
highlight_query,
|
||||||
|
local_facts,
|
||||||
|
}
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
Arc::new(ParseTreeBundle {
|
Arc::new(ParseTreeBundle {
|
||||||
|
|
@ -1316,6 +1406,148 @@ impl SyntaxRegistry {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn query_uses_local_predicates(query: &tree_sitter::Query) -> bool {
|
||||||
|
(0..query.pattern_count()).any(|pattern| {
|
||||||
|
query
|
||||||
|
.property_predicates(pattern)
|
||||||
|
.iter()
|
||||||
|
.any(|(property, _)| property.key.as_ref() == "local")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct LocalDefinition {
|
||||||
|
name_range: std::ops::Range<usize>,
|
||||||
|
value_range: std::ops::Range<usize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct LocalScope {
|
||||||
|
inherits: bool,
|
||||||
|
range: std::ops::Range<usize>,
|
||||||
|
definitions: Vec<LocalDefinition>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LocalFacts {
|
||||||
|
fn contains(&self, start_byte: usize, end_byte: usize) -> bool {
|
||||||
|
let (Ok(start_byte), Ok(end_byte)) = (u32::try_from(start_byte), u32::try_from(end_byte))
|
||||||
|
else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
self.ranges.binary_search(&(start_byte, end_byte)).is_ok()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolve lexical definitions and references according to Tree-sitter's
|
||||||
|
/// standard `locals.scm` capture conventions.
|
||||||
|
fn compute_local_facts(
|
||||||
|
query: &tree_sitter::Query,
|
||||||
|
tree: &tree_sitter::Tree,
|
||||||
|
source: &[u8],
|
||||||
|
) -> LocalFacts {
|
||||||
|
let scope_capture = query.capture_index_for_name("local.scope");
|
||||||
|
let definition_capture = query.capture_index_for_name("local.definition");
|
||||||
|
let value_capture = query.capture_index_for_name("local.definition-value");
|
||||||
|
let reference_capture = query.capture_index_for_name("local.reference");
|
||||||
|
|
||||||
|
let mut scopes = vec![LocalScope {
|
||||||
|
inherits: false,
|
||||||
|
range: 0..source.len(),
|
||||||
|
definitions: Vec::new(),
|
||||||
|
}];
|
||||||
|
let mut ranges = Vec::new();
|
||||||
|
let mut cursor = tree_sitter::QueryCursor::new();
|
||||||
|
let mut captures = cursor.captures(query, tree.root_node(), source);
|
||||||
|
|
||||||
|
while let Some((query_match, capture_index)) = captures.next() {
|
||||||
|
let capture = query_match.captures[*capture_index];
|
||||||
|
let node_range = capture.node.byte_range();
|
||||||
|
while scopes.len() > 1
|
||||||
|
&& node_range.start > scopes.last().expect("root scope exists").range.end
|
||||||
|
{
|
||||||
|
scopes.pop();
|
||||||
|
}
|
||||||
|
|
||||||
|
if Some(capture.index) == scope_capture {
|
||||||
|
let mut inherits = true;
|
||||||
|
for property in query.property_settings(query_match.pattern_index) {
|
||||||
|
if property.key.as_ref() == "local.scope-inherits" {
|
||||||
|
inherits = property
|
||||||
|
.value
|
||||||
|
.as_deref()
|
||||||
|
.is_none_or(|value| value == "true");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
scopes.push(LocalScope {
|
||||||
|
inherits,
|
||||||
|
range: node_range,
|
||||||
|
definitions: Vec::new(),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if Some(capture.index) == definition_capture {
|
||||||
|
let Some(_) = source.get(node_range.clone()) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let value_range = query_match
|
||||||
|
.captures
|
||||||
|
.iter()
|
||||||
|
.find(|candidate| Some(candidate.index) == value_capture)
|
||||||
|
.map_or(0..0, |candidate| candidate.node.byte_range());
|
||||||
|
scopes
|
||||||
|
.last_mut()
|
||||||
|
.expect("root scope exists")
|
||||||
|
.definitions
|
||||||
|
.push(LocalDefinition {
|
||||||
|
name_range: node_range.clone(),
|
||||||
|
value_range,
|
||||||
|
});
|
||||||
|
if let (Ok(start), Ok(end)) = (
|
||||||
|
u32::try_from(node_range.start),
|
||||||
|
u32::try_from(node_range.end),
|
||||||
|
) {
|
||||||
|
ranges.push((start, end));
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if Some(capture.index) != reference_capture {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(name) = source.get(node_range.clone()) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let mut resolved = false;
|
||||||
|
for scope in scopes.iter().rev() {
|
||||||
|
if scope.definitions.iter().rev().any(|definition| {
|
||||||
|
node_range.start >= definition.value_range.end
|
||||||
|
&& source.get(definition.name_range.clone()) == Some(name)
|
||||||
|
}) {
|
||||||
|
resolved = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if !scope.inherits {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if resolved
|
||||||
|
&& let (Ok(start), Ok(end)) = (
|
||||||
|
u32::try_from(node_range.start),
|
||||||
|
u32::try_from(node_range.end),
|
||||||
|
)
|
||||||
|
{
|
||||||
|
ranges.push((start, end));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ranges.sort_unstable();
|
||||||
|
ranges.dedup();
|
||||||
|
LocalFacts {
|
||||||
|
ranges: ranges.into_boxed_slice(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl Default for SyntaxRegistry {
|
impl Default for SyntaxRegistry {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self::new()
|
Self::new()
|
||||||
|
|
@ -1418,20 +1650,22 @@ pub fn compute_highlight_spans_in_range(
|
||||||
query,
|
query,
|
||||||
bundle.root_tree(),
|
bundle.root_tree(),
|
||||||
bundle.source.as_ref(),
|
bundle.source.as_ref(),
|
||||||
|
bundle.layers[0].local_facts.as_deref(),
|
||||||
byte_range,
|
byte_range,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Like [`compute_highlight_spans_in_range`] but over an explicit
|
/// Like [`compute_highlight_spans_in_range`] but over an explicit
|
||||||
/// `(tree, source)` — the per-layer form the producers call for each
|
/// `(tree, source, local_facts)` layer tuple — the form producers call for
|
||||||
/// injection layer (framing Q#IJ7). `source` is the whole buffer; a
|
/// each injection layer (framing Q#IJ7 and Q#LQ5). `source` is the whole
|
||||||
/// child layer's tree carries absolute offsets into it, so the same
|
/// buffer; a child layer's tree carries absolute offsets into it, so the same
|
||||||
/// capture walk works unchanged.
|
/// capture walk works unchanged.
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn compute_highlight_spans_for(
|
pub fn compute_highlight_spans_for(
|
||||||
query: &tree_sitter::Query,
|
query: &tree_sitter::Query,
|
||||||
tree: &tree_sitter::Tree,
|
tree: &tree_sitter::Tree,
|
||||||
source: &[u8],
|
source: &[u8],
|
||||||
|
local_facts: Option<&LocalFacts>,
|
||||||
byte_range: Option<std::ops::Range<usize>>,
|
byte_range: Option<std::ops::Range<usize>>,
|
||||||
) -> Vec<HighlightSpan> {
|
) -> Vec<HighlightSpan> {
|
||||||
let mut spans = Vec::new();
|
let mut spans = Vec::new();
|
||||||
|
|
@ -1441,30 +1675,33 @@ pub fn compute_highlight_spans_for(
|
||||||
}
|
}
|
||||||
let root = tree.root_node();
|
let root = tree.root_node();
|
||||||
let mut iter = cursor.captures(query, root, source);
|
let mut iter = cursor.captures(query, root, source);
|
||||||
while let Some((qmatch, capture_idx)) = iter.next() {
|
while let Some((query_match, capture_index)) = iter.next() {
|
||||||
// Fail-closed on the locals property predicate. The capture
|
let capture = query_match.captures[*capture_index];
|
||||||
// iterator already applies text predicates (`#eq?`/`#match?`/
|
let local_predicates_match = query
|
||||||
// `#any-of?`), but `#is? local` / `#is-not? local` are *property*
|
.property_predicates(query_match.pattern_index)
|
||||||
// predicates (`Query::property_predicates`) that need a scope map
|
|
||||||
// built from the grammar's LOCALS_QUERY, which pmacs does not run.
|
|
||||||
// Applying such a capture regardless mis-styles shadowed locals —
|
|
||||||
// e.g. a local `console`/`require` in JS/TS would still capture as
|
|
||||||
// `@variable.builtin`/`@function.builtin`. Until locals processing
|
|
||||||
// exists, drop captures whose pattern carries one; the identifier
|
|
||||||
// falls back to its non-builtin capture. `#set!` (property
|
|
||||||
// *settings*) is a different API and is not consulted here.
|
|
||||||
if query
|
|
||||||
.property_predicates(qmatch.pattern_index)
|
|
||||||
.iter()
|
.iter()
|
||||||
.any(|(prop, _)| &*prop.key == "local")
|
.filter(|(property, _)| property.key.as_ref() == "local")
|
||||||
{
|
.all(|(property, positive)| {
|
||||||
|
let node = property.capture_id.map_or(Some(capture.node), |target| {
|
||||||
|
query_match
|
||||||
|
.captures
|
||||||
|
.iter()
|
||||||
|
.find(|candidate| candidate.index as usize == target)
|
||||||
|
.map(|candidate| candidate.node)
|
||||||
|
});
|
||||||
|
let is_local = node.is_some_and(|node| {
|
||||||
|
local_facts
|
||||||
|
.is_some_and(|facts| facts.contains(node.start_byte(), node.end_byte()))
|
||||||
|
});
|
||||||
|
is_local == *positive
|
||||||
|
});
|
||||||
|
if !local_predicates_match {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let cap = qmatch.captures[*capture_idx];
|
|
||||||
spans.push(HighlightSpan {
|
spans.push(HighlightSpan {
|
||||||
start_byte: cap.node.start_byte() as u32,
|
start_byte: capture.node.start_byte() as u32,
|
||||||
end_byte: cap.node.end_byte() as u32,
|
end_byte: capture.node.end_byte() as u32,
|
||||||
capture_index: cap.index,
|
capture_index: capture.index,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
// Wider-first ordering at equal start: later writes (the
|
// Wider-first ordering at equal start: later writes (the
|
||||||
|
|
@ -1684,7 +1921,13 @@ mod tests {
|
||||||
.highlight_query
|
.highlight_query
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.expect("inline highlights resolved at settle");
|
.expect("inline highlights resolved at settle");
|
||||||
let spans = compute_highlight_spans_for(hquery, &inline.tree, &bundle.source, None);
|
let spans = compute_highlight_spans_for(
|
||||||
|
hquery,
|
||||||
|
&inline.tree,
|
||||||
|
&bundle.source,
|
||||||
|
inline.local_facts.as_deref(),
|
||||||
|
None,
|
||||||
|
);
|
||||||
assert!(
|
assert!(
|
||||||
!spans.is_empty(),
|
!spans.is_empty(),
|
||||||
"the inline layer produces highlight spans across both ranges"
|
"the inline layer produces highlight spans across both ranges"
|
||||||
|
|
@ -2221,7 +2464,13 @@ mod tests {
|
||||||
.highlight_query
|
.highlight_query
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.expect("yaml highlights resolved");
|
.expect("yaml highlights resolved");
|
||||||
let spans = compute_highlight_spans_for(query, &yaml.tree, &bundle.source, None);
|
let spans = compute_highlight_spans_for(
|
||||||
|
query,
|
||||||
|
&yaml.tree,
|
||||||
|
&bundle.source,
|
||||||
|
yaml.local_facts.as_deref(),
|
||||||
|
None,
|
||||||
|
);
|
||||||
assert!(!spans.is_empty(), "the yaml frontmatter layer highlights");
|
assert!(!spans.is_empty(), "the yaml frontmatter layer highlights");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2418,54 +2667,270 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn javascript_shadowed_builtin_is_not_mislabeled() {
|
fn javascript_local_predicates_distinguish_lexical_scope() {
|
||||||
// `#is-not? local` (JS/TS use it for console/require/etc.) needs a
|
let registry = SyntaxRegistry::new();
|
||||||
// scope map from the LOCALS_QUERY we don't run, so
|
let source = b"console.log('outer');\n\
|
||||||
// `compute_highlight_spans` drops captures guarded by it. Here
|
require('outer');\n\
|
||||||
// `console` is a LOCAL declaration — it must not surface as a
|
function f(console, require) {\n\
|
||||||
// `*.builtin` capture (which is what a naive run of the shared JS
|
console.log('inner');\n\
|
||||||
// query would produce).
|
require('inner');\n\
|
||||||
let reg = SyntaxRegistry::new();
|
}\n\
|
||||||
let language = reg.language("javascript").expect("javascript loads");
|
window.alert('outer');\n";
|
||||||
let query = reg
|
let bundle = parse_layered(®istry, "javascript", source);
|
||||||
.highlights_query("javascript")
|
let layer = &bundle.layers[0];
|
||||||
|
let query = layer
|
||||||
|
.highlight_query
|
||||||
|
.as_deref()
|
||||||
.expect("javascript highlights compile");
|
.expect("javascript highlights compile");
|
||||||
let mut buf = fresh_buffer("shadow.js");
|
|
||||||
buf.apply_edit(EditOp::Insert {
|
|
||||||
pos: 0,
|
|
||||||
bytes: b"const console = 5;\nconsole;\n",
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
let view = ParseView::new(&buf, language, "javascript".to_owned());
|
|
||||||
let handle = view.handle();
|
|
||||||
let _vid = buf.attach_view(Box::new(view));
|
|
||||||
let bundle = parse_synchronously(&handle);
|
|
||||||
let spans = compute_highlight_spans(&query, &bundle);
|
|
||||||
assert!(!spans.is_empty(), "the JS query produced highlight spans");
|
|
||||||
let names = query.capture_names();
|
|
||||||
let builtin: Vec<&str> = spans
|
|
||||||
.iter()
|
|
||||||
.map(|s| names[s.capture_index as usize])
|
|
||||||
.filter(|n| n.contains("builtin"))
|
|
||||||
.collect();
|
|
||||||
assert!(
|
assert!(
|
||||||
builtin.is_empty(),
|
layer.local_facts.is_some(),
|
||||||
"a locally-shadowed `console` must not get a *.builtin capture; got {builtin:?}"
|
"a local-sensitive highlight query must settle lexical facts"
|
||||||
);
|
);
|
||||||
// ...and dropping the builtin pattern must not strip *all* styling:
|
let spans = compute_highlight_spans(query, &bundle);
|
||||||
// each `console` occurrence still keeps its ordinary `@variable`
|
let names = query.capture_names();
|
||||||
// capture (the fallback), so the loss is only the `.builtin` refine.
|
let captures_at = |start: usize, len: usize| -> Vec<&str> {
|
||||||
let src = "const console = 5;\nconsole;\n";
|
spans
|
||||||
for (pos, _) in src.match_indices("console") {
|
|
||||||
let (start, end) = (pos as u32, (pos + "console".len()) as u32);
|
|
||||||
let caps: Vec<&str> = spans
|
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|s| s.start_byte == start && s.end_byte == end)
|
.filter(|span| {
|
||||||
.map(|s| names[s.capture_index as usize])
|
span.start_byte == start as u32 && span.end_byte == (start + len) as u32
|
||||||
|
})
|
||||||
|
.map(|span| names[span.capture_index as usize])
|
||||||
|
.collect()
|
||||||
|
};
|
||||||
|
|
||||||
|
for identifier in ["console", "require"] {
|
||||||
|
let positions: Vec<usize> = std::str::from_utf8(source)
|
||||||
|
.expect("fixture is UTF-8")
|
||||||
|
.match_indices(identifier)
|
||||||
|
.map(|(position, _)| position)
|
||||||
.collect();
|
.collect();
|
||||||
|
assert_eq!(positions.len(), 3, "fixture has three `{identifier}` uses");
|
||||||
assert!(
|
assert!(
|
||||||
caps.iter().any(|n| n.starts_with("variable")),
|
captures_at(positions[0], identifier.len())
|
||||||
"`console` at byte {pos} keeps a variable capture; got {caps:?}"
|
.iter()
|
||||||
|
.any(|name| name.ends_with(".builtin")),
|
||||||
|
"unshadowed outer `{identifier}` keeps its builtin refinement"
|
||||||
|
);
|
||||||
|
for position in &positions[1..] {
|
||||||
|
let captures = captures_at(*position, identifier.len());
|
||||||
|
assert!(
|
||||||
|
!captures.iter().any(|name| name.ends_with(".builtin")),
|
||||||
|
"local `{identifier}` at byte {position} is not builtin: {captures:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
captures.iter().any(|name| name.starts_with("variable")),
|
||||||
|
"local `{identifier}` keeps an ordinary variable capture: {captures:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let window = std::str::from_utf8(source)
|
||||||
|
.expect("fixture is UTF-8")
|
||||||
|
.find("window")
|
||||||
|
.expect("window fixture");
|
||||||
|
assert!(
|
||||||
|
captures_at(window, "window".len())
|
||||||
|
.iter()
|
||||||
|
.any(|name| name == &"variable.builtin"),
|
||||||
|
"an unresolved builtin after the function remains builtin"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn positive_and_capture_qualified_local_predicates_use_resolved_facts() {
|
||||||
|
let registry = SyntaxRegistry::new();
|
||||||
|
let source = b"let f = () => {};\nf();\ng();\n";
|
||||||
|
let bundle = parse_layered(®istry, "javascript", source);
|
||||||
|
let language = registry.language("javascript").expect("javascript loads");
|
||||||
|
let facts = bundle.layers[0]
|
||||||
|
.local_facts
|
||||||
|
.as_deref()
|
||||||
|
.expect("javascript local facts settle");
|
||||||
|
|
||||||
|
let positive = tree_sitter::Query::new(&language, "((identifier) @local-id (#is? local))")
|
||||||
|
.expect("positive local predicate compiles");
|
||||||
|
let positive_spans =
|
||||||
|
compute_highlight_spans_for(&positive, bundle.root_tree(), source, Some(facts), None);
|
||||||
|
let f_positions: Vec<usize> = std::str::from_utf8(source)
|
||||||
|
.expect("fixture is UTF-8")
|
||||||
|
.match_indices('f')
|
||||||
|
.map(|(position, _)| position)
|
||||||
|
.collect();
|
||||||
|
assert_eq!(f_positions.len(), 2);
|
||||||
|
for position in f_positions {
|
||||||
|
assert!(
|
||||||
|
positive_spans.iter().any(|span| {
|
||||||
|
span.start_byte == position as u32 && span.end_byte == (position + 1) as u32
|
||||||
|
}),
|
||||||
|
"definition/reference `f` at byte {position} is local"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let g_position = std::str::from_utf8(source)
|
||||||
|
.expect("fixture is UTF-8")
|
||||||
|
.find("g()")
|
||||||
|
.expect("g call");
|
||||||
|
assert!(
|
||||||
|
positive_spans
|
||||||
|
.iter()
|
||||||
|
.all(|span| span.start_byte != g_position as u32),
|
||||||
|
"unresolved `g` does not satisfy #is? local"
|
||||||
|
);
|
||||||
|
|
||||||
|
let qualified = tree_sitter::Query::new(
|
||||||
|
&language,
|
||||||
|
"((call_expression function: (identifier) @callee) @call \
|
||||||
|
(#is? @callee local))",
|
||||||
|
)
|
||||||
|
.expect("capture-qualified local predicate compiles");
|
||||||
|
let qualified_spans =
|
||||||
|
compute_highlight_spans_for(&qualified, bundle.root_tree(), source, Some(facts), None);
|
||||||
|
let qualified_names = qualified.capture_names();
|
||||||
|
assert!(
|
||||||
|
qualified_spans.iter().any(|span| {
|
||||||
|
qualified_names[span.capture_index as usize] == "call"
|
||||||
|
&& span.start_byte
|
||||||
|
== source
|
||||||
|
.windows(4)
|
||||||
|
.position(|window| window == b"f();")
|
||||||
|
.expect("f call") as u32
|
||||||
|
}),
|
||||||
|
"the call whose @callee is local satisfies the qualified predicate"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
qualified_spans
|
||||||
|
.iter()
|
||||||
|
.all(|span| span.start_byte != g_position as u32),
|
||||||
|
"the call whose @callee is unresolved fails the qualified predicate"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn local_definition_value_and_scope_inheritance_control_resolution() {
|
||||||
|
let registry = SyntaxRegistry::new();
|
||||||
|
let language = registry.language("javascript").expect("javascript loads");
|
||||||
|
|
||||||
|
let value_source = b"let x = x;\nx;\n";
|
||||||
|
let value_tree = {
|
||||||
|
let mut parser = tree_sitter::Parser::new();
|
||||||
|
parser
|
||||||
|
.set_language(&language)
|
||||||
|
.expect("set javascript language");
|
||||||
|
parser
|
||||||
|
.parse(value_source, None)
|
||||||
|
.expect("parse value fixture")
|
||||||
|
};
|
||||||
|
let value_locals = tree_sitter::Query::new(
|
||||||
|
&language,
|
||||||
|
"(variable_declarator \
|
||||||
|
name: (identifier) @local.definition \
|
||||||
|
value: (identifier) @local.definition-value) \
|
||||||
|
(identifier) @local.reference",
|
||||||
|
)
|
||||||
|
.expect("definition-value locals query compiles");
|
||||||
|
let value_facts = compute_local_facts(&value_locals, &value_tree, value_source);
|
||||||
|
let x_positions: Vec<usize> = std::str::from_utf8(value_source)
|
||||||
|
.expect("fixture is UTF-8")
|
||||||
|
.match_indices('x')
|
||||||
|
.map(|(position, _)| position)
|
||||||
|
.collect();
|
||||||
|
assert_eq!(x_positions.len(), 3);
|
||||||
|
assert!(value_facts.contains(x_positions[0], x_positions[0] + 1));
|
||||||
|
assert!(
|
||||||
|
!value_facts.contains(x_positions[1], x_positions[1] + 1),
|
||||||
|
"a definition is not visible inside its own value"
|
||||||
|
);
|
||||||
|
assert!(value_facts.contains(x_positions[2], x_positions[2] + 1));
|
||||||
|
|
||||||
|
let scope_source = b"let x = 1;\nfunction f() { x; }\nx;\n";
|
||||||
|
let scope_tree = {
|
||||||
|
let mut parser = tree_sitter::Parser::new();
|
||||||
|
parser
|
||||||
|
.set_language(&language)
|
||||||
|
.expect("set javascript language");
|
||||||
|
parser
|
||||||
|
.parse(scope_source, None)
|
||||||
|
.expect("parse scope fixture")
|
||||||
|
};
|
||||||
|
let scope_locals = tree_sitter::Query::new(
|
||||||
|
&language,
|
||||||
|
"((function_declaration) @local.scope \
|
||||||
|
(#set! local.scope-inherits false)) \
|
||||||
|
(variable_declarator name: (identifier) @local.definition) \
|
||||||
|
(identifier) @local.reference",
|
||||||
|
)
|
||||||
|
.expect("non-inheriting locals query compiles");
|
||||||
|
let scope_facts = compute_local_facts(&scope_locals, &scope_tree, scope_source);
|
||||||
|
let x_positions: Vec<usize> = std::str::from_utf8(scope_source)
|
||||||
|
.expect("fixture is UTF-8")
|
||||||
|
.match_indices('x')
|
||||||
|
.map(|(position, _)| position)
|
||||||
|
.collect();
|
||||||
|
assert_eq!(x_positions.len(), 3);
|
||||||
|
assert!(scope_facts.contains(x_positions[0], x_positions[0] + 1));
|
||||||
|
assert!(
|
||||||
|
!scope_facts.contains(x_positions[1], x_positions[1] + 1),
|
||||||
|
"a non-inheriting scope cannot see the outer `x`"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
scope_facts.contains(x_positions[2], x_positions[2] + 1),
|
||||||
|
"leaving the scope restores outer resolution"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn typescript_locals_compose_javascript_scopes_and_parameter_delta() {
|
||||||
|
let registry = SyntaxRegistry::new();
|
||||||
|
for (language_name, source) in [
|
||||||
|
(
|
||||||
|
"typescript",
|
||||||
|
&b"function f(console: string) { console.log('x'); }\n\
|
||||||
|
window.alert('x');\n"[..],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"typescriptreact",
|
||||||
|
&b"function F(console: string) { return <div>{console}</div>; }\n\
|
||||||
|
window.alert('x');\n"[..],
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let locals = registry
|
||||||
|
.locals_query(language_name)
|
||||||
|
.unwrap_or_else(|| panic!("{language_name} locals compile"));
|
||||||
|
assert!(
|
||||||
|
locals.capture_index_for_name("local.scope").is_some()
|
||||||
|
&& locals.capture_index_for_name("local.definition").is_some()
|
||||||
|
&& locals.capture_index_for_name("local.reference").is_some(),
|
||||||
|
"{language_name} includes JavaScript's scopes and references"
|
||||||
|
);
|
||||||
|
|
||||||
|
let bundle = parse_layered(®istry, language_name, source);
|
||||||
|
let layer = &bundle.layers[0];
|
||||||
|
let query = layer
|
||||||
|
.highlight_query
|
||||||
|
.as_deref()
|
||||||
|
.expect("highlights compile");
|
||||||
|
let spans = compute_highlight_spans(query, &bundle);
|
||||||
|
let names = query.capture_names();
|
||||||
|
let text = std::str::from_utf8(source).expect("fixture is UTF-8");
|
||||||
|
for (position, _) in text.match_indices("console") {
|
||||||
|
assert!(
|
||||||
|
spans
|
||||||
|
.iter()
|
||||||
|
.filter(|span| {
|
||||||
|
span.start_byte == position as u32
|
||||||
|
&& span.end_byte == (position + "console".len()) as u32
|
||||||
|
})
|
||||||
|
.all(|span| !names[span.capture_index as usize].ends_with(".builtin")),
|
||||||
|
"{language_name} parameter/reference `console` is local"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let window = text.find("window").expect("window fixture");
|
||||||
|
assert!(
|
||||||
|
spans.iter().any(|span| {
|
||||||
|
span.start_byte == window as u32
|
||||||
|
&& span.end_byte == (window + "window".len()) as u32
|
||||||
|
&& names[span.capture_index as usize] == "variable.builtin"
|
||||||
|
}),
|
||||||
|
"{language_name} unresolved `window` remains builtin"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -771,6 +771,98 @@ fn m4_3_theming_via_lua_color_scheme() {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Locals-query acceptance: the shipped JavaScript grammar must distinguish an
|
||||||
|
/// unresolved builtin from a lexically-shadowed parameter, then replace the
|
||||||
|
/// classification with the fresh parse bundle after an edit removes the
|
||||||
|
/// shadow. The theme maps only `variable.builtin`, making the classification
|
||||||
|
/// observable in rendered cells rather than through an internal scope map.
|
||||||
|
#[test]
|
||||||
|
fn m4_locals_query_shadowing_and_edit_freshness() {
|
||||||
|
use pmacs::cell::Color;
|
||||||
|
|
||||||
|
const COLS: usize = 40;
|
||||||
|
const BUILTIN: Color = Color::Indexed(6);
|
||||||
|
let initial = "console;\nfunction f(console) {\n console;\n}\n";
|
||||||
|
let edited = "console;\nfunction f(logger) {\n console;\n}\n";
|
||||||
|
let dir = tempfile::tempdir().expect("tempdir");
|
||||||
|
let path = dir.path().join("locals.js");
|
||||||
|
std::fs::write(&path, initial.as_bytes()).expect("write JavaScript fixture");
|
||||||
|
|
||||||
|
let mut state = open_and_wait_for_parse(path);
|
||||||
|
state
|
||||||
|
.lua_host
|
||||||
|
.lua()
|
||||||
|
.load(
|
||||||
|
r#"
|
||||||
|
pmacs.theme.set {
|
||||||
|
["variable.builtin"] = { fg = 6, bold = true },
|
||||||
|
}
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.exec()
|
||||||
|
.expect("apply builtin-only theme");
|
||||||
|
|
||||||
|
let before = render_active_window_to_grid(&mut state, 5, COLS as u32);
|
||||||
|
for (col, cell) in before.iter().take(7).enumerate() {
|
||||||
|
assert_eq!(
|
||||||
|
cell.style.fg, BUILTIN,
|
||||||
|
"unshadowed row-0 `console` byte {col} is builtin"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
cell.style.bold,
|
||||||
|
"builtin-only theme reaches row-0 `console` byte {col}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for col in 11..18 {
|
||||||
|
assert_ne!(
|
||||||
|
before[COLS + col].style.fg,
|
||||||
|
BUILTIN,
|
||||||
|
"parameter `console` byte {col} is lexically local"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for col in 2..9 {
|
||||||
|
assert_ne!(
|
||||||
|
before[2 * COLS + col].style.fg,
|
||||||
|
BUILTIN,
|
||||||
|
"reference to shadowing parameter at row 2, col {col} is not builtin"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
state
|
||||||
|
.lua_host
|
||||||
|
.lua()
|
||||||
|
.load(
|
||||||
|
r"
|
||||||
|
local buf = pmacs.window.buffer()
|
||||||
|
buf:replace(20, 27, 'logger')
|
||||||
|
pmacs.parse._dispatch(buf, 'javascript')
|
||||||
|
",
|
||||||
|
)
|
||||||
|
.exec()
|
||||||
|
.expect("rename shadowing parameter");
|
||||||
|
pump_async(&mut state, |s| {
|
||||||
|
current_tree_text(s).as_deref() == Some(edited)
|
||||||
|
});
|
||||||
|
assert_eq!(
|
||||||
|
current_tree_text(&state).as_deref(),
|
||||||
|
Some(edited),
|
||||||
|
"the edited JavaScript parse settled"
|
||||||
|
);
|
||||||
|
|
||||||
|
let after = render_active_window_to_grid(&mut state, 5, COLS as u32);
|
||||||
|
for col in 2..9 {
|
||||||
|
assert_eq!(
|
||||||
|
after[2 * COLS + col].style.fg,
|
||||||
|
BUILTIN,
|
||||||
|
"fresh local facts restore builtin styling at row 2, col {col}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
after[2 * COLS + col].style.bold,
|
||||||
|
"fresh builtin capture reaches the rendered cell at row 2, col {col}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Sanity: the bundled `default_dark` theme produces a non-empty
|
/// Sanity: the bundled `default_dark` theme produces a non-empty
|
||||||
/// capture map and resolves common captures to non-default styles.
|
/// capture map and resolves common captures to non-default styles.
|
||||||
/// Catches accidental regressions to a literally empty theme that
|
/// Catches accidental regressions to a literally empty theme that
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue