Merge pull request #122 from levineuwirth/multi-language-injections
feat(injections): multi-language injection layers
This commit is contained in:
GitHub
commit
5e73966316
|
|
@ -22,6 +22,10 @@ local inflight_parse_by_buffer = {}
|
|||
local parse_buffer_by_key = {}
|
||||
local parse_lang_by_buffer = {}
|
||||
local reparse_requested_by_buffer = {}
|
||||
-- Buffers already warned about hitting the injection layer cap (Q#IJ3);
|
||||
-- keyed like the others so we warn once, and re-arm if the file stops
|
||||
-- capping (an edit removed the excess regions).
|
||||
local injection_cap_warned = {}
|
||||
|
||||
local raw_dispatch = pmacs.parse._dispatch
|
||||
|
||||
|
|
@ -44,6 +48,20 @@ function pmacs.parse._dispatch(buf, lang)
|
|||
return job_id
|
||||
end
|
||||
|
||||
-- Injection language aliases (framing Q#IJ4). The registry holds the
|
||||
-- merged map (seeded with defaults on the Rust side), and each dispatch
|
||||
-- snapshots it into the parse request so the worker can resolve dynamic
|
||||
-- fence names (`py` → python, `ts` → typescript). Exposed as a
|
||||
-- write-through proxy so users add fence-name aliases from init.lua:
|
||||
-- pmacs.parse.injection_aliases.mylang = "rust"
|
||||
-- Reads are not proxied (the canonical map lives Rust-side); this is a
|
||||
-- write-only extension surface.
|
||||
pmacs.parse.injection_aliases = setmetatable({}, {
|
||||
__newindex = function(_, alias, lang)
|
||||
pmacs.parse._register_injection_alias(alias, lang)
|
||||
end,
|
||||
})
|
||||
|
||||
-- Shebang → language detection ------------------------------------------
|
||||
--
|
||||
-- Extension detection (`language_for_path`) misses extensionless scripts
|
||||
|
|
@ -324,6 +342,19 @@ pmacs._async.tick = function(...)
|
|||
if pmacs._async._is_complete(job_id) then
|
||||
local key = parse_job_buffer_keys[job_id]
|
||||
pmacs.parse._install_settled(job_id)
|
||||
-- Surface the injection layer backstop (Q#IJ3) once per buffer rather
|
||||
-- than dropping embedded regions silently. Best-effort: a missing
|
||||
-- buffer or error here must not stall the settle loop.
|
||||
local capped_buf = key and parse_buffer_by_key[key]
|
||||
if capped_buf and pmacs.parse._injection_capped(capped_buf) then
|
||||
if not injection_cap_warned[key] and pmacs.error then
|
||||
injection_cap_warned[key] = true
|
||||
pmacs.error(
|
||||
"syntax: injection layer cap reached; some embedded regions are unhighlighted")
|
||||
end
|
||||
elseif key then
|
||||
injection_cap_warned[key] = nil
|
||||
end
|
||||
pending_parse_jobs[job_id] = nil
|
||||
parse_job_buffer_keys[job_id] = nil
|
||||
if key and inflight_parse_by_buffer[key] == job_id then
|
||||
|
|
|
|||
|
|
@ -0,0 +1,387 @@
|
|||
# Multi-language injections — framing (side quest, highlight family)
|
||||
|
||||
**Intent.** Teach the syntax engine that one buffer can hold more than
|
||||
one language. Today pmacs parses a buffer with exactly one grammar and
|
||||
runs exactly one `highlights.scm` over it ("first language wins"). This
|
||||
adds tree-sitter *injection layers*: after the root parse, run the
|
||||
grammar's `injections.scm` to find embedded regions (a markdown code
|
||||
fence, an HTML `<script>`, a rust `macro!` body), parse each with the
|
||||
injected language's grammar, and merge every layer's highlight spans.
|
||||
It is the load-bearing enabler the side-quest backlog names: it unlocks
|
||||
markdown fenced code, embedded languages, and is the honest gate on the
|
||||
per-cell notebook path.
|
||||
|
||||
The first *consumer* that ships with the engine is **markdown fenced
|
||||
code blocks plus inline markdown**, chosen because it needs **zero new
|
||||
grammars**: the markdown block grammar is already bundled and already
|
||||
ships an injection query, and every language a fence can name (rust,
|
||||
python, bash, js, ts, go, toml, …) already has a grammar from the
|
||||
grammar-gap PR (#118). The engine is grammar-agnostic; other injection
|
||||
sites (HTML embedding, JS template literals, comment-embedded langs)
|
||||
become follow-ups as their grammars land.
|
||||
|
||||
---
|
||||
|
||||
## Ground truth (as of `main` @ `0ba01fe`, #119)
|
||||
|
||||
The syntax stack assumes **one tree, one language, one query per
|
||||
buffer** end to end. Every layer of that assumption has to grow.
|
||||
|
||||
- **`ParseView` / `ParseTreeBundle`** (`src/syntax.rs`) — the per-buffer
|
||||
parse state holds a single `language`, source mirror, pending-`InputEdit`
|
||||
list, and `current: Option<Arc<ParseTreeBundle>>`. `ParseTreeBundle`
|
||||
holds one `tree`, one `source`, one `language_name`. `run_parse`
|
||||
(`syntax.rs:106`) sets one language and returns one tree.
|
||||
- **Highlight query** — `SyntaxRegistry::highlights_query(name)`
|
||||
(`syntax.rs:655`) lazily compiles and caches **one** `highlights.scm`
|
||||
per language into the main-thread `Rc<SyntaxRegistry>`. `LanguageEntry`
|
||||
(`syntax.rs:318`) carries `highlights_query: &'static [&'static str]`
|
||||
but **no injection query**.
|
||||
- **Two style producers, both single-tree:**
|
||||
- Grid/TUI: `SyntaxHighlightView` (`src/highlight.rs:289`) is
|
||||
constructed with **one** `Arc<Query>` and caches spans for the one
|
||||
`parse.current()` bundle, invalidated by `Arc::ptr_eq`.
|
||||
- Daemon→GPU wire: `scoped_style_spans` (`src/semantic_render.rs:1640`)
|
||||
reads the one `bundle`, the one `highlights_query(language_name)`,
|
||||
runs `compute_highlight_spans_in_range` scoped to the viewport, maps
|
||||
captures→theme→`StyleSpan`. Perf-gated by `StyleGate`
|
||||
(`semantic_render.rs:1612`) keyed on the bundle `Arc`.
|
||||
- **The wire path re-sorts spans by start — producer order is NOT
|
||||
preserved.** The GPU applies `StyleSpans` through
|
||||
`replace_style_spans` (full, `pmacs-gpu/src/main.rs:4112` — collects
|
||||
segments then `sort_by_key(range.start)`) and `merge_style_spans`
|
||||
(incremental, `main.rs:4130` — clips, appends, re-sorts). So any
|
||||
"emit root-then-depth order and rely on it downstream" scheme is dead
|
||||
on arrival; overlaps must be resolved **before** the wire (Q#IJ6).
|
||||
- **The two `StyleSpan` consumers also disagree on overlap:**
|
||||
`SemanticModel::effective_style_at` (`src/semantic_client.rs:334`)
|
||||
folds **every** covering span via `merge_styles`; the GPU
|
||||
`source_color_at` (`pmacs-gpu/src/main.rs:6718`) returns the **first**
|
||||
covering span's fg and stops. Both are addressed by Q#IJ6.
|
||||
- **Policy A** — grammar-backed buffer ⇒ styled solely by tree-sitter;
|
||||
else solely by LSP tokens; never both on the wire
|
||||
(`semantic_render.rs:1641`).
|
||||
- **Dispatch/settle** (`builtin/runtime/syntax.lua`) — one language per
|
||||
buffer, pinned at first attach; the `_dispatch` wrapper seam is
|
||||
`syntax.lua:26`. The async tick installs the settled bundle.
|
||||
- **Worker discipline** — `ParseRequest` is fully owned (`Send`); parsing
|
||||
runs on a worker. The registry (`Rc`) and any Lua table are main-thread
|
||||
only — neither highlight-query resolution nor Lua-set config can happen
|
||||
on the worker (Q#IJ2, Q#IJ4).
|
||||
|
||||
**Confirmed tree-sitter mechanics this design rests on** (tree-sitter
|
||||
0.26, verified against vendored sources + the tree-sitter injection
|
||||
docs):
|
||||
|
||||
1. `Parser::set_included_ranges(&[Range])` restricts a parse to given
|
||||
byte ranges of the **full source**; resulting node offsets stay
|
||||
**absolute** into the full buffer — injected spans are already in
|
||||
buffer coordinates. Ranges must be **sorted, non-overlapping,
|
||||
non-empty** or the call returns `IncludedRangesError`.
|
||||
2. `tree_sitter_md::INJECTION_QUERY_BLOCK` exists on the already-bundled
|
||||
crate, using **both** forms: dynamic `(info_string (language)
|
||||
@injection.language)` + `(code_fence_content) @injection.content`, and
|
||||
static `((inline) @injection.content (#set! injection.language
|
||||
"markdown_inline"))` (also `html`/`yaml`/`toml`).
|
||||
3. **The injection contract excludes child ranges and intersects with the
|
||||
parent.** Unless `#set! injection.include-children` is set, the injected
|
||||
ranges are the content node's extent **minus its NAMED children's
|
||||
ranges**, then **intersected with the parent layer's own included
|
||||
ranges** so a nested injection cannot reintroduce bytes its parent
|
||||
excluded. *(Implementation note, round 1: excluding ALL children — not
|
||||
just named — shreds a markdown block `inline` node, whose children are
|
||||
anonymous text tokens, into unparseable fragments. Excluding only named
|
||||
children matches `tree-sitter-md`'s own inline splitter,
|
||||
`parser.rs:406-425`, and is correct for every real injection site.)*
|
||||
This makes `markdown_inline` a genuine multi-range case (Q#IJ5).
|
||||
4. `LanguageEntry.loader` (`fn() -> Language`) and query-source `&'static
|
||||
str` consts are `Send` and touch no grammar C-object until called — a
|
||||
worker resolves injected languages **lazily** by indexing `&'static
|
||||
BUILTIN_LANGUAGES`, preserving the M4.2 lazy-load invariant.
|
||||
5. Injection-query availability is per-crate and inconsistently named
|
||||
(rust `INJECTIONS_QUERY`; markdown `INJECTION_QUERY_BLOCK`; bash none)
|
||||
— the same pattern `highlights_query` already absorbs as a `&'static
|
||||
[&'static str]` slice.
|
||||
|
||||
---
|
||||
|
||||
## Decisions
|
||||
|
||||
### Q#IJ1 — Two node types: worker `RawLayer`, settled `Layer`; the layer set lives in `ParseTreeBundle`
|
||||
|
||||
```
|
||||
RawLayer { language_name: String, tree: Tree, depth: u16 } // worker out
|
||||
Layer { language_name: String, tree: Tree, depth: u16, // settled
|
||||
highlight_query: Option<Arc<Query>> }
|
||||
```
|
||||
|
||||
`ParseTreeBundle` grows to an ordered `Vec<Layer>` (root = layer 0,
|
||||
whole buffer; children depth-ascending). One `Arc<ParseTreeBundle>` is
|
||||
installed **atomically** per settle, so the `StyleGate` `Arc::ptr_eq`
|
||||
gate and the `SyntaxHighlightView` cache stay **unchanged** — a layered
|
||||
reparse mints a fresh Arc and flips both gates. `bundle.language_name`
|
||||
stays (root label). Per-layer `included_ranges: Vec<Range>` (Q#IJ5) are
|
||||
the worker's parse *input*, not retained on the settled `Layer` (styling
|
||||
reads the tree; offsets are absolute).
|
||||
|
||||
### Q#IJ2 — Two-stage handoff: worker builds trees, settle resolves queries
|
||||
|
||||
- **Stage 1 — worker (`run_parse_layered`)** builds the whole layer
|
||||
*structure*: root tree (incremental) + every child tree, via injection
|
||||
queries and `set_included_ranges`. It resolves injected languages by
|
||||
indexing `&'static BUILTIN_LANGUAGES` (loaders + a new `injections_query:
|
||||
&'static [&'static str]` field on `LanguageEntry`, mirroring
|
||||
`highlights_query`), loading/compiling only what a file injects.
|
||||
Touches no highlight query, no theme. Output `Vec<RawLayer>`.
|
||||
- **Stage 2 — settle/install (main thread)** resolves each raw layer's
|
||||
`highlights_query(name)` from the registry cache, builds `Vec<Layer>`,
|
||||
wraps one `ParseTreeBundle`, installs the `Arc` **atomically**.
|
||||
|
||||
Highlight-query compilation stays main-thread/cached/shared with the
|
||||
producers; tree parsing stays on the worker. The only dynamic worker
|
||||
input beyond the static table is the alias snapshot (Q#IJ4), carried in
|
||||
`ParseRequest`.
|
||||
|
||||
*Rejected:* (a) layer-build on the main thread in settle — moves child
|
||||
parsing onto the frame path; (b) a `Send + Sync` query store so the
|
||||
worker resolves everything — larger registry blast radius, deferred as
|
||||
an option if Stage 2 bottlenecks.
|
||||
|
||||
**Limitation named:** injection targets resolve only against
|
||||
`BUILTIN_LANGUAGES`; runtime/Lua-registered languages are not injectable
|
||||
in v1. Every headline case is bundled.
|
||||
|
||||
### Q#IJ3 — Bounded recursion: depth cap, generous layer backstop, visited guard, child-only failure
|
||||
|
||||
- **max depth** (default 3),
|
||||
- **max total layer count** — a *runaway backstop* set well above any
|
||||
real document (default **4096**), decoupled from performance; the real
|
||||
perf bound is the Q#IJ10 settle-time guard. Hitting it is **surfaced, not
|
||||
silent**: `run_parse` sets a `ParseTreeBundle::injection_capped` flag,
|
||||
and `syntax.lua`'s settle tick raises it once per buffer via
|
||||
`pmacs.error` (`_injection_capped`). A modest markdown doc's inline
|
||||
layers (one per paragraph/heading) sit far under the backstop. A real
|
||||
boundary test drives just over 4096 fences and asserts the flag + capped
|
||||
count + intact root.
|
||||
- a **visited set on `(language_name, ranges)`** so a same-language
|
||||
self-injection over a non-shrinking region can't reproduce itself.
|
||||
|
||||
**Failure is isolated to the child:** unknown/unresolvable language, cap
|
||||
hit, or child parse error drops **that child layer only** — never the
|
||||
root, never a sibling. The root always installs.
|
||||
|
||||
### Q#IJ4 — Dynamic fence names: registry-held alias map, case-folded, snapshot to the worker
|
||||
|
||||
Raw `@injection.language` text (`JS`, `ts`, `sh`, `py`, `Rust`, `c++`,
|
||||
`jsx`, `tsx`) won't exact-match bundled names — the job tree-sitter's
|
||||
per-language injection-regex does. pmacs does it with a **case-folded
|
||||
alias table**: lowercase the text, look up an alias table before the
|
||||
bundled-name table (`js`→javascript, `jsx`→javascriptreact,
|
||||
`ts`→typescript, `tsx`→typescriptreact, `py`→python, `rs`→rust,
|
||||
`sh`/`shell`→bash, `c++`/`cxx`→cpp, `yml`→yaml, …). Unresolved → region
|
||||
skipped (no error, root intact).
|
||||
|
||||
**Worker-safe extensibility:** the map lives in the registry (static
|
||||
defaults + Lua-driven overrides via a Rust setter that
|
||||
`pmacs.parse.injection_aliases` writes through). Because the worker can't
|
||||
read Lua or the `Rc` registry, each `_dispatch` (the `syntax.lua:26`
|
||||
seam) **snapshots the merged map into an `Arc<HashMap<String,String>>`
|
||||
carried in `ParseRequest`**. Acceptance mutates the alias set from Lua,
|
||||
then runs an **asynchronous** injected parse and asserts the new alias
|
||||
resolves — proving the bridge, not just the static resolver.
|
||||
|
||||
### Q#IJ5 — Included ranges are `Vec<Range>` per layer: exclude children, intersect with parent
|
||||
|
||||
A layer's ranges are built from its `@injection.content` node(s):
|
||||
|
||||
- `#set! injection.include-children` → `[node.range]`;
|
||||
- otherwise (default; the markdown-inline case) → the node's extent
|
||||
**minus its NAMED children's ranges** (anonymous token children are the
|
||||
injected text itself and are kept — mechanic #3), **then intersected
|
||||
with the parent layer's included ranges**. Ranges come out ordered and
|
||||
non-overlapping; empty ranges dropped.
|
||||
|
||||
Fenced code (`code_fence_content`, no children) → one range; a **multi-line**
|
||||
container (a blockquote/list whose inline node carries a named
|
||||
`block_continuation` child) → several. Core, not deferred —
|
||||
`markdown_inline` needs it. Acceptance asserts `content_node_ranges`
|
||||
returns **>1 range** for a multi-line blockquote and that both sides parse
|
||||
and highlight (a one-line paragraph would give a single range and could
|
||||
not falsify multi-range support).
|
||||
|
||||
### Q#IJ6 — The wire producer flattens layers into disjoint effective spans
|
||||
|
||||
Because the wire re-sorts by start (`main.rs:4117`/`4130`), producer
|
||||
order cannot carry overlap precedence. So overlaps are resolved **in the
|
||||
producer, before the wire**:
|
||||
|
||||
- **`scoped_style_spans` flattens** all layers into **disjoint effective
|
||||
spans** — an ordered active-set **event sweep**: activate a span at its
|
||||
start boundary, expire it at its end, and at each interval fold the
|
||||
active set (deeper layer / later same-depth sibling / narrower capture
|
||||
wins, keyed by `(layer_index, capture_order)`). Cost is
|
||||
**O(n·log n + Σ active)** in the span count — linear in practice, since
|
||||
the active set is bounded by overlap depth, not the total span count.
|
||||
This bound matters because the file-style summary runs the flattener
|
||||
over the **whole buffer**, not just the viewport. Positional re-sorting
|
||||
downstream is then a no-op on precedence, and the disjoint output aligns
|
||||
with the model's existing style-tile disjointness invariant. (Per-byte
|
||||
effective style is unchanged from today for single-layer buffers; only
|
||||
the *shape* goes overlapping→disjoint, so existing `StyleSpans` shape
|
||||
assertions are updated to match.)
|
||||
- **The GPU `source_color_at` fold fix is kept** (fold all covering spans
|
||||
in order, matching `effective_style_at`) — defense-in-depth and a
|
||||
contract alignment, correct even for any residual same-start overlap.
|
||||
- The **grid** producer (`SyntaxHighlightView`) has no wire and no sort:
|
||||
it paints layer-by-layer in depth order (later overrides via the cell
|
||||
merge), which is already correct.
|
||||
|
||||
Acceptance test #9 drives the real full-frame message transform
|
||||
(`spans_from_segments`, extracted from `replace_style_spans`) then
|
||||
`source_color_at`, with an overlapping parent-red / child-green case —
|
||||
exercising the start-sort + fold, not a hand-rolled sort. (A live-`State`
|
||||
render pass needs a GPU device and is out of unit-test scope; the extracted
|
||||
transform is the code that matters here.)
|
||||
|
||||
### Q#IJ7 — Both producers walk layers; Policy A unchanged at buffer scope
|
||||
|
||||
`scoped_style_spans` and `SyntaxHighlightView` iterate `bundle.layers`
|
||||
using each `layer.highlight_query`, reusing viewport-scoped
|
||||
`compute_highlight_spans_in_range` per layer. Policy A unchanged at the
|
||||
buffer level (grammar-backed ⇒ tree-sitter across *all* layers; LSP-only
|
||||
untouched; no new `LspStyleView` interaction). `SyntaxHighlightView`
|
||||
drops its single-`query` constructor param and reads per-layer
|
||||
`Arc<Query>` from the bundle, so it still holds only `Send` state — the
|
||||
largest single code change.
|
||||
|
||||
### Q#IJ8 — Incrementality: root incremental, children cold each settle
|
||||
|
||||
Root keeps `InputEdit`-accumulation + `prior_tree`. Child layers rebuild
|
||||
**cold** each settle (an edit can add/remove/resize regions, so child
|
||||
identity isn't stable). **Deferred:** child-tree incrementality and
|
||||
range-scoped rebuild. Named cost: one cold inline layer per markdown
|
||||
paragraph/heading — made a measured acceptance guard by Q#IJ10, not a
|
||||
hope.
|
||||
|
||||
### Q#IJ9 — `injection.combined` deferred
|
||||
|
||||
Each injection **match** is its own layer/parse (multi-range *within* a
|
||||
match is Q#IJ5, in scope). Combined injections (many matches → one shared
|
||||
parse; PHP-in-HTML, some comment schemes) are **deferred**. Markdown
|
||||
fenced/inline are not combined.
|
||||
|
||||
### Q#IJ10 — First consumer: markdown fenced code **and** inline
|
||||
|
||||
Ships fenced-code **and** `markdown_inline` (zero new grammars; same
|
||||
crate; block grammar already injects inline via `#set!`). Inline
|
||||
exercises the static path and the multi-range path (Q#IJ5), completes
|
||||
markdown, and retires the M9.7 "block-only" floor. Its cost is real (a
|
||||
cold child parse per paragraph/heading), so a **many-paragraph
|
||||
settle-time acceptance test** guards it **and asserts the final paragraph
|
||||
receives a layer/capture** (not merely that parsing finished). While that
|
||||
stays green, child incrementality (Q#IJ8) is not required in v1. HTML,
|
||||
JS/TS template literals, and doc-comment code are follow-ups gated on
|
||||
their grammars.
|
||||
|
||||
### Q#IJ11 — Perf gate & parse-in-flight stay correct for free
|
||||
|
||||
`StyleGate.bundle` is the root bundle Arc; a layered reparse installs a
|
||||
fresh Arc atomically (single dispatch), flipping the gate with no
|
||||
half-styled frame. `grammar_style_parse_not_ready` unchanged.
|
||||
|
||||
---
|
||||
|
||||
## Bets
|
||||
|
||||
1. Absolute node offsets under `set_included_ranges` make layer spans
|
||||
buffer-coordinate-native (mechanic #1).
|
||||
2. Static-table worker + two-stage settle + alias snapshot (Q#IJ2/IJ4)
|
||||
preserves lazy loading, keeps parsing off the main thread, and keeps
|
||||
query caching where it is.
|
||||
3. Producer-side flattening (Q#IJ6) is the one definite overlap strategy
|
||||
given the wire re-sort; the GPU fold fix rides along.
|
||||
4. Cold child reparse is fast enough at real paragraph/fence counts; the
|
||||
Q#IJ10 guard is the measured backstop, Q#IJ8 the escape hatch.
|
||||
|
||||
## Deferred (named)
|
||||
|
||||
- `injection.combined` (Q#IJ9).
|
||||
- Child-tree incrementality + range-scoped rebuild (Q#IJ8), gated open by
|
||||
the Q#IJ10 perf guard.
|
||||
- Injectable runtime/Lua-registered languages (Q#IJ2).
|
||||
- HTML/CSS/GraphQL/SQL grammars and their injection sites (Q#IJ10).
|
||||
- Notebook per-cell layering (needs JSON grammar + this engine).
|
||||
- `Send + Sync` query store so the worker resolves highlight queries
|
||||
directly (Q#IJ2 alternative b).
|
||||
|
||||
## Acceptance (bite-verified where it guards a real gap)
|
||||
|
||||
1. `injection_query_block_compiles` — `INJECTION_QUERY_BLOCK` compiles
|
||||
against the md grammar.
|
||||
2. `layered_parse_builds_child_for_fenced_code` — a ` ```rust ` fence
|
||||
yields ≥2 layers; child language == rust, roots at `source_file`.
|
||||
3. `child_layer_offsets_are_absolute` — a `fn` inside the fence has byte
|
||||
offsets matching its position in the **full** markdown source.
|
||||
4. `dynamic_alias_resolves` — ` ```py `→python, ` ```JS `→javascript
|
||||
(case-folded); ` ```nonsense ` → no child, no error, root intact.
|
||||
5. `lua_alias_override_resolves_on_async_parse` — mutate
|
||||
`injection_aliases` in Lua, then an **async** injected parse resolves
|
||||
the new alias (Q#IJ4 bridge). Plus `sync_parse_now_resolves_alias`
|
||||
(round 1): the same must hold on the **synchronous** `_parse_now` path.
|
||||
6. `inline_layer_multi_range_excludes_block_continuation` — a **multi-line**
|
||||
blockquote's inline node carries a named `block_continuation`;
|
||||
`content_node_ranges` returns **>1 range** and both sides parse +
|
||||
highlight (Q#IJ5). (A one-line paragraph gives a single range and can't
|
||||
falsify multi-range.)
|
||||
7. `recursion_bounds_terminate` — rust macro self-injection terminates
|
||||
within the depth bound; `injection_layer_cap_surfaces_and_preserves_root`
|
||||
drives >4096 fences (bundle-flag level), and
|
||||
`injection_cap_surfaced_once_and_rearms_via_lua` (round 2) drives the
|
||||
**observable** Lua settle path: `pmacs.error` once, suppressed on an
|
||||
unchanged re-parse, and re-armed after dropping below the cap and
|
||||
exceeding it again. A failing child drops only itself (Q#IJ3).
|
||||
8. `wire_producer_emits_disjoint_child_spans_in_fence` — `scoped_style_spans`
|
||||
over a ` ```rust ` fence emits disjoint `StyleSpan`s covering a rust
|
||||
keyword **inside** the fence. **Bite-verified** vs the single-layer
|
||||
producer. `flatten_same_depth_sibling_later_layer_wins` (round 2): a
|
||||
later same-depth sibling layer wins the fold, matching grid paint order.
|
||||
`full_buffer_summary_flatten_scales_on_large_grammar_file` (round 1):
|
||||
the whole-buffer **flatten** stays ~linear under the event sweep (a
|
||||
quadratic flatten regresses it; the summary's per-line tally is a
|
||||
separate pre-existing loop).
|
||||
9. `source_color_folds_overlapping_child_over_parent` — parent-red /
|
||||
child-green overlap driven through the real `spans_from_segments`
|
||||
(`replace_style_spans` body): the child (green) wins the fold.
|
||||
**Bite-verified** vs the first-span-wins path (Q#IJ6).
|
||||
10. `grid_paints_injected_child_keyword` — `SyntaxHighlightView` paints a
|
||||
rust-keyword cell inside the fence.
|
||||
11. `incremental_edit_reflects_in_child_and_new_fence_adds_layer` — editing
|
||||
inside a fence shows in child spans after reparse; a **new** fence adds
|
||||
a layer (Q#IJ8).
|
||||
12. `many_paragraph_settle_under_budget_with_tail_covered` — a large
|
||||
all-inline markdown buffer settles within a comfortable budget **and
|
||||
the final paragraph receives an inline layer** (Q#IJ3 cap + Q#IJ10
|
||||
guard).
|
||||
13. `non_injecting_buffer_single_layer` — a plain `.rs` buffer still
|
||||
yields exactly one layer (regression guard).
|
||||
|
||||
## Risks / interactions
|
||||
|
||||
- **Perf** (Q#IJ8/IJ10) — cold child reparse per settle, one inline layer
|
||||
per paragraph; test 12 is the measured guard (with tail coverage),
|
||||
Q#IJ8 the release valve.
|
||||
- **Wire overlap** (Q#IJ6) — the wire re-sorts by start, so flattening in
|
||||
the producer is mandatory; the GPU fold fix rides along. Tests 8/9 pin
|
||||
both.
|
||||
- **Single-layer wire shape** — flattening turns overlapping spans
|
||||
disjoint for *all* grammar buffers; per-byte style is unchanged, but
|
||||
existing `StyleSpans` shape assertions are updated.
|
||||
- **`set_included_ranges` contract** — sorted, non-overlapping, non-empty;
|
||||
the Q#IJ5 exclusion+intersection yields this by construction, but guard
|
||||
empty inline nodes and non-UTF-8 info strings.
|
||||
- **M9.7 prompt-result markdown buffers** now get fenced + inline
|
||||
highlighting (a bonus, retiring the block-only floor); verify the
|
||||
`_attach_highlight`-for-markdown path doesn't crash.
|
||||
- **Themes main quest** — untouched. Highlight *structure*, not color;
|
||||
capture→style still flows through `Theme::lookup`. No protocol bump
|
||||
(`StyleSpan` wire shape unchanged).
|
||||
|
|
@ -4126,11 +4126,7 @@ impl State {
|
|||
/// `full = true` path: discard prior styling, take the segments'
|
||||
/// spans as authoritative for the declared viewport.
|
||||
fn replace_style_spans(&mut self, segments: Vec<StyleSegment>) {
|
||||
self.current_spans.clear();
|
||||
for seg in segments {
|
||||
self.current_spans.extend(seg.spans);
|
||||
}
|
||||
self.current_spans.sort_by_key(|s| s.range.start);
|
||||
self.current_spans = spans_from_segments(segments);
|
||||
}
|
||||
|
||||
/// `full = false` path: each segment's `range` authoritatively
|
||||
|
|
@ -6731,13 +6727,40 @@ fn adornment_text_color(fg: CellColor) -> glyphon::Color {
|
|||
cell_color_to_glyphon(fg).unwrap_or_else(|| glyphon::Color::rgb(130, 130, 140))
|
||||
}
|
||||
|
||||
/// The `StyleSpans { full: true }` transform `replace_style_spans` applies:
|
||||
/// flatten every segment's spans and start-sort. Extracted as a free
|
||||
/// function so it (and `source_color_at`) can be exercised without a live
|
||||
/// `State` — the start-sort is the step that makes producer depth-order
|
||||
/// irrelevant on the wire, which is why the daemon flattens to disjoint
|
||||
/// spans and this consumer folds (framing Q#IJ6).
|
||||
fn spans_from_segments(segments: Vec<StyleSegment>) -> Vec<StyleSpan> {
|
||||
let mut spans: Vec<StyleSpan> = Vec::new();
|
||||
for seg in segments {
|
||||
spans.extend(seg.spans);
|
||||
}
|
||||
spans.sort_by_key(|s| s.range.start);
|
||||
spans
|
||||
}
|
||||
|
||||
fn source_color_at(byte: u64, spans: &[StyleSpan]) -> Option<glyphon::Color> {
|
||||
for sp in spans {
|
||||
if sp.range.start <= byte && byte < sp.range.end {
|
||||
return cell_color_to_glyphon(sp.style.fg);
|
||||
// Fold every covering span in order, matching the semantic-client
|
||||
// `effective_style_at` contract (last covering span with a non-default
|
||||
// fg wins) rather than returning the first. The daemon flattens
|
||||
// injection layers into disjoint spans (framing Q#IJ6), so usually at
|
||||
// most one covers a byte — but where spans do overlap (a styled
|
||||
// markdown parent under an injected child), the topmost color must win,
|
||||
// never the outermost. Spans arrive start-sorted, so a narrower nested
|
||||
// span (later start) folds after and overrides its enclosing span.
|
||||
let mut color = None;
|
||||
for sp in spans
|
||||
.iter()
|
||||
.filter(|sp| sp.range.start <= byte && byte < sp.range.end)
|
||||
{
|
||||
if let Some(c) = cell_color_to_glyphon(sp.style.fg) {
|
||||
color = Some(c);
|
||||
}
|
||||
}
|
||||
None
|
||||
color
|
||||
}
|
||||
|
||||
/// Convert a `pmacs-protocol::cell::Color` to a `glyphon::Color`.
|
||||
|
|
@ -8969,4 +8992,45 @@ mod tests {
|
|||
"the peer cursor line must keep the CurrentLine constant"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_color_folds_overlapping_child_over_parent() {
|
||||
// Framing acceptance #9 (GPU consumer): a styled markdown parent
|
||||
// span [0,10) (red) with an injected child span [3,6) (green) on top,
|
||||
// driven through the ACTUAL `StyleSpans { full: true }` transform
|
||||
// (`spans_from_segments`, the body of `replace_style_spans`) rather
|
||||
// than a hand-rolled sort. `source_color_at` must FOLD the covering
|
||||
// spans (child green wins at a shared byte), not return the first
|
||||
// (parent) span. Bite: the pre-fix first-covering-span code returns
|
||||
// red at byte 4.
|
||||
let red = style_with_fg(CellColor::Rgb(200, 0, 0));
|
||||
let green = style_with_fg(CellColor::Rgb(0, 200, 0));
|
||||
let segments = vec![StyleSegment {
|
||||
range: ByteRange { start: 0, end: 10 },
|
||||
spans: vec![
|
||||
StyleSpan {
|
||||
range: ByteRange { start: 0, end: 10 },
|
||||
style: red,
|
||||
},
|
||||
StyleSpan {
|
||||
range: ByteRange { start: 3, end: 6 },
|
||||
style: green,
|
||||
},
|
||||
],
|
||||
}];
|
||||
let spans = spans_from_segments(segments); // full-frame message application
|
||||
|
||||
// Byte 4 is covered by both: the child (green) wins the fold.
|
||||
assert_eq!(
|
||||
source_color_at(4, &spans),
|
||||
Some(glyphon::Color::rgb(0, 200, 0)),
|
||||
"the injected child color wins over the parent at a shared byte"
|
||||
);
|
||||
// Byte 1 is parent-only: stays red.
|
||||
assert_eq!(
|
||||
source_color_at(1, &spans),
|
||||
Some(glyphon::Color::rgb(200, 0, 0)),
|
||||
"a parent-only byte keeps the parent color"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2700,6 +2700,7 @@ mod tests {
|
|||
language_name: "rust".to_owned(),
|
||||
prior_tree: None,
|
||||
edits: Vec::new(),
|
||||
injection_aliases: Arc::new(std::collections::HashMap::new()),
|
||||
};
|
||||
let id = rt.dispatch_parse(req, None);
|
||||
pump_until(&rt, || rt.is_complete(id));
|
||||
|
|
@ -2713,7 +2714,7 @@ mod tests {
|
|||
other => panic!("unexpected outcome: {other:?}"),
|
||||
}
|
||||
assert_eq!(bundle.language_name, "rust");
|
||||
assert_eq!(bundle.tree.root_node().kind(), "source_file");
|
||||
assert_eq!(bundle.root_tree().root_node().kind(), "source_file");
|
||||
// take_parse_tree was already drained, so handoff is empty.
|
||||
assert_eq!(rt.parse_handoff_len(), 0);
|
||||
}
|
||||
|
|
@ -2729,6 +2730,7 @@ mod tests {
|
|||
language_name: "rust".to_owned(),
|
||||
prior_tree: None,
|
||||
edits: Vec::new(),
|
||||
injection_aliases: Arc::new(std::collections::HashMap::new()),
|
||||
};
|
||||
let id = rt.dispatch_parse(req, None);
|
||||
pump_until(&rt, || rt.is_complete(id));
|
||||
|
|
|
|||
257
src/highlight.rs
257
src/highlight.rs
|
|
@ -39,7 +39,7 @@ use crate::buffer::Buffer;
|
|||
use crate::cell::{CellCoord, CellGrid, Color, Style, UnderlineStyle};
|
||||
use crate::lsp::SharedLspManager;
|
||||
use crate::overlay::merge_styles;
|
||||
use crate::syntax::{HighlightSpan, ParseTreeBundle, ParseViewHandle, compute_highlight_spans};
|
||||
use crate::syntax::{HighlightSpan, ParseTreeBundle, ParseViewHandle, compute_highlight_spans_for};
|
||||
use crate::view::{View, Viewport};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -248,6 +248,19 @@ pub type ThemeHandle = Arc<Mutex<Theme>>;
|
|||
// SyntaxHighlightView
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Per-layer cached highlight spans + capture names (framing Q#IJ7).
|
||||
/// One entry per bundle layer that has a highlight query; kept in the
|
||||
/// bundle's depth order so the render paints shallow-to-deep.
|
||||
struct LayerSpans {
|
||||
/// Spans for this layer, sorted wider-first per
|
||||
/// [`compute_highlight_spans_for`].
|
||||
spans: Vec<HighlightSpan>,
|
||||
/// Capture names indexed by `HighlightSpan::capture_index` for this
|
||||
/// layer's own query. Layers may use different grammars, so this is
|
||||
/// per-layer, not shared.
|
||||
capture_names: Arc<[String]>,
|
||||
}
|
||||
|
||||
/// Cached per-bundle highlight state. Keyed by the underlying
|
||||
/// `Arc<ParseTreeBundle>`'s identity (compared via `Arc::ptr_eq`),
|
||||
/// so a freshly-installed bundle invalidates the cache.
|
||||
|
|
@ -255,25 +268,19 @@ struct HighlightCache {
|
|||
/// The bundle the cache was built against. `None` until the
|
||||
/// first render observes a settled bundle.
|
||||
bundle: Option<Arc<ParseTreeBundle>>,
|
||||
/// Compiled spans, sorted wider-first per
|
||||
/// [`compute_highlight_spans`].
|
||||
spans: Vec<HighlightSpan>,
|
||||
/// Per-layer spans in depth-ascending order (framing Q#IJ6).
|
||||
layers: Vec<LayerSpans>,
|
||||
/// Per-row first-byte offsets into `bundle.source`. `Vec<u32>`
|
||||
/// because pmacs files cap at 4 GiB.
|
||||
line_offsets: Vec<u32>,
|
||||
/// Capture names indexed by `HighlightSpan::capture_index`.
|
||||
/// Populated alongside `spans` so the render path doesn't need
|
||||
/// to keep a reference into the [`tree_sitter::Query`].
|
||||
capture_names: Arc<[String]>,
|
||||
}
|
||||
|
||||
impl HighlightCache {
|
||||
fn empty() -> Self {
|
||||
Self {
|
||||
bundle: None,
|
||||
spans: Vec::new(),
|
||||
layers: Vec::new(),
|
||||
line_offsets: Vec::new(),
|
||||
capture_names: Arc::from(Vec::<String>::new().into_boxed_slice()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -283,39 +290,41 @@ impl HighlightCache {
|
|||
const TAB_WIDTH: u32 = 8;
|
||||
|
||||
/// View that renders syntax highlighting from a tree-sitter parse
|
||||
/// tree. Composes over [`crate::text_view::TextView`] per the M2.9
|
||||
/// view-composition contract --- it never writes glyphs, only merges
|
||||
/// styles into cells the base view has already painted.
|
||||
/// tree, including its injection layers. Composes over
|
||||
/// [`crate::text_view::TextView`] per the M2.9 view-composition
|
||||
/// contract --- it never writes glyphs, only merges styles into cells
|
||||
/// the base view has already painted.
|
||||
pub struct SyntaxHighlightView {
|
||||
parse: ParseViewHandle,
|
||||
query: Arc<tree_sitter::Query>,
|
||||
theme: ThemeHandle,
|
||||
cache: HighlightCache,
|
||||
}
|
||||
|
||||
impl SyntaxHighlightView {
|
||||
/// Construct a highlight view over `parse` using `query` and
|
||||
/// `theme`. The initial cache is empty; the first render with
|
||||
/// a settled bundle populates it.
|
||||
/// Construct a highlight view over `parse` using `theme`. Per-layer
|
||||
/// highlight queries come from each `Layer::highlight_query` in the
|
||||
/// bundle (resolved at settle, framing Q#IJ2), so no query is passed
|
||||
/// here. The initial cache is empty; the first render with a settled
|
||||
/// bundle populates it.
|
||||
#[must_use]
|
||||
pub fn new(parse: ParseViewHandle, query: Arc<tree_sitter::Query>, theme: ThemeHandle) -> Self {
|
||||
pub fn new(parse: ParseViewHandle, theme: ThemeHandle) -> Self {
|
||||
Self {
|
||||
parse,
|
||||
query,
|
||||
theme,
|
||||
cache: HighlightCache::empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Test helper: number of cached highlight spans.
|
||||
/// Test helper: total cached highlight spans across all layers.
|
||||
#[must_use]
|
||||
pub fn cached_span_count(&self) -> usize {
|
||||
self.cache.spans.len()
|
||||
self.cache.layers.iter().map(|l| l.spans.len()).sum()
|
||||
}
|
||||
|
||||
/// Refresh `self.cache` if the parse view's current bundle
|
||||
/// differs from the cached one. No-op when the bundle pointer
|
||||
/// is unchanged --- the steady-state cost between parses.
|
||||
/// is unchanged --- the steady-state cost between parses. Rebuilds
|
||||
/// spans for every layer that carries a highlight query.
|
||||
fn refresh_cache_if_stale(&mut self) {
|
||||
let Some(bundle) = self.parse.current() else {
|
||||
return;
|
||||
|
|
@ -328,26 +337,39 @@ impl SyntaxHighlightView {
|
|||
if !stale {
|
||||
return;
|
||||
}
|
||||
let spans = compute_highlight_spans(&self.query, &bundle);
|
||||
let line_offsets = compute_line_offsets(bundle.source.as_ref());
|
||||
let capture_names: Arc<[String]> = self
|
||||
.query
|
||||
.capture_names()
|
||||
.iter()
|
||||
.map(|s| (*s).to_owned())
|
||||
.collect();
|
||||
let source = bundle.source.as_ref();
|
||||
let mut layers = Vec::new();
|
||||
for layer in &bundle.layers {
|
||||
let Some(query) = layer.highlight_query.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
let spans = compute_highlight_spans_for(query, &layer.tree, source, None);
|
||||
if spans.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let capture_names: Arc<[String]> = query
|
||||
.capture_names()
|
||||
.iter()
|
||||
.map(|s| (*s).to_owned())
|
||||
.collect();
|
||||
layers.push(LayerSpans {
|
||||
spans,
|
||||
capture_names,
|
||||
});
|
||||
}
|
||||
let line_offsets = compute_line_offsets(source);
|
||||
self.cache = HighlightCache {
|
||||
bundle: Some(bundle),
|
||||
spans,
|
||||
layers,
|
||||
line_offsets,
|
||||
capture_names,
|
||||
};
|
||||
}
|
||||
|
||||
/// Look up the style for span `s`, consulting the active theme.
|
||||
fn style_for(&self, theme: &Theme, s: HighlightSpan) -> Style {
|
||||
/// Look up the style for span `s` within `layer`, consulting the
|
||||
/// active theme.
|
||||
fn style_for(theme: &Theme, layer: &LayerSpans, s: HighlightSpan) -> Style {
|
||||
let idx = s.capture_index as usize;
|
||||
let Some(name) = self.cache.capture_names.get(idx) else {
|
||||
let Some(name) = layer.capture_names.get(idx) else {
|
||||
return theme.default_style;
|
||||
};
|
||||
theme.lookup(name)
|
||||
|
|
@ -364,7 +386,7 @@ impl View for SyntaxHighlightView {
|
|||
let Some(bundle) = self.cache.bundle.clone() else {
|
||||
return;
|
||||
};
|
||||
if self.cache.spans.is_empty() || self.cache.line_offsets.is_empty() {
|
||||
if self.cache.layers.is_empty() || self.cache.line_offsets.is_empty() {
|
||||
return;
|
||||
}
|
||||
let source: &[u8] = bundle.source.as_ref();
|
||||
|
|
@ -376,61 +398,64 @@ impl View for SyntaxHighlightView {
|
|||
let cell_origin = viewport.cell_origin;
|
||||
let total_lines = self.cache.line_offsets.len() as u32;
|
||||
|
||||
for row_offset in 0..max_rows {
|
||||
let line_idx = start_line + row_offset;
|
||||
if line_idx >= total_lines {
|
||||
break;
|
||||
}
|
||||
let line_start = self.cache.line_offsets[line_idx as usize];
|
||||
let line_end = self
|
||||
.cache
|
||||
.line_offsets
|
||||
.get(line_idx as usize + 1)
|
||||
.copied()
|
||||
.unwrap_or(source.len() as u32);
|
||||
// Trim a single trailing newline if any --- the text
|
||||
// view doesn't paint it as a glyph either.
|
||||
let line_end_no_nl = if line_end > line_start
|
||||
&& source.get(line_end as usize - 1).copied() == Some(b'\n')
|
||||
{
|
||||
line_end - 1
|
||||
} else {
|
||||
line_end
|
||||
};
|
||||
let line_bytes = &source[line_start as usize..line_end_no_nl as usize];
|
||||
// Paint layers shallow-to-deep (framing Q#IJ6): a deeper layer
|
||||
// merges on top, so an injected child's styling wins within its
|
||||
// region. Within a layer, wider-first ordering (from
|
||||
// `compute_highlight_spans_for`) lets narrower captures override.
|
||||
for layer in &self.cache.layers {
|
||||
for row_offset in 0..max_rows {
|
||||
let line_idx = start_line + row_offset;
|
||||
if line_idx >= total_lines {
|
||||
break;
|
||||
}
|
||||
let line_start = self.cache.line_offsets[line_idx as usize];
|
||||
let line_end = self
|
||||
.cache
|
||||
.line_offsets
|
||||
.get(line_idx as usize + 1)
|
||||
.copied()
|
||||
.unwrap_or(source.len() as u32);
|
||||
// Trim a single trailing newline if any --- the text
|
||||
// view doesn't paint it as a glyph either.
|
||||
let line_end_no_nl = if line_end > line_start
|
||||
&& source.get(line_end as usize - 1).copied() == Some(b'\n')
|
||||
{
|
||||
line_end - 1
|
||||
} else {
|
||||
line_end
|
||||
};
|
||||
let line_bytes = &source[line_start as usize..line_end_no_nl as usize];
|
||||
|
||||
// Spans whose start lies on this line. Wider-first
|
||||
// ordering means parents apply before children.
|
||||
// Spans that *cross* lines apply on every line they
|
||||
// touch --- this loop only filters by start, then
|
||||
// intersects with the line range below.
|
||||
for span in self
|
||||
.cache
|
||||
.spans
|
||||
.iter()
|
||||
.filter(|s| s.start_byte < line_end_no_nl && s.end_byte > line_start)
|
||||
.copied()
|
||||
{
|
||||
let style = self.style_for(&theme, span);
|
||||
if style == Style::default() {
|
||||
// Nothing to merge --- skip the per-cell loop.
|
||||
continue;
|
||||
}
|
||||
let s_start = span.start_byte.max(line_start);
|
||||
let s_end = span.end_byte.min(line_end_no_nl);
|
||||
let byte_col_start = (s_start - line_start) as usize;
|
||||
let byte_col_end = (s_end - line_start) as usize;
|
||||
let (start_col, end_col) =
|
||||
byte_range_to_display_cols(line_bytes, byte_col_start, byte_col_end);
|
||||
if end_col <= start_col {
|
||||
continue;
|
||||
}
|
||||
let cell_row = cell_origin.row + row_offset;
|
||||
let clamped_start = start_col.min(max_cols);
|
||||
let clamped_end = end_col.min(max_cols);
|
||||
for col in clamped_start..clamped_end {
|
||||
let cell = cells.at(CellCoord::new(cell_row, cell_origin.col + col));
|
||||
cell.style = merge_styles(cell.style, style);
|
||||
// Spans whose start lies on this line. Spans that *cross*
|
||||
// lines apply on every line they touch --- this loop filters
|
||||
// by intersection, then clips to the line range below.
|
||||
for span in layer
|
||||
.spans
|
||||
.iter()
|
||||
.filter(|s| s.start_byte < line_end_no_nl && s.end_byte > line_start)
|
||||
.copied()
|
||||
{
|
||||
let style = Self::style_for(&theme, layer, span);
|
||||
if style == Style::default() {
|
||||
// Nothing to merge --- skip the per-cell loop.
|
||||
continue;
|
||||
}
|
||||
let s_start = span.start_byte.max(line_start);
|
||||
let s_end = span.end_byte.min(line_end_no_nl);
|
||||
let byte_col_start = (s_start - line_start) as usize;
|
||||
let byte_col_end = (s_end - line_start) as usize;
|
||||
let (start_col, end_col) =
|
||||
byte_range_to_display_cols(line_bytes, byte_col_start, byte_col_end);
|
||||
if end_col <= start_col {
|
||||
continue;
|
||||
}
|
||||
let cell_row = cell_origin.row + row_offset;
|
||||
let clamped_start = start_col.min(max_cols);
|
||||
let clamped_end = end_col.min(max_cols);
|
||||
for col in clamped_start..clamped_end {
|
||||
let cell = cells.at(CellCoord::new(cell_row, cell_origin.col + col));
|
||||
cell.style = merge_styles(cell.style, style);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1228,4 +1253,56 @@ mod tests {
|
|||
);
|
||||
assert!(!c.style.bold);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grid_paints_injected_child_keyword() {
|
||||
// Framing acceptance #10: the grid `SyntaxHighlightView` paints a
|
||||
// rust keyword cell INSIDE a markdown ```rust fence — styling only
|
||||
// the injected rust layer can produce. Layers paint shallow-to-deep
|
||||
// (framing Q#IJ6), so the rust keyword wins over the markdown root.
|
||||
use crate::buffer::{Buffer, BufferId, EditOp};
|
||||
use crate::cell::{Cell, CellSize};
|
||||
use crate::syntax::{ParseView, SyntaxRegistry};
|
||||
|
||||
let reg = SyntaxRegistry::new();
|
||||
let language = reg.language("markdown").expect("markdown grammar");
|
||||
// Line 0: ```rust ; line 1: fn demo() {} ; line 2: ```
|
||||
let src = b"```rust\nfn demo() {}\n```\n";
|
||||
let mut buf = Buffer::new(BufferId::next(), "doc.md");
|
||||
buf.apply_edit(EditOp::Insert { pos: 0, bytes: src })
|
||||
.unwrap();
|
||||
let view = ParseView::new(&buf, language, "markdown".to_owned());
|
||||
let handle = view.handle();
|
||||
let _vid = buf.attach_view(Box::new(view));
|
||||
let mut req = handle.make_request();
|
||||
req.injection_aliases = reg.injection_alias_snapshot();
|
||||
let bundle = crate::syntax::run_parse(req).expect("markdown parse");
|
||||
handle.install(reg.resolve_layer_queries(&bundle));
|
||||
|
||||
let mut hv = SyntaxHighlightView::new(handle, reg.theme());
|
||||
let cols = 40usize;
|
||||
let rows = 3usize;
|
||||
let mut backing: Vec<Cell> = vec![Cell::default(); rows * cols];
|
||||
let mut grid = CellGrid {
|
||||
cells: &mut backing,
|
||||
stride: cols as u32,
|
||||
size: CellSize::new(rows as u32, cols as u32),
|
||||
};
|
||||
let viewport = Viewport {
|
||||
buffer_start: 0,
|
||||
buffer_end: u64::MAX,
|
||||
cell_origin: CellCoord::new(0, 0),
|
||||
cell_size: CellSize::new(rows as u32, cols as u32),
|
||||
gutter_w: 0,
|
||||
};
|
||||
let registry = buf; // keep buf alive
|
||||
hv.render(®istry, viewport, &mut grid);
|
||||
|
||||
// `fn` is at line 1, cols 0..2. default_dark styles `keyword` bold.
|
||||
let c = grid.get(CellCoord::new(1, 0));
|
||||
assert!(
|
||||
c.style.bold,
|
||||
"the injected rust `fn` keyword is painted (bold) inside the fence"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6362,7 +6362,7 @@ pub struct ParseNodeLua {
|
|||
|
||||
impl ParseNodeLua {
|
||||
fn resolve(&self) -> Option<tree_sitter::Node<'_>> {
|
||||
let mut node = self.bundle.tree.root_node();
|
||||
let mut node = self.bundle.root_tree().root_node();
|
||||
for &idx in &self.path {
|
||||
node = node.child(idx)?;
|
||||
}
|
||||
|
|
@ -6395,7 +6395,9 @@ impl UserData for ParseTreeLua {
|
|||
methods.add_method("text", |lua, this, ()| {
|
||||
lua.create_string(this.0.source.as_ref())
|
||||
});
|
||||
methods.add_method("sexp", |_, this, ()| Ok(this.0.tree.root_node().to_sexp()));
|
||||
methods.add_method("sexp", |_, this, ()| {
|
||||
Ok(this.0.root_tree().root_node().to_sexp())
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -6617,6 +6619,21 @@ pub fn install_parse(
|
|||
)?;
|
||||
}
|
||||
|
||||
// Injection alias write-through (framing Q#IJ4). `syntax.lua` wraps
|
||||
// this in a `pmacs.parse.injection_aliases` proxy table so users write
|
||||
// `pmacs.parse.injection_aliases.mylang = "rust"`. The registry holds
|
||||
// the merged map (defaults + overrides); each dispatch snapshots it.
|
||||
{
|
||||
let s = syntax.clone();
|
||||
parse_mod.set(
|
||||
"_register_injection_alias",
|
||||
lua.create_function(move |_, (alias, lang): (String, String)| {
|
||||
s.register_injection_alias(alias, lang);
|
||||
Ok(())
|
||||
})?,
|
||||
)?;
|
||||
}
|
||||
|
||||
{
|
||||
let s = syntax.clone();
|
||||
parse_mod.set(
|
||||
|
|
@ -6635,6 +6652,21 @@ pub fn install_parse(
|
|||
)?;
|
||||
}
|
||||
|
||||
// True if the buffer's installed bundle hit the injection layer backstop
|
||||
// (framing Q#IJ3). `syntax.lua` reads this after settle to surface the
|
||||
// cap once via `pmacs.error` rather than dropping regions silently.
|
||||
{
|
||||
let s = syntax.clone();
|
||||
parse_mod.set(
|
||||
"_injection_capped",
|
||||
lua.create_function(move |_, id: BufferIdLua| {
|
||||
Ok(s.view(id.0)
|
||||
.and_then(|h| h.current())
|
||||
.is_some_and(|b| b.injection_capped))
|
||||
})?,
|
||||
)?;
|
||||
}
|
||||
|
||||
{
|
||||
let s = syntax.clone();
|
||||
parse_mod.set(
|
||||
|
|
@ -6656,9 +6688,16 @@ pub fn install_parse(
|
|||
"_parse_now",
|
||||
lua.create_function(move |_, (id, lang): (BufferIdLua, String)| {
|
||||
let handle = get_or_create_parse_view(&s, ®, id.0, &lang)?;
|
||||
let req = handle.make_request();
|
||||
let mut req = handle.make_request();
|
||||
// Snapshot the alias map on the sync path too (framing Q#IJ4)
|
||||
// — otherwise a `py` fence or a Lua-added alias would resolve
|
||||
// asynchronously but not through `_parse_now`.
|
||||
req.injection_aliases = s.injection_alias_snapshot();
|
||||
let bundle = syntax::run_parse(req).map_err(mlua::Error::external)?;
|
||||
let arc = Arc::new(bundle);
|
||||
// Resolve each layer's highlight query from the registry
|
||||
// cache before install so producers can style every layer
|
||||
// (framing Q#IJ2 stage 2).
|
||||
let arc = s.resolve_layer_queries(&bundle);
|
||||
handle.install(arc.clone());
|
||||
Ok(ParseTreeLua(arc))
|
||||
})?,
|
||||
|
|
@ -6676,7 +6715,10 @@ pub fn install_parse(
|
|||
"_dispatch",
|
||||
lua.create_function(move |_, (id, lang): (BufferIdLua, String)| {
|
||||
let handle = get_or_create_parse_view(&s, ®, id.0, &lang)?;
|
||||
let req = handle.make_request();
|
||||
let mut req = handle.make_request();
|
||||
// Snapshot the alias map into the request so the worker can
|
||||
// resolve dynamic fence names off the main thread (Q#IJ4).
|
||||
req.injection_aliases = s.injection_alias_snapshot();
|
||||
let job_id = rt.dispatch_parse(req, None);
|
||||
s.record_parse_job(job_id, id.0);
|
||||
Ok(job_id)
|
||||
|
|
@ -6707,7 +6749,10 @@ pub fn install_parse(
|
|||
return Ok(false);
|
||||
};
|
||||
if let Some(handle) = s.view(buf_id) {
|
||||
handle.install(bundle);
|
||||
// Stage 2 (framing Q#IJ2): resolve each layer's highlight
|
||||
// query on the main thread before install.
|
||||
let resolved = s.resolve_layer_queries(&bundle);
|
||||
handle.install(resolved);
|
||||
Ok(true)
|
||||
} else {
|
||||
Ok(false)
|
||||
|
|
@ -6737,12 +6782,14 @@ pub fn install_parse(
|
|||
id.0
|
||||
)));
|
||||
};
|
||||
let Some(query) = s.highlights_query(&lang) else {
|
||||
// No highlights query for this language --- treat as
|
||||
// a benign no-op so callers don't need to special-case
|
||||
// grammars without highlights bundled.
|
||||
if s.highlights_query(&lang).is_none() {
|
||||
// Root language ships no highlights --- treat as a benign
|
||||
// no-op so callers don't need to special-case grammars
|
||||
// without highlights bundled. (Injected child layers still
|
||||
// resolve their own queries at settle; a root-highlight-less
|
||||
// injector is out of v1 scope.)
|
||||
return Ok(false);
|
||||
};
|
||||
}
|
||||
let theme = s.theme();
|
||||
let core = lua
|
||||
.app_data_ref::<SharedCore>()
|
||||
|
|
@ -6755,7 +6802,7 @@ pub fn install_parse(
|
|||
id.0
|
||||
)));
|
||||
}
|
||||
let overlay = SyntaxHighlightView::new(handle, query, theme);
|
||||
let overlay = SyntaxHighlightView::new(handle, theme);
|
||||
win.push_overlay(Box::new(overlay));
|
||||
Ok(true)
|
||||
})?,
|
||||
|
|
|
|||
|
|
@ -3078,9 +3078,15 @@ mod tests {
|
|||
let pidfile = dir.path().join(format!("pid{round}"));
|
||||
let mut sup = ProcessSupervisor::new();
|
||||
sup.set_group_term_grace(Duration::from_millis(300));
|
||||
// Do not let the group leader exit until the background child has
|
||||
// completed `setsid` and published its pid. Without this readiness
|
||||
// gate, teardown can TERM the old process group before `setsid`
|
||||
// runs; the child then dies before creating the pidfile (a race
|
||||
// exposed consistently by the Ubuntu 20260714 runner image).
|
||||
let script = format!(
|
||||
"setsid /bin/sh -c 'echo $$ > {}; exec sleep 30' & echo started",
|
||||
pidfile.display()
|
||||
"setsid /bin/sh -c 'echo $$ > {pid}; exec sleep 30' & \
|
||||
while [ ! -s {pid} ]; do sleep 0.01; done; echo started",
|
||||
pid = pidfile.display()
|
||||
);
|
||||
let id = sup.spawn(sh_group_spec("escapee", &script)).expect("spawn");
|
||||
let ready = drain_until(&mut sup, id, Duration::from_secs(2), |evs| {
|
||||
|
|
|
|||
|
|
@ -1743,12 +1743,6 @@ fn scoped_style_spans(state: &EditorState, vp: &DeclaredViewport) -> Vec<StyleSp
|
|||
let Some(bundle) = handle.current() else {
|
||||
return Vec::new();
|
||||
};
|
||||
let Some(query) = state
|
||||
.syntax_registry
|
||||
.highlights_query(&bundle.language_name)
|
||||
else {
|
||||
return Vec::new();
|
||||
};
|
||||
let theme = state
|
||||
.syntax_registry
|
||||
.theme()
|
||||
|
|
@ -1756,39 +1750,152 @@ fn scoped_style_spans(state: &EditorState, vp: &DeclaredViewport) -> Vec<StyleSp
|
|||
.expect("theme mutex poisoned")
|
||||
.clone();
|
||||
|
||||
let source_len = bundle.source.len() as u64;
|
||||
let source: &[u8] = bundle.source.as_ref();
|
||||
let source_len = source.len() as u64;
|
||||
let vis_start = vp.visible.start.min(source_len);
|
||||
let vis_end = vp.visible.end.min(source_len);
|
||||
if vis_end <= vis_start {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let capture_names = query.capture_names();
|
||||
// Scope the tree-sitter capture walk to the visible byte range so
|
||||
// re-styling on each edit is O(visible), not O(file) — the typing
|
||||
// bottleneck on large files (framing Q#S6). Captures whose nodes
|
||||
// intersect the range are returned, then clipped exactly below.
|
||||
let highlights = crate::syntax::compute_highlight_spans_in_range(
|
||||
&query,
|
||||
&bundle,
|
||||
Some(vis_start as usize..vis_end as usize),
|
||||
);
|
||||
let mut out = Vec::new();
|
||||
for hs in highlights {
|
||||
let s = u64::from(hs.start_byte).max(vis_start);
|
||||
let e = u64::from(hs.end_byte).min(vis_end);
|
||||
if e <= s {
|
||||
continue; // No overlap with the viewport.
|
||||
}
|
||||
let Some(name) = capture_names.get(hs.capture_index as usize) else {
|
||||
// Collect the styled spans from every injection layer, scoping each
|
||||
// capture walk to the visible byte range so re-styling on each edit is
|
||||
// O(visible), not O(file) (framing Q#S6). Each span carries a priority
|
||||
// `(layer_index, capture_order)`: a deeper layer wins over a shallower
|
||||
// one, a later same-depth sibling wins over an earlier one, and within a
|
||||
// layer the wider-first order lets narrower captures override (framing
|
||||
// Q#IJ6). Fully-default styles are dropped (they fold as identity anyway).
|
||||
let mut styled: Vec<StyledLayerSpan> = Vec::new();
|
||||
for (layer_idx, layer) in bundle.layers.iter().enumerate() {
|
||||
let Some(query) = layer.highlight_query.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
let style = theme.lookup(name);
|
||||
let names = query.capture_names();
|
||||
let highlights = crate::syntax::compute_highlight_spans_for(
|
||||
query,
|
||||
&layer.tree,
|
||||
source,
|
||||
Some(vis_start as usize..vis_end as usize),
|
||||
);
|
||||
for (order, hs) in highlights.iter().enumerate() {
|
||||
let s = u64::from(hs.start_byte).max(vis_start);
|
||||
let e = u64::from(hs.end_byte).min(vis_end);
|
||||
if e <= s {
|
||||
continue;
|
||||
}
|
||||
let Some(name) = names.get(hs.capture_index as usize) else {
|
||||
continue;
|
||||
};
|
||||
let style = theme.lookup(name);
|
||||
if style == Style::default() {
|
||||
continue;
|
||||
}
|
||||
styled.push(StyledLayerSpan {
|
||||
start: s,
|
||||
end: e,
|
||||
style,
|
||||
priority: layer_span_priority(layer_idx, layer.depth, order),
|
||||
});
|
||||
}
|
||||
}
|
||||
flatten_layer_spans(&styled)
|
||||
}
|
||||
|
||||
type LayerSpanPriority = (u32, u32);
|
||||
|
||||
/// Build the total ordering shared by the wire flattener and its sibling
|
||||
/// precedence regression test. Keeping the layer index and depth as separate
|
||||
/// inputs makes the former `(depth, capture_order)` bug directly falsifiable:
|
||||
/// two siblings tie on depth but must differ on layer index.
|
||||
fn layer_span_priority(layer_index: usize, _depth: u16, capture_order: usize) -> LayerSpanPriority {
|
||||
(layer_index as u32, capture_order as u32)
|
||||
}
|
||||
|
||||
/// One styled span from a single injection layer, tagged with a priority
|
||||
/// used to resolve overlaps: `(layer_index, capture_order)`, higher wins.
|
||||
/// `layer_index` is the position in `bundle.layers` — depth-ascending, so a
|
||||
/// deeper layer AND a later same-depth sibling both sort after (and thus
|
||||
/// override) an earlier one, exactly matching the grid's shallow-to-deep,
|
||||
/// layer-by-layer paint order. `capture_order` is the index in the layer's
|
||||
/// wider-first list, so within a layer a narrower capture overrides a wider
|
||||
/// one. The pair is unique per span, so the fold order is total (no ties).
|
||||
struct StyledLayerSpan {
|
||||
start: u64,
|
||||
end: u64,
|
||||
style: Style,
|
||||
priority: LayerSpanPriority,
|
||||
}
|
||||
|
||||
/// Flatten possibly-overlapping per-layer styled spans into **disjoint**
|
||||
/// `StyleSpan`s whose per-byte style is the priority-ordered fold of every
|
||||
/// covering span (framing Q#IJ6). Emitting disjoint spans makes the result
|
||||
/// robust to the GPU wire re-sorting spans by start (`replace_style_spans`
|
||||
/// / `merge_style_spans`), which would otherwise destroy producer order.
|
||||
/// A boundary sweep over the (viewport-bounded) span endpoints; adjacent
|
||||
/// equal-style runs are merged for wire economy.
|
||||
fn flatten_layer_spans(styled: &[StyledLayerSpan]) -> Vec<StyleSpan> {
|
||||
if styled.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
// Boundary sweep. Every unique span endpoint is a boundary; between
|
||||
// consecutive boundaries the covering set is constant. An ordered
|
||||
// active-set (activate on start, expire on end) keeps each interval's
|
||||
// fold O(active) rather than O(all spans), so the whole pass is
|
||||
// O(spans·log spans + Σ active) — linear in practice (active is bounded
|
||||
// by overlap depth, not the total span count). This matters because the
|
||||
// file-style summary runs this over the *entire* buffer, not just the
|
||||
// viewport.
|
||||
let mut bounds: Vec<u64> = Vec::with_capacity(styled.len() * 2);
|
||||
for sp in styled {
|
||||
bounds.push(sp.start);
|
||||
bounds.push(sp.end);
|
||||
}
|
||||
bounds.sort_unstable();
|
||||
bounds.dedup();
|
||||
|
||||
// Span indices ordered by start; activated as the sweep reaches them.
|
||||
let mut by_start: Vec<usize> = (0..styled.len()).collect();
|
||||
by_start.sort_by_key(|&i| styled[i].start);
|
||||
let mut next = 0usize;
|
||||
|
||||
// Active covering spans, kept in ascending-priority order so the fold
|
||||
// is a single in-order pass.
|
||||
let mut active: Vec<usize> = Vec::new();
|
||||
|
||||
let mut out: Vec<StyleSpan> = Vec::new();
|
||||
for win in bounds.windows(2) {
|
||||
let (a, b) = (win[0], win[1]);
|
||||
// Activate spans starting at or before `a` (each activates once).
|
||||
while next < by_start.len() && styled[by_start[next]].start <= a {
|
||||
let idx = by_start[next];
|
||||
let pos = active.partition_point(|&j| styled[j].priority < styled[idx].priority);
|
||||
active.insert(pos, idx);
|
||||
next += 1;
|
||||
}
|
||||
// Expire spans that ended at or before `a` (ranges are half-open).
|
||||
active.retain(|&j| styled[j].end > a);
|
||||
if active.is_empty() {
|
||||
continue;
|
||||
}
|
||||
// Fold the active set in ascending priority: a deeper/narrower span
|
||||
// overrides, an attribute-only span still composes (matches the
|
||||
// semantic-client `effective_style_at` contract).
|
||||
let mut style = Style::default();
|
||||
for &j in &active {
|
||||
style = crate::overlay::merge_styles(style, styled[j].style);
|
||||
}
|
||||
if style == Style::default() {
|
||||
continue; // Nothing to render — skip the wire byte.
|
||||
continue;
|
||||
}
|
||||
if let Some(last) = out.last_mut()
|
||||
&& last.range.end == a
|
||||
&& last.style == style
|
||||
{
|
||||
last.range.end = b;
|
||||
continue;
|
||||
}
|
||||
out.push(StyleSpan {
|
||||
range: ByteRange { start: s, end: e },
|
||||
range: ByteRange { start: a, end: b },
|
||||
style,
|
||||
});
|
||||
}
|
||||
|
|
@ -3300,7 +3407,9 @@ mod tests {
|
|||
let handle = parse_view.handle();
|
||||
let req = handle.make_request();
|
||||
let bundle = crate::syntax::run_parse(req).expect("initial rust parse");
|
||||
handle.install(std::sync::Arc::new(bundle));
|
||||
// Mirror the production settle path: resolve each layer's highlight
|
||||
// query before install so the producer can style it (framing Q#IJ2).
|
||||
handle.install(state.syntax_registry.resolve_layer_queries(&bundle));
|
||||
buf.attach_view(Box::new(parse_view));
|
||||
drop(registry);
|
||||
core.set_buffer_path(buffer_id, Some(std::path::PathBuf::from("/tmp/x.rs")));
|
||||
|
|
@ -3309,6 +3418,187 @@ mod tests {
|
|||
handle
|
||||
}
|
||||
|
||||
fn seed_markdown_parse_view(
|
||||
state: &EditorState,
|
||||
buffer_id: BufferId,
|
||||
text: &[u8],
|
||||
) -> crate::syntax::ParseViewHandle {
|
||||
let language = state
|
||||
.syntax_registry
|
||||
.language("markdown")
|
||||
.expect("markdown grammar");
|
||||
let mut core = state.core.borrow_mut();
|
||||
let registry_handle = core.registry.clone();
|
||||
let mut registry = registry_handle.borrow_mut();
|
||||
let buf = registry.get_mut(buffer_id).expect("active buffer");
|
||||
if !text.is_empty() {
|
||||
buf.apply_edit(crate::buffer::EditOp::Insert {
|
||||
pos: 0,
|
||||
bytes: text,
|
||||
})
|
||||
.expect("seed markdown text");
|
||||
}
|
||||
let parse_view = crate::syntax::ParseView::new(buf, language, "markdown".to_owned());
|
||||
let handle = parse_view.handle();
|
||||
let mut req = handle.make_request();
|
||||
req.injection_aliases = state.syntax_registry.injection_alias_snapshot();
|
||||
let bundle = crate::syntax::run_parse(req).expect("markdown parse");
|
||||
handle.install(state.syntax_registry.resolve_layer_queries(&bundle));
|
||||
buf.attach_view(Box::new(parse_view));
|
||||
drop(registry);
|
||||
core.set_buffer_path(buffer_id, Some(std::path::PathBuf::from("/tmp/x.md")));
|
||||
drop(core);
|
||||
state.syntax_registry.attach_view(buffer_id, handle.clone());
|
||||
handle
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wire_producer_emits_disjoint_child_spans_in_fence() {
|
||||
// Framing acceptance #8: `scoped_style_spans` over a ```rust fence
|
||||
// emits a styled span on the `fn` keyword INSIDE the fence — which
|
||||
// only the injected rust layer can produce (the markdown root has
|
||||
// no keyword styling there). The pre-injection single-layer producer
|
||||
// fails this. Emitted spans are disjoint (framing Q#IJ6 flatten).
|
||||
let state = empty_state();
|
||||
let bid = active_buffer(&state);
|
||||
let src = b"# T\n\n```rust\nfn demo() {}\n```\n";
|
||||
seed_markdown_parse_view(&state, bid, src);
|
||||
let vp = DeclaredViewport {
|
||||
buffer_id: bid,
|
||||
visible: ByteRange {
|
||||
start: 0,
|
||||
end: src.len() as u64,
|
||||
},
|
||||
frontend_generation: 0,
|
||||
};
|
||||
let spans = scoped_style_spans(&state, &vp);
|
||||
assert!(!spans.is_empty(), "layered producer emits style spans");
|
||||
|
||||
// Disjoint (flattened): no two spans overlap.
|
||||
let mut sorted = spans.clone();
|
||||
sorted.sort_by_key(|s| s.range.start);
|
||||
for w in sorted.windows(2) {
|
||||
assert!(
|
||||
w[0].range.end <= w[1].range.start,
|
||||
"flattened spans must be disjoint: {:?} then {:?}",
|
||||
w[0].range,
|
||||
w[1].range
|
||||
);
|
||||
}
|
||||
|
||||
// The `fn` keyword inside the fence carries a non-default style.
|
||||
let fn_off = src
|
||||
.windows(2)
|
||||
.position(|w| w == b"fn")
|
||||
.expect("`fn` present in source") as u64;
|
||||
let covering = spans
|
||||
.iter()
|
||||
.find(|s| s.range.start <= fn_off && fn_off < s.range.end)
|
||||
.expect("a span covers the `fn` keyword inside the fence");
|
||||
assert_ne!(
|
||||
covering.style,
|
||||
Style::default(),
|
||||
"the injected rust keyword is styled"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_buffer_summary_flatten_scales_on_large_grammar_file() {
|
||||
// Perf gate (round-1 finding 1): the file-style summary runs the
|
||||
// FLATTENER over the WHOLE buffer, not the viewport. The ordered
|
||||
// active-set sweep keeps the flatten O(n·log n + Σ active); the
|
||||
// pre-sweep O(spans^2) flatten stalled a large grammar-backed file
|
||||
// here. (This guards the *flattener* regression specifically — the
|
||||
// summary's separate per-line dominant-style tally is a pre-existing
|
||||
// O(lines × spans) loop, not addressed or claimed linear here.)
|
||||
use std::fmt::Write as _;
|
||||
let state = empty_state();
|
||||
let bid = active_buffer(&state);
|
||||
let mut src = String::new();
|
||||
for i in 0..1500 {
|
||||
writeln!(
|
||||
src,
|
||||
"pub fn f_{i}(x: u32) -> u32 {{ let y = x + {i}; y * 2 }}"
|
||||
)
|
||||
.expect("write");
|
||||
}
|
||||
seed_rust_parse_view(&state, bid, src.as_bytes());
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
let summary = scoped_file_summary(&state, bid, false);
|
||||
let elapsed = start.elapsed();
|
||||
assert!(!summary.is_empty(), "summary produced for a styled buffer");
|
||||
assert!(
|
||||
elapsed < std::time::Duration::from_secs(1),
|
||||
"full-buffer flatten took {elapsed:?}; the event sweep must stay \
|
||||
~linear (a quadratic flatten regresses here)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flatten_same_depth_sibling_later_layer_wins() {
|
||||
// Round-2 finding: two overlapping spans from different sibling
|
||||
// layers must resolve to the LATER layer (higher layer_index),
|
||||
// matching the grid's layer-by-layer paint order. The old
|
||||
// `(depth, order)` priority tied same-depth siblings, and the
|
||||
// active-set insert then reversed them — making the earlier sibling
|
||||
// win, the opposite of the grid. Layer index in the priority fixes
|
||||
// it.
|
||||
use crate::cell::Color;
|
||||
let red = Style {
|
||||
fg: Color::Indexed(1),
|
||||
..Style::default()
|
||||
};
|
||||
let green = Style {
|
||||
fg: Color::Indexed(2),
|
||||
..Style::default()
|
||||
};
|
||||
// Both spans are depth-1 siblings. Layer index 2 follows layer index
|
||||
// 1, so green must win even though their depth/capture order tie.
|
||||
let sibling_depth = 1;
|
||||
let earlier = layer_span_priority(1, sibling_depth, 0);
|
||||
let later = layer_span_priority(2, sibling_depth, 0);
|
||||
assert!(
|
||||
earlier < later,
|
||||
"the later sibling has higher priority despite equal depth"
|
||||
);
|
||||
let styled = vec![
|
||||
StyledLayerSpan {
|
||||
start: 0,
|
||||
end: 10,
|
||||
style: red,
|
||||
priority: earlier,
|
||||
},
|
||||
StyledLayerSpan {
|
||||
start: 3,
|
||||
end: 6,
|
||||
style: green,
|
||||
priority: later,
|
||||
},
|
||||
];
|
||||
let out = flatten_layer_spans(&styled);
|
||||
// Byte 4 (covered by both) folds to the later layer (green).
|
||||
let covering = out
|
||||
.iter()
|
||||
.find(|s| s.range.start <= 4 && 4 < s.range.end)
|
||||
.expect("byte 4 covered");
|
||||
assert_eq!(
|
||||
covering.style.fg,
|
||||
Color::Indexed(2),
|
||||
"the later sibling layer wins at the overlap"
|
||||
);
|
||||
// Byte 1 (layer 0 only) keeps the base layer's color.
|
||||
let c1 = out
|
||||
.iter()
|
||||
.find(|s| s.range.start <= 1 && 1 < s.range.end)
|
||||
.expect("byte 1 covered");
|
||||
assert_eq!(
|
||||
c1.style.fg,
|
||||
Color::Indexed(1),
|
||||
"a non-overlapping byte keeps the base layer color"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cpp_style_comes_from_lsp_when_no_tree_sitter_grammar() {
|
||||
let state = empty_state();
|
||||
|
|
@ -3402,7 +3692,7 @@ mod tests {
|
|||
);
|
||||
|
||||
let bundle = crate::syntax::run_parse(req).expect("settled rust parse");
|
||||
handle.install(std::sync::Arc::new(bundle));
|
||||
handle.install(state.syntax_registry.resolve_layer_queries(&bundle));
|
||||
assert_eq!(state.syntax_registry.take_parse_job(9001), Some(bid));
|
||||
let settled = s.render_frame(&state);
|
||||
assert!(
|
||||
|
|
|
|||
960
src/syntax.rs
960
src/syntax.rs
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,307 @@
|
|||
// injection_acceptance.rs --- multi-language injection acceptance gates.
|
||||
|
||||
//! Multi-language injection acceptance gates that need the public API
|
||||
//! surface (the Lua async path and a settle-time perf budget). The
|
||||
//! layer-structure, alias-resolution, producer, and GPU gates live next to
|
||||
//! the code they exercise (`syntax.rs` / `semantic_render.rs` /
|
||||
//! `highlight.rs` / `pmacs-gpu`), where `run_parse`, `scoped_style_spans`,
|
||||
//! and `source_color_at` are reachable. See the framing acceptance list in
|
||||
//! `docs/multi-language-injections-framing.md`.
|
||||
|
||||
use std::fmt::Write as _;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use pmacs::buffer::{Buffer, BufferId, EditOp};
|
||||
use pmacs::editor::EditorState;
|
||||
use pmacs::lua_bindings::BufferIdLua;
|
||||
use pmacs::rope::Range;
|
||||
use pmacs::syntax::{self, ParseView, SyntaxRegistry};
|
||||
|
||||
/// Drive `tick_async` until `predicate` holds or a deadline passes.
|
||||
fn pump_async<F: Fn(&EditorState) -> bool>(state: &mut EditorState, predicate: F) {
|
||||
let deadline = Instant::now() + Duration::from_secs(3);
|
||||
while !predicate(state) {
|
||||
assert!(Instant::now() < deadline, "async pump deadline exceeded");
|
||||
state.tick_async();
|
||||
std::thread::sleep(Duration::from_millis(2));
|
||||
}
|
||||
}
|
||||
|
||||
/// Framing acceptance #5 (full Lua async path): an alias added from Lua via
|
||||
/// `pmacs.parse.injection_aliases` must reach the parse worker through the
|
||||
/// dispatch snapshot, so an *asynchronously* dispatched injected parse
|
||||
/// resolves a fence named by the new alias. Testing only the static
|
||||
/// resolver would leave the Lua write-through + snapshot bridge unproven.
|
||||
#[test]
|
||||
fn lua_alias_override_resolves_on_async_parse() {
|
||||
let mut state = EditorState::new();
|
||||
|
||||
// Add a bespoke fence alias from Lua (write-through to the registry).
|
||||
state
|
||||
.lua_host
|
||||
.lua()
|
||||
.load(r#"pmacs.parse.injection_aliases.mydsl = "rust""#)
|
||||
.exec()
|
||||
.expect("set injection alias from Lua");
|
||||
|
||||
// A markdown buffer whose fence uses the new alias.
|
||||
let src = b"# Doc\n\n```mydsl\nfn injected() { let x = 1; }\n```\n";
|
||||
let buf_id = state
|
||||
.lua_host
|
||||
.registry()
|
||||
.borrow_mut()
|
||||
.create_from_bytes("doc.md".to_owned(), src);
|
||||
state
|
||||
.lua_host
|
||||
.lua()
|
||||
.globals()
|
||||
.set("BUF", BufferIdLua(buf_id))
|
||||
.expect("bind BUF");
|
||||
|
||||
// Dispatch asynchronously (the wrapped `_dispatch` records the job; the
|
||||
// per-tick settle path installs the resolved bundle).
|
||||
state
|
||||
.lua_host
|
||||
.lua()
|
||||
.load("pmacs.parse._dispatch(BUF, 'markdown')")
|
||||
.exec()
|
||||
.expect("async dispatch");
|
||||
|
||||
pump_async(&mut state, |s| {
|
||||
s.syntax_registry
|
||||
.view(buf_id)
|
||||
.and_then(|h| h.current())
|
||||
.is_some()
|
||||
});
|
||||
|
||||
let bundle = state
|
||||
.syntax_registry
|
||||
.view(buf_id)
|
||||
.and_then(|h| h.current())
|
||||
.expect("settled bundle");
|
||||
assert!(
|
||||
bundle.layers.iter().any(|l| l.language_name == "rust"),
|
||||
"the Lua-set `mydsl` alias resolved the fence to a rust child layer; \
|
||||
layers: {:?}",
|
||||
bundle
|
||||
.layers
|
||||
.iter()
|
||||
.map(|l| l.language_name.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
/// Round-2 finding: the synchronous parse path (`_parse_now`) must snapshot
|
||||
/// injection aliases too — otherwise a `py` fence (or a Lua-added alias)
|
||||
/// injects asynchronously but not synchronously. The default `py`→python
|
||||
/// alias discriminates the fix: with the empty map it would not resolve.
|
||||
#[test]
|
||||
fn sync_parse_now_resolves_alias() {
|
||||
let state = EditorState::new();
|
||||
let src = b"# Doc\n\n```py\nx = 1\n```\n";
|
||||
let buf_id = state
|
||||
.lua_host
|
||||
.registry()
|
||||
.borrow_mut()
|
||||
.create_from_bytes("doc.md".to_owned(), src);
|
||||
state
|
||||
.lua_host
|
||||
.lua()
|
||||
.globals()
|
||||
.set("BUF", BufferIdLua(buf_id))
|
||||
.expect("bind BUF");
|
||||
state
|
||||
.lua_host
|
||||
.lua()
|
||||
.load("pmacs.parse._parse_now(BUF, 'markdown')")
|
||||
.exec()
|
||||
.expect("synchronous parse");
|
||||
|
||||
let bundle = state
|
||||
.syntax_registry
|
||||
.view(buf_id)
|
||||
.and_then(|h| h.current())
|
||||
.expect("installed bundle");
|
||||
assert!(
|
||||
bundle.layers.iter().any(|l| l.language_name == "python"),
|
||||
"the `py` fence resolved to python on the synchronous `_parse_now` path"
|
||||
);
|
||||
}
|
||||
|
||||
/// Round-2 finding: the injection layer cap must be *observably* surfaced,
|
||||
/// not merely flagged. Drives the real Lua settle path (`syntax.lua`'s tick
|
||||
/// → `_injection_capped` → `pmacs.error`) and asserts the three behaviors:
|
||||
/// surfaced once, suppressed on an unchanged re-parse, and re-armed after
|
||||
/// the file drops below the cap and exceeds it again.
|
||||
#[test]
|
||||
fn injection_cap_surfaced_once_and_rearms_via_lua() {
|
||||
let mut state = EditorState::new();
|
||||
// Capture pmacs.error messages into a Lua global.
|
||||
state
|
||||
.lua_host
|
||||
.lua()
|
||||
.load("_CAP = {}\npmacs.error = function(msg) _CAP[#_CAP + 1] = tostring(msg) end")
|
||||
.exec()
|
||||
.expect("install error capture");
|
||||
|
||||
let capping: String = "```rust\nx\n```\n\n".repeat(4096 + 8);
|
||||
let buf_id = state
|
||||
.lua_host
|
||||
.registry()
|
||||
.borrow_mut()
|
||||
.create_from_bytes("big.md".to_owned(), capping.as_bytes());
|
||||
state
|
||||
.lua_host
|
||||
.lua()
|
||||
.globals()
|
||||
.set("BUF", BufferIdLua(buf_id))
|
||||
.expect("bind BUF");
|
||||
|
||||
let dispatch = |state: &EditorState| {
|
||||
state
|
||||
.lua_host
|
||||
.lua()
|
||||
.load("pmacs.parse._dispatch(BUF, 'markdown')")
|
||||
.exec()
|
||||
.expect("dispatch");
|
||||
};
|
||||
let cap_count = |state: &EditorState| -> usize {
|
||||
state.lua_host.lua().load("return #_CAP").eval().unwrap()
|
||||
};
|
||||
let current =
|
||||
|state: &EditorState| state.syntax_registry.view(buf_id).and_then(|h| h.current());
|
||||
let replace_all = |state: &EditorState, bytes: &[u8]| {
|
||||
let core = state.core.borrow();
|
||||
let mut reg = core.registry.borrow_mut();
|
||||
let buf = reg.get_mut(buf_id).expect("buffer");
|
||||
let len = buf.len();
|
||||
buf.apply_edit(EditOp::Replace {
|
||||
range: Range::new(0, len),
|
||||
bytes,
|
||||
})
|
||||
.expect("replace");
|
||||
};
|
||||
|
||||
// 1) First settle → surfaced exactly once, message names the cap.
|
||||
dispatch(&state);
|
||||
pump_async(&mut state, |s| current(s).is_some());
|
||||
assert_eq!(cap_count(&state), 1, "cap surfaced once on first settle");
|
||||
let msg: String = state.lua_host.lua().load("return _CAP[1]").eval().unwrap();
|
||||
assert!(
|
||||
msg.contains("injection layer cap"),
|
||||
"message names the cap: {msg}"
|
||||
);
|
||||
|
||||
// 2) Re-dispatch with no change → suppressed (still once).
|
||||
let b1 = current(&state).unwrap();
|
||||
dispatch(&state);
|
||||
pump_async(&mut state, |s| {
|
||||
current(s).is_some_and(|b| !Arc::ptr_eq(&b1, &b))
|
||||
});
|
||||
assert_eq!(
|
||||
cap_count(&state),
|
||||
1,
|
||||
"once-per-buffer: no re-warn without change"
|
||||
);
|
||||
|
||||
// 3) Shrink below the cap → warned flag clears, no new error.
|
||||
replace_all(&state, b"# small\n\n```rust\nx\n```\n");
|
||||
let b2 = current(&state).unwrap();
|
||||
dispatch(&state);
|
||||
pump_async(&mut state, |s| {
|
||||
current(s).is_some_and(|b| !Arc::ptr_eq(&b2, &b))
|
||||
});
|
||||
assert_eq!(
|
||||
cap_count(&state),
|
||||
1,
|
||||
"dropping below the cap surfaces no new error"
|
||||
);
|
||||
|
||||
// 4) Grow back above the cap → re-armed, warns once more.
|
||||
replace_all(&state, capping.as_bytes());
|
||||
let b3 = current(&state).unwrap();
|
||||
dispatch(&state);
|
||||
pump_async(&mut state, |s| {
|
||||
current(s).is_some_and(|b| !Arc::ptr_eq(&b3, &b))
|
||||
});
|
||||
assert_eq!(
|
||||
cap_count(&state),
|
||||
2,
|
||||
"re-armed: exceeding the cap again warns once more"
|
||||
);
|
||||
}
|
||||
|
||||
/// Framing acceptance #12: a large all-inline markdown buffer settles
|
||||
/// (root + one cold inline layer per paragraph) within a comfortable
|
||||
/// budget, and the FINAL paragraph still receives an inline layer — the
|
||||
/// tail is not silently dropped. This is the measured guard that keeps
|
||||
/// child-incrementality (Q#IJ8) out of v1.
|
||||
#[test]
|
||||
fn many_paragraph_settle_under_budget_with_tail_covered() {
|
||||
let reg = SyntaxRegistry::new();
|
||||
let n = 200usize;
|
||||
let mut src = String::new();
|
||||
for i in 0..n {
|
||||
// A blank line separates paragraphs; each carries emphasis + a link
|
||||
// so the block grammar injects a markdown_inline layer for it.
|
||||
writeln!(
|
||||
src,
|
||||
"Paragraph {i} with *emphasis* and a [link](http://x).\n"
|
||||
)
|
||||
.expect("write");
|
||||
}
|
||||
// A distinctly-marked final paragraph.
|
||||
let marker = "FINALPARAGRAPH";
|
||||
writeln!(src, "{marker} with *stress*.\n").expect("write");
|
||||
|
||||
let language = reg.language("markdown").expect("markdown grammar");
|
||||
let mut buf = Buffer::new(BufferId::next(), "big.md");
|
||||
buf.apply_edit(EditOp::Insert {
|
||||
pos: 0,
|
||||
bytes: src.as_bytes(),
|
||||
})
|
||||
.expect("seed markdown");
|
||||
let view = ParseView::new(&buf, language, "markdown".to_owned());
|
||||
let handle = view.handle();
|
||||
let _vid = buf.attach_view(Box::new(view));
|
||||
let mut req = handle.make_request();
|
||||
req.injection_aliases = reg.injection_alias_snapshot();
|
||||
|
||||
let start = Instant::now();
|
||||
let bundle = syntax::run_parse(req).expect("layered markdown parse");
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// Catastrophic-regression guard: cold-parsing ~200 tiny inline layers
|
||||
// is milliseconds of work; a generous ceiling avoids debug/CI flakiness
|
||||
// while still catching a blow-up (e.g. accidental O(n^2) layer work).
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(2),
|
||||
"many-paragraph settle took {elapsed:?}, exceeds the budget"
|
||||
);
|
||||
|
||||
let inline_count = bundle
|
||||
.layers
|
||||
.iter()
|
||||
.filter(|l| l.language_name == "markdown_inline")
|
||||
.count();
|
||||
assert!(
|
||||
inline_count >= 100,
|
||||
"most paragraphs produce an inline layer; got {inline_count}"
|
||||
);
|
||||
|
||||
// Tail coverage: an inline layer's tree spans the final paragraph.
|
||||
let marker_off = src.find(marker).expect("marker present");
|
||||
let tail_covered = bundle
|
||||
.layers
|
||||
.iter()
|
||||
.filter(|l| l.language_name == "markdown_inline")
|
||||
.any(|l| {
|
||||
let r = l.tree.root_node();
|
||||
(r.start_byte()..r.end_byte()).contains(&marker_off)
|
||||
});
|
||||
assert!(
|
||||
tail_covered,
|
||||
"the final paragraph still receives an inline layer (no tail loss)"
|
||||
);
|
||||
}
|
||||
|
|
@ -125,9 +125,10 @@ fn m4_1_initial_parse_of_5000_line_file_under_100ms() {
|
|||
language_name: "rust".to_owned(),
|
||||
prior_tree: None,
|
||||
edits: Vec::new(),
|
||||
injection_aliases: Arc::new(std::collections::HashMap::new()),
|
||||
};
|
||||
let bundle = syntax::run_parse(req).expect("parse succeeds");
|
||||
assert_eq!(bundle.tree.root_node().kind(), "source_file");
|
||||
assert_eq!(bundle.root_tree().root_node().kind(), "source_file");
|
||||
assert!(
|
||||
bundle.parse_duration < Duration::from_millis(100),
|
||||
"5000-line cold parse took {:?}, exceeds 100 ms budget",
|
||||
|
|
@ -204,7 +205,7 @@ fn m4_1_incremental_parse_under_5ms() {
|
|||
|
||||
let req = handle.make_request();
|
||||
let bundle = syntax::run_parse(req).expect("incremental parse");
|
||||
assert_eq!(bundle.tree.root_node().kind(), "source_file");
|
||||
assert_eq!(bundle.root_tree().root_node().kind(), "source_file");
|
||||
assert!(
|
||||
bundle.parse_duration < Duration::from_millis(5),
|
||||
"incremental parse took {:?}, exceeds 5 ms budget",
|
||||
|
|
@ -226,6 +227,7 @@ fn m4_1_dispatch_parse_round_trips_via_runtime() {
|
|||
language_name: "rust".to_owned(),
|
||||
prior_tree: None,
|
||||
edits: Vec::new(),
|
||||
injection_aliases: Arc::new(std::collections::HashMap::new()),
|
||||
};
|
||||
let id = rt.dispatch_parse(req, None);
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
|
|
@ -243,7 +245,7 @@ fn m4_1_dispatch_parse_round_trips_via_runtime() {
|
|||
}
|
||||
other => panic!("unexpected outcome: {other:?}"),
|
||||
}
|
||||
assert_eq!(bundle.tree.root_node().kind(), "source_file");
|
||||
assert_eq!(bundle.root_tree().root_node().kind(), "source_file");
|
||||
assert_eq!(bundle.language_name, "rust");
|
||||
}
|
||||
|
||||
|
|
@ -518,11 +520,12 @@ fn open_and_wait_for_parse(path: std::path::PathBuf) -> pmacs::editor::EditorSta
|
|||
state
|
||||
}
|
||||
|
||||
/// Cold parse + highlight-spans extraction for a 4000-line synthetic
|
||||
/// rust file completes in under 100 ms. The acceptance criterion
|
||||
/// covers "rust file opens with full syntax highlighting" --- "open"
|
||||
/// here means the path that produces the data the highlight view
|
||||
/// reads on render: parse + capture-walk. Run under `--release`.
|
||||
/// Root parse + highlight-spans extraction for a 4000-line synthetic
|
||||
/// rust file completes in under 100 ms. Injection expansion is an
|
||||
/// additive phase with its own many-paragraph settle budget in
|
||||
/// `injection_acceptance`; excluding it here keeps this M4 gate aligned
|
||||
/// with `ParseTreeBundle::parse_duration`'s documented root-only
|
||||
/// boundary. Run under `--release`.
|
||||
#[test]
|
||||
#[ignore = "perf gate; requires release build"]
|
||||
fn m4_3_open_rust_file_highlights_under_100ms() {
|
||||
|
|
@ -535,17 +538,18 @@ fn m4_3_open_rust_file_highlights_under_100ms() {
|
|||
.highlights_query("rust")
|
||||
.expect("rust highlights query");
|
||||
|
||||
let started = Instant::now();
|
||||
let req = ParseRequest {
|
||||
source: Arc::from(source),
|
||||
language,
|
||||
language_name: "rust".to_owned(),
|
||||
prior_tree: None,
|
||||
edits: Vec::new(),
|
||||
injection_aliases: Arc::new(std::collections::HashMap::new()),
|
||||
};
|
||||
let bundle = syntax::run_parse(req).expect("parse succeeds");
|
||||
let highlight_started = Instant::now();
|
||||
let spans = syntax::compute_highlight_spans(&query, &bundle);
|
||||
let elapsed = started.elapsed();
|
||||
let elapsed = bundle.parse_duration + highlight_started.elapsed();
|
||||
|
||||
assert!(
|
||||
!spans.is_empty(),
|
||||
|
|
|
|||
Loading…
Reference in New Issue