From ea37a8f7ce7fdcde36699abb58fdf4837a79992e Mon Sep 17 00:00:00 2001 From: Levi Neuwirth Date: Tue, 19 May 2026 13:50:05 -0400 Subject: [PATCH] T M4.5: semantic tokens /range + /full/delta MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backlog item 2 — perf refinement over the v1 full-only request. - src/semantic_tokens.rs: SemanticTokensResponse retains the raw int stream; factored decode(); new apply_delta(prev_raw, v) splices a SemanticTokensDelta (edits:[{start,deleteCount,data}]) over the previous raw — descending-start application so unordered server edits stay valid, bounds clamped, spec-allowed full-instead-of-delta detected and parsed. +4 unit tests. - src/lsp.rs: request_semantic_tokens_range (reuses the SemanticTokens route/store) and request_semantic_tokens_delta (new ResponseRoute::SemanticTokensDelta; absorb splices against the store's retained raw). Capability upgraded to requests:{ full:{ delta:true }, range:true }. - src/lua_bindings.rs: _request_semantic_tokens_range_raw, _request_semantic_tokens_delta_raw, pmacs.semantic_tokens.result_id(sid,uri). - builtin/runtime/lsp.lua: range/delta wrappers; pmacs.lsp.semantic_tokens() auto-prefers delta when a prior result id exists (else full), no longer clears the store (delta needs the retained raw), tags the modeline "(delta)". The range wrapper is exposed without a default command (no viewport source in the bundle yet). - pmacs_fake_lsp.rs: /range and /full/delta arms (delta is an edit script over the /full data). - tests/m4_acceptance.rs: m4_20 (range decode), m4_21 (full seeds rid-1; delta against it splices to the updated 3rd token + rid-2). Gates: lib 1289/0, m4 76/0, m8_1 10/0, m8_9 26/0, m8_10 19/0, m9_1 18/0, m5_8 5/0, m11_5 (--features crdt) 2/0; fmt + clippy clean. Co-Authored-By: Claude Opus 4.7 --- builtin/runtime/lsp.lua | 39 ++++++-- src/bin/pmacs_fake_lsp.rs | 27 ++++++ src/lsp.rs | 100 +++++++++++++++++--- src/lua_bindings.rs | 58 ++++++++++++ src/semantic_tokens.rs | 192 ++++++++++++++++++++++++++++++-------- tests/m4_acceptance.rs | 114 ++++++++++++++++++++++ 6 files changed, 469 insertions(+), 61 deletions(-) diff --git a/builtin/runtime/lsp.lua b/builtin/runtime/lsp.lua index c734888..85c2eae 100644 --- a/builtin/runtime/lsp.lua +++ b/builtin/runtime/lsp.lua @@ -276,6 +276,10 @@ pmacs.lsp.request_code_action = wrap_request(pmacs.lsp._request_code_action_raw) pmacs.lsp.request_execute_command = wrap_request(pmacs.lsp._request_execute_command_raw) pmacs.lsp.request_inlay_hint = wrap_request(pmacs.lsp._request_inlay_hint_raw) pmacs.lsp.request_semantic_tokens = wrap_request(pmacs.lsp._request_semantic_tokens_raw) +pmacs.lsp.request_semantic_tokens_range = + wrap_request(pmacs.lsp._request_semantic_tokens_range_raw) +pmacs.lsp.request_semantic_tokens_delta = + wrap_request(pmacs.lsp._request_semantic_tokens_delta_raw) -- Render an `:await()` failure into a modeline-friendly reason. -- `Handle:await()` raises `{ tag = "cancelled", ... }` when the @@ -699,23 +703,37 @@ function pmacs.lsp.inlay_hints() end) end --- T M4.5 — semantic tokens for the whole buffer. Requests --- `textDocument/semanticTokens/full`, stores the decoded absolute --- tokens, and surfaces a modeline summary (count + first token's --- type, resolved through the server's legend). Data only: wiring --- LSP tokens into styling (a second authority alongside tree-sitter) --- is a separate rendering milestone — a render layer subscribes to --- the same `pmacs.semantic_tokens` store when it lands. +-- T M4.5 — semantic tokens for the whole buffer. Incremental: if a +-- prior result id exists for this buffer, request a +-- `/full/delta` against it (the store keeps the raw int stream to +-- splice on); otherwise a `/full` pull. Either way the store ends +-- with the complete token set + a fresh result id, and a modeline +-- summary (count + first token's type, resolved through the legend) +-- is shown. Data only: wiring LSP tokens into styling (a second +-- authority alongside tree-sitter) is a separate rendering +-- milestone — a render layer subscribes to the same +-- `pmacs.semantic_tokens` store when it lands. +-- +-- `pmacs.lsp.request_semantic_tokens_range` is also exposed (no +-- default command) for a future viewport-aware caller: the bundle +-- has no on-screen-range source, so a "range" command here would +-- just duplicate `/full`. function pmacs.lsp.semantic_tokens() local rec = attached_for_active() if not rec then pmacs.editor.set_status("LSP: no server for active buffer") return end - pmacs.semantic_tokens.clear(rec.server, rec.uri) + -- Don't clear: a delta splices against the retained raw stream. + local prev = pmacs.semantic_tokens.result_id(rec.server, rec.uri) pmacs.async(function() local ok, err = pcall(function() - pmacs.lsp.request_semantic_tokens(rec.server, rec.uri):await() + if prev then + pmacs.lsp.request_semantic_tokens_delta( + rec.server, rec.uri, prev):await() + else + pmacs.lsp.request_semantic_tokens(rec.server, rec.uri):await() + end end) if not ok then pmacs.editor.set_status("LSP: " .. lsp_await_error(err)) @@ -734,8 +752,9 @@ function pmacs.lsp.semantic_tokens() and legend.token_types[first.token_type + 1] or tostring(first.token_type) pmacs.editor.set_status(string.format( - "LSP: %d semantic token%s; first '%s' at %d:%d", + "LSP: %d semantic token%s%s; first '%s' at %d:%d", #toks, (#toks == 1 and "" or "s"), + (prev and " (delta)" or ""), tname, first.line + 1, first.start + 1)) end) end diff --git a/src/bin/pmacs_fake_lsp.rs b/src/bin/pmacs_fake_lsp.rs index b3a176a..dfee2c3 100644 --- a/src/bin/pmacs_fake_lsp.rs +++ b/src/bin/pmacs_fake_lsp.rs @@ -629,6 +629,33 @@ fn main() { }); write_frame(&mut stdout, &resp); } + ("textDocument/semanticTokens/range", Some(idv)) => { + // T M4.5: same shape as /full, scoped to a range. + // One token: line 1 col 0 len 3, variable. + let resp = serde_json::json!({ + "jsonrpc": "2.0", + "id": idv, + "result": { "resultId": "rid-range", "data": [1, 0, 3, 2, 0] } + }); + write_frame(&mut stdout, &resp); + } + ("textDocument/semanticTokens/full/delta", Some(idv)) => { + // T M4.5: a `SemanticTokensDelta` over the /full data + // `[0,0,4,1,1, 0,5,3,2,0, 2,2,7,0,2]` — replace the + // last 5-int group (idx 10..15) with [3,0,9,1,0], so + // token 3 becomes line 3 col 0 len 9, function. + let resp = serde_json::json!({ + "jsonrpc": "2.0", + "id": idv, + "result": { + "resultId": "rid-2", + "edits": [ + { "start": 10, "deleteCount": 5, "data": [3, 0, 9, 1, 0] } + ] + } + }); + write_frame(&mut stdout, &resp); + } ("textDocument/inlayHint", Some(idv)) => { // T M4.5: a type hint (string label, kind 1) and a // parameter hint (label *parts*, kind 2) so both diff --git a/src/lsp.rs b/src/lsp.rs index e87e17f..14b66db 100644 --- a/src/lsp.rs +++ b/src/lsp.rs @@ -837,10 +837,14 @@ enum ResponseRoute { /// Absorb a `textDocument/inlayHint` response into /// [`crate::inlay_hint::InlayHintStore`] at `(server, uri)`. InlayHint { uri: String }, - /// Absorb a `textDocument/semanticTokens/full` response into - /// [`crate::semantic_tokens::SemanticTokenStore`] at `(server, - /// uri)`. + /// Absorb a `textDocument/semanticTokens/full` (or `/range`) + /// response into [`crate::semantic_tokens::SemanticTokenStore`] + /// at `(server, uri)`. SemanticTokens { uri: String }, + /// Absorb a `textDocument/semanticTokens/full/delta` response — + /// spliced against the store's retained raw int stream at + /// `(server, uri)` — back into that same entry. + SemanticTokensDelta { uri: String }, /// Absorb a Location-shaped nav response (references / declaration /// / typeDefinition / implementation) into /// [`crate::locations::LocationsStore`] at `(server, uri, kind)`. @@ -886,6 +890,7 @@ impl ResponseRoute { | ResponseRoute::CodeAction { uri } | ResponseRoute::InlayHint { uri } | ResponseRoute::SemanticTokens { uri } + | ResponseRoute::SemanticTokensDelta { uri } | ResponseRoute::Locations { uri, .. } | ResponseRoute::DocumentSymbol { uri } | ResponseRoute::DocumentHighlight { uri } => uri, @@ -1871,9 +1876,8 @@ impl LspManager { /// Send `textDocument/semanticTokens/full` for `uri`. The response /// (the relative-encoded `data` array) is decoded and absorbed - /// into the semantic-token store at `(sid, uri)`. v1 is full-only - /// — no `/range` or `/full/delta`. Returns the async-runtime - /// [`JobId`] the response will settle. + /// into the semantic-token store at `(sid, uri)`. Returns the + /// async-runtime [`JobId`] the response will settle. pub fn request_semantic_tokens( &mut self, sid: LspServerId, @@ -1888,6 +1892,59 @@ impl LspManager { Ok(job_id) } + /// Send `textDocument/semanticTokens/range` for the `[start, end]` + /// slice of `uri` (the visible viewport, for large files). The + /// response shape is identical to `/full` and shares the same + /// store entry / route. + #[allow(clippy::too_many_arguments)] + pub fn request_semantic_tokens_range( + &mut self, + sid: LspServerId, + uri: impl Into, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Result { + let uri = uri.into(); + let params = json!({ + "textDocument": { "uri": uri.clone() }, + "range": { + "start": { "line": start_line, "character": start_col }, + "end": { "line": end_line, "character": end_col }, + }, + }); + let req_id = self.send_request(sid, "textDocument/semanticTokens/range", params)?; + let job_id = self.register_awaiter(sid, req_id, "textDocument/semanticTokens/range", &uri); + self.pending_routes + .insert((sid, req_id), ResponseRoute::SemanticTokens { uri }); + Ok(job_id) + } + + /// Send `textDocument/semanticTokens/full/delta` for `uri`, + /// passing the `previous_result_id` the last full/delta response + /// carried. The response (a `SemanticTokensDelta`, or a full + /// `SemanticTokens` if the server declined a delta) is spliced + /// against the store's retained raw int stream. + pub fn request_semantic_tokens_delta( + &mut self, + sid: LspServerId, + uri: impl Into, + previous_result_id: impl Into, + ) -> Result { + let uri = uri.into(); + let params = json!({ + "textDocument": { "uri": uri.clone() }, + "previousResultId": previous_result_id.into(), + }); + let req_id = self.send_request(sid, "textDocument/semanticTokens/full/delta", params)?; + let job_id = + self.register_awaiter(sid, req_id, "textDocument/semanticTokens/full/delta", &uri); + self.pending_routes + .insert((sid, req_id), ResponseRoute::SemanticTokensDelta { uri }); + Ok(job_id) + } + /// Send `workspace/executeCommand`. No response route is /// registered: the command result is usually `null` and the real /// effect arrives as a server→client `workspace/applyEdit` (the @@ -2524,6 +2581,21 @@ impl LspManager { .expect("semantic token store mutex poisoned"); guard.set(key, resp); } + ResponseRoute::SemanticTokensDelta { uri } => { + let key = crate::semantic_tokens::SemanticTokenKey::new(server_key, uri.clone()); + let mut guard = self + .semantic_token_store + .lock() + .expect("semantic token store mutex poisoned"); + // Splice against whatever raw stream the previous + // full/delta left here; empty if none (the server + // should then have answered full, which apply_delta + // detects and parses). + let prev_raw = guard.get(&key).map(|r| r.raw.clone()).unwrap_or_default(); + let resp = + crate::semantic_tokens::SemanticTokensResponse::apply_delta(&prev_raw, result); + guard.set(key, resp); + } } } @@ -2983,16 +3055,16 @@ fn default_capabilities() -> Value { // `workspace/inlayHint/refresh`; on-demand re-query is // the v1 model. "inlayHint": { "dynamicRegistration": false }, - // T M4.5 semantic tokens. v1 requests `full` only (no - // `/range`, no `/full/delta`). `formats: ["relative"]` is - // the only encoding LSP defines; the tokenTypes/ - // tokenModifiers lists are the LSP-standard legend the - // client understands — the server intersects its legend - // with these and reports the agreed legend back via - // `semanticTokensProvider.legend`. + // T M4.5 semantic tokens. We support `/full`, the + // `/full/delta` follow-up, and the `/range` viewport + // request. `formats: ["relative"]` is the only encoding + // LSP defines; the tokenTypes/tokenModifiers lists are + // the LSP-standard legend the client understands — the + // server intersects its legend with these and reports the + // agreed legend back via `semanticTokensProvider.legend`. "semanticTokens": { "dynamicRegistration": false, - "requests": { "full": true, "range": false }, + "requests": { "full": { "delta": true }, "range": true }, "formats": ["relative"], "tokenTypes": [ "namespace", "type", "class", "enum", "interface", diff --git a/src/lua_bindings.rs b/src/lua_bindings.rs index 1ce6b55..3002943 100644 --- a/src/lua_bindings.rs +++ b/src/lua_bindings.rs @@ -7567,6 +7567,46 @@ pub fn install_lsp(lua: &Lua, manager: &SharedLspManager) -> mlua::Result<()> { )?; } + { + let m = manager.clone(); + lsp_mod.set( + "_request_semantic_tokens_range_raw", + lua.create_function( + move |_, + (id, uri, sl, sc, el, ec): ( + LspServerIdLua, + String, + u32, + u32, + u32, + u32, + )| { + let job_id = m + .borrow_mut() + .request_semantic_tokens_range(id.0, uri, sl, sc, el, ec) + .map_err(mlua::Error::external)?; + Ok(job_id) + }, + )?, + )?; + } + + { + let m = manager.clone(); + lsp_mod.set( + "_request_semantic_tokens_delta_raw", + lua.create_function( + move |_, (id, uri, prev): (LspServerIdLua, String, String)| { + let job_id = m + .borrow_mut() + .request_semantic_tokens_delta(id.0, uri, prev) + .map_err(mlua::Error::external)?; + Ok(job_id) + }, + )?, + )?; + } + { let m = manager.clone(); lsp_mod.set( @@ -9668,6 +9708,24 @@ pub fn install_semantic_tokens(lua: &Lua, manager: &SharedLspManager) -> mlua::R )?; } + { + // The opaque server cursor from the last full/delta response, + // or nil. Pass it as the `previousResultId` of the next + // `/full/delta` request. + let mgr = manager.clone(); + m.set( + "result_id", + lua.create_function(move |_, (id, uri): (LspServerIdLua, String)| { + let store_handle = mgr.borrow().semantic_token_store(); + let guard = store_handle + .lock() + .expect("semantic token store mutex poisoned"); + let key = SemanticTokenKey::new(id.0.raw().to_string(), uri); + Ok(guard.get(&key).and_then(|r| r.result_id.clone())) + })?, + )?; + } + { let mgr = manager.clone(); m.set( diff --git a/src/semantic_tokens.rs b/src/semantic_tokens.rs index 030abaa..a753927 100644 --- a/src/semantic_tokens.rs +++ b/src/semantic_tokens.rs @@ -45,59 +45,116 @@ pub struct SemanticToken { pub token_modifiers: u32, } -/// Parsed `textDocument/semanticTokens` response. +/// Parsed `textDocument/semanticTokens` (`/full`, `/range`, or a +/// delta-applied `/full/delta`) response. #[derive(Clone, Debug, Default)] pub struct SemanticTokensResponse { /// Tokens in document order (decoded from the relative encoding). pub tokens: Vec, - /// Opaque server cursor for a future delta request (unused by the - /// v1 full-only path; surfaced for completeness). + /// Opaque server cursor. Pass back as `previousResultId` on the + /// next `/full/delta` request. pub result_id: Option, + /// The raw relative-encoded int stream this response decoded + /// from. Retained so a subsequent `/full/delta` can splice its + /// edits against it (the delta is expressed over the *previous + /// data array*, not the decoded tokens). + pub raw: Vec, +} + +/// Decode the flat relative-encoded int stream into absolute tokens. +/// A trailing partial (<5) group is ignored rather than panicking. +fn decode(ints: &[u32]) -> Vec { + let mut tokens = Vec::with_capacity(ints.len() / 5); + let mut line = 0u32; + let mut start = 0u32; + for chunk in ints.chunks_exact(5) { + let (d_line, d_start, length, tt, tm) = (chunk[0], chunk[1], chunk[2], chunk[3], chunk[4]); + // deltaLine is relative to the previous token's line; + // deltaStartChar is relative to the previous token's start + // *iff* on the same line, else absolute from col 0. + line += d_line; + start = if d_line == 0 { + start + d_start + } else { + d_start + }; + tokens.push(SemanticToken { + line, + start, + length, + token_type: tt, + token_modifiers: tm, + }); + } + tokens +} + +fn ints_of(v: &Value, key: &str) -> Vec { + v.get(key) + .and_then(Value::as_array) + .map(|a| a.iter().map(|n| n.as_u64().unwrap_or(0) as u32).collect()) + .unwrap_or_default() } impl SemanticTokensResponse { - /// Parse `SemanticTokens | null`. - /// - /// `data` must be a flat array whose length is a multiple of 5; a - /// trailing partial group (malformed server) is ignored rather - /// than panicking. A `null` / shapeless result yields no tokens. + /// Parse a `SemanticTokens | null` (the `/full` and `/range` + /// shape). A `null` / shapeless result yields no tokens. #[must_use] pub fn from_lsp_value(v: &Value) -> Self { let result_id = v.get("resultId").and_then(Value::as_str).map(str::to_owned); - let Some(data) = v.get("data").and_then(Value::as_array) else { + if v.get("data").and_then(Value::as_array).is_none() { return Self { tokens: Vec::new(), result_id, + raw: Vec::new(), }; - }; - let ints: Vec = data - .iter() - .map(|n| n.as_u64().unwrap_or(0) as u32) - .collect(); - let mut tokens = Vec::with_capacity(ints.len() / 5); - let mut line = 0u32; - let mut start = 0u32; - for chunk in ints.chunks_exact(5) { - let (d_line, d_start, length, tt, tm) = - (chunk[0], chunk[1], chunk[2], chunk[3], chunk[4]); - // deltaLine is relative to the previous token's line; - // deltaStartChar is relative to the previous token's - // start *iff* on the same line, else absolute from col 0. - line += d_line; - start = if d_line == 0 { - start + d_start - } else { - d_start - }; - tokens.push(SemanticToken { - line, - start, - length, - token_type: tt, - token_modifiers: tm, - }); } - Self { tokens, result_id } + let raw = ints_of(v, "data"); + Self { + tokens: decode(&raw), + result_id, + raw, + } + } + + /// Apply a `/full/delta` response against the previous raw int + /// stream. The server is allowed by the spec to answer a delta + /// request with a *full* `SemanticTokens` instead — detected by a + /// `data` array and handled by [`Self::from_lsp_value`]. + /// + /// A `SemanticTokensDelta` is `{ resultId?, edits: [{ start, + /// deleteCount, data? }] }`, each edit a splice over the previous + /// data array. Edits are applied in descending `start` order so + /// earlier indices stay valid regardless of server ordering, and + /// bounds are clamped defensively. + #[must_use] + pub fn apply_delta(prev_raw: &[u32], v: &Value) -> Self { + if v.get("data").and_then(Value::as_array).is_some() { + return Self::from_lsp_value(v); + } + let result_id = v.get("resultId").and_then(Value::as_str).map(str::to_owned); + let mut data = prev_raw.to_vec(); + if let Some(edits) = v.get("edits").and_then(Value::as_array) { + let mut parsed: Vec<(usize, usize, Vec)> = edits + .iter() + .filter_map(|e| { + let start = e.get("start")?.as_u64()? as usize; + let delete = e.get("deleteCount")?.as_u64()? as usize; + Some((start, delete, ints_of(e, "data"))) + }) + .collect(); + parsed.sort_by_key(|e| std::cmp::Reverse(e.0)); + for (start, delete, ins) in parsed { + let s = start.min(data.len()); + let end = start.saturating_add(delete).min(data.len()); + data.splice(s..end, ins); + } + } + Self { + tokens: decode(&data), + result_id, + raw: data, + } } /// True iff the server returned no tokens. @@ -327,10 +384,71 @@ mod tests { token_modifiers: 0, }], result_id: None, + raw: vec![0, 0, 1, 0, 0], }, ); assert_eq!(s.get(&key).unwrap().tokens.len(), 1); s.clear(&key); assert!(s.get(&key).is_none()); } + + #[test] + fn from_lsp_value_retains_raw() { + let v = json!({ "resultId": "r1", "data": [0, 0, 4, 1, 1, 0, 5, 3, 2, 0] }); + let r = SemanticTokensResponse::from_lsp_value(&v); + assert_eq!(r.raw, vec![0, 0, 4, 1, 1, 0, 5, 3, 2, 0]); + assert_eq!(r.tokens.len(), 2); + assert_eq!(r.result_id.as_deref(), Some("r1")); + } + + #[test] + fn apply_delta_splices_previous_raw() { + // prev: two tokens. Delta replaces the 2nd group (indices + // 5..10) with a different 5-int group and bumps resultId. + let prev = [0u32, 0, 4, 1, 1, 0, 5, 3, 2, 0]; + let delta = json!({ + "resultId": "r2", + "edits": [{ "start": 5, "deleteCount": 5, "data": [1, 2, 6, 0, 0] }] + }); + let r = SemanticTokensResponse::apply_delta(&prev, &delta); + assert_eq!(r.result_id.as_deref(), Some("r2")); + assert_eq!(r.raw, vec![0, 0, 4, 1, 1, 1, 2, 6, 0, 0]); + // 2nd token: deltaLine 1 ⇒ line 1, start absolute 2, len 6. + assert_eq!( + r.tokens[1], + SemanticToken { + line: 1, + start: 2, + length: 6, + token_type: 0, + token_modifiers: 0 + } + ); + } + + #[test] + fn apply_delta_multi_edit_descending_safe() { + // Two edits given in ascending order; applying ascending + // would invalidate the second's indices. Delete first group, + // insert a group after the (original) second. + let prev = [0u32, 0, 1, 0, 0, 0, 1, 1, 0, 0]; + let delta = json!({ "edits": [ + { "start": 0, "deleteCount": 5, "data": [] }, + { "start": 10, "deleteCount": 0, "data": [2, 0, 3, 0, 0] } + ]}); + let r = SemanticTokensResponse::apply_delta(&prev, &delta); + assert_eq!(r.raw, vec![0, 1, 1, 0, 0, 2, 0, 3, 0, 0]); + } + + #[test] + fn apply_delta_accepts_full_fallback() { + // Server answered a delta request with a full result. + let r = SemanticTokensResponse::apply_delta( + &[9, 9, 9, 9, 9], + &json!({ "resultId": "f", "data": [0, 0, 2, 1, 0] }), + ); + assert_eq!(r.raw, vec![0, 0, 2, 1, 0]); + assert_eq!(r.tokens.len(), 1); + assert_eq!(r.result_id.as_deref(), Some("f")); + } } diff --git a/tests/m4_acceptance.rs b/tests/m4_acceptance.rs index ff4b5a4..73ac418 100644 --- a/tests/m4_acceptance.rs +++ b/tests/m4_acceptance.rs @@ -3973,6 +3973,120 @@ fn m4_19_semantic_tokens_refresh_repulls_via_server_request() { ); } +/// T M4.5 — `textDocument/semanticTokens/range` through the Lua +/// surface. Same decode path as `/full`, scoped to a range; the +/// fake returns one token. +#[test] +fn m4_20_semantic_tokens_range() { + let mut s = pmacs::editor::EditorState::new(); + spawn_lsp_and_init(&mut s, None); + + let uri = "file:///tmp/m4_20_sem.rs"; + s.lua_host + .lua() + .load(format!( + "pmacs.lsp.did_open(_G._lsp, '{uri}', 1, 'let x = 1\\nlet y = 2\\n') + pmacs.lsp.request_semantic_tokens_range(_G._lsp, '{uri}', 0, 0, 5, 0)" + )) + .exec() + .expect("kick off semantic tokens range request"); + + assert!( + pump_lua_flag( + &mut s, + &format!("#pmacs.semantic_tokens.tokens(_G._lsp, '{uri}') > 0"), + 5, + ), + "range response did not land in the store" + ); + + let (count, tok): (usize, Vec) = s + .lua_host + .lua() + .load(format!( + "local t = pmacs.semantic_tokens.tokens(_G._lsp, '{uri}') + local x = t[1] + return #t, {{ x.line, x.start, x.length, x.token_type, + x.token_modifiers }}" + )) + .eval() + .expect("read range tokens back"); + assert_eq!(count, 1); + assert_eq!(tok, vec![1, 0, 3, 2, 0]); +} + +/// T M4.5 — `/full` then `/full/delta`. The first pull seeds the +/// store (3 tokens, `resultId` "rid-1"); the delta request (driven +/// with that previous id) splices the server's edit over the +/// retained raw stream, yielding the updated 3rd token and the new +/// `resultId` "rid-2". +#[test] +fn m4_21_semantic_tokens_full_then_delta() { + let mut s = pmacs::editor::EditorState::new(); + spawn_lsp_and_init(&mut s, None); + + let uri = "file:///tmp/m4_21_sem.rs"; + s.lua_host + .lua() + .load(format!( + "pmacs.lsp.did_open(_G._lsp, '{uri}', 1, 'fn a() {{}}\\n') + pmacs.lsp.request_semantic_tokens(_G._lsp, '{uri}')" + )) + .exec() + .expect("kick off full request"); + assert!( + pump_lua_flag( + &mut s, + &format!("pmacs.semantic_tokens.result_id(_G._lsp, '{uri}') == 'rid-1'"), + 5, + ), + "full response did not seed the store" + ); + let third_full: Vec = s + .lua_host + .lua() + .load(format!( + "local t = pmacs.semantic_tokens.tokens(_G._lsp, '{uri}') + local x = t[3] + return {{ x.line, x.start, x.length, x.token_type, + x.token_modifiers }}" + )) + .eval() + .expect("read full token 3"); + assert_eq!(third_full, vec![2, 2, 7, 0, 2]); + + // Delta against the seeded result id. + s.lua_host + .lua() + .load(format!( + "pmacs.lsp.request_semantic_tokens_delta(_G._lsp, '{uri}', 'rid-1')" + )) + .exec() + .expect("kick off delta request"); + assert!( + pump_lua_flag( + &mut s, + &format!("pmacs.semantic_tokens.result_id(_G._lsp, '{uri}') == 'rid-2'"), + 5, + ), + "delta response did not update the store" + ); + let (count, third_delta): (usize, Vec) = s + .lua_host + .lua() + .load(format!( + "local t = pmacs.semantic_tokens.tokens(_G._lsp, '{uri}') + local x = t[3] + return #t, {{ x.line, x.start, x.length, x.token_type, + x.token_modifiers }}" + )) + .eval() + .expect("read delta token 3"); + assert_eq!(count, 3, "delta replaced one group, still 3 tokens"); + // [3,0,9,1,0] spliced as the 3rd group: line 0+0+3, abs col 0. + assert_eq!(third_delta, vec![3, 0, 9, 1, 0]); +} + /// Default LSP bundle (`builtin/runtime/lsp.lua`) is wired in: the /// hooks are defined, the namespace tables exist, the user-facing /// commands are registered with the command registry, and the default