T M4.5: semantic tokens /range + /full/delta

Backlog item 2 — perf refinement over the v1 full-only request.

- src/semantic_tokens.rs: SemanticTokensResponse retains the raw
  int stream; factored decode(); new apply_delta(prev_raw, v)
  splices a SemanticTokensDelta (edits:[{start,deleteCount,data}])
  over the previous raw — descending-start application so unordered
  server edits stay valid, bounds clamped, spec-allowed
  full-instead-of-delta detected and parsed. +4 unit tests.
- src/lsp.rs: request_semantic_tokens_range (reuses the
  SemanticTokens route/store) and request_semantic_tokens_delta
  (new ResponseRoute::SemanticTokensDelta; absorb splices against
  the store's retained raw). Capability upgraded to
  requests:{ full:{ delta:true }, range:true }.
- src/lua_bindings.rs: _request_semantic_tokens_range_raw,
  _request_semantic_tokens_delta_raw,
  pmacs.semantic_tokens.result_id(sid,uri).
- builtin/runtime/lsp.lua: range/delta wrappers;
  pmacs.lsp.semantic_tokens() auto-prefers delta when a prior
  result id exists (else full), no longer clears the store (delta
  needs the retained raw), tags the modeline "(delta)". The range
  wrapper is exposed without a default command (no viewport source
  in the bundle yet).
- pmacs_fake_lsp.rs: /range and /full/delta arms (delta is an
  edit script over the /full data).
- tests/m4_acceptance.rs: m4_20 (range decode), m4_21 (full seeds
  rid-1; delta against it splices to the updated 3rd token + rid-2).

Gates: lib 1289/0, m4 76/0, m8_1 10/0, m8_9 26/0, m8_10 19/0,
m9_1 18/0, m5_8 5/0, m11_5 (--features crdt) 2/0; fmt + clippy
clean.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

builtin/runtime/lsp.lua

@@ -276,6 +276,10 @@ pmacs.lsp.request_code_action = wrap_request(pmacs.lsp._request_code_action_raw)
 pmacs.lsp.request_execute_command = wrap_request(pmacs.lsp._request_execute_command_raw)
 pmacs.lsp.request_inlay_hint = wrap_request(pmacs.lsp._request_inlay_hint_raw)
 pmacs.lsp.request_semantic_tokens = wrap_request(pmacs.lsp._request_semantic_tokens_raw)
+pmacs.lsp.request_semantic_tokens_range =
+  wrap_request(pmacs.lsp._request_semantic_tokens_range_raw)
+pmacs.lsp.request_semantic_tokens_delta =
+  wrap_request(pmacs.lsp._request_semantic_tokens_delta_raw)
 
 -- Render an `:await()` failure into a modeline-friendly reason.
 -- `Handle:await()` raises `{ tag = "cancelled", ... }` when the
@@ -699,23 +703,37 @@ function pmacs.lsp.inlay_hints()
   end)
 end
 
--- T M4.5 — semantic tokens for the whole buffer. Requests
--- `textDocument/semanticTokens/full`, stores the decoded absolute
--- tokens, and surfaces a modeline summary (count + first token's
--- type, resolved through the server's legend). Data only: wiring
--- LSP tokens into styling (a second authority alongside tree-sitter)
--- is a separate rendering milestone — a render layer subscribes to
--- the same `pmacs.semantic_tokens` store when it lands.
+-- T M4.5 — semantic tokens for the whole buffer. Incremental: if a
+-- prior result id exists for this buffer, request a
+-- `/full/delta` against it (the store keeps the raw int stream to
+-- splice on); otherwise a `/full` pull. Either way the store ends
+-- with the complete token set + a fresh result id, and a modeline
+-- summary (count + first token's type, resolved through the legend)
+-- is shown. Data only: wiring LSP tokens into styling (a second
+-- authority alongside tree-sitter) is a separate rendering
+-- milestone — a render layer subscribes to the same
+-- `pmacs.semantic_tokens` store when it lands.
+--
+-- `pmacs.lsp.request_semantic_tokens_range` is also exposed (no
+-- default command) for a future viewport-aware caller: the bundle
+-- has no on-screen-range source, so a "range" command here would
+-- just duplicate `/full`.
 function pmacs.lsp.semantic_tokens()
   local rec = attached_for_active()
   if not rec then
     pmacs.editor.set_status("LSP: no server for active buffer")
     return
   end
-  pmacs.semantic_tokens.clear(rec.server, rec.uri)
+  -- Don't clear: a delta splices against the retained raw stream.
+  local prev = pmacs.semantic_tokens.result_id(rec.server, rec.uri)
   pmacs.async(function()
     local ok, err = pcall(function()
-      pmacs.lsp.request_semantic_tokens(rec.server, rec.uri):await()
+      if prev then
+        pmacs.lsp.request_semantic_tokens_delta(
+          rec.server, rec.uri, prev):await()
+      else
+        pmacs.lsp.request_semantic_tokens(rec.server, rec.uri):await()
+      end
     end)
     if not ok then
       pmacs.editor.set_status("LSP: " .. lsp_await_error(err))
@@ -734,8 +752,9 @@ function pmacs.lsp.semantic_tokens()
       and legend.token_types[first.token_type + 1]
       or tostring(first.token_type)
     pmacs.editor.set_status(string.format(
-      "LSP: %d semantic token%s; first '%s' at %d:%d",
+      "LSP: %d semantic token%s%s; first '%s' at %d:%d",
       #toks, (#toks == 1 and "" or "s"),
+      (prev and " (delta)" or ""),
       tname, first.line + 1, first.start + 1))
   end)
 end

src/bin/pmacs_fake_lsp.rs

@@ -629,6 +629,33 @@ fn main() {
                 });
                 write_frame(&mut stdout, &resp);
             }
+            ("textDocument/semanticTokens/range", Some(idv)) => {
+                // T M4.5: same shape as /full, scoped to a range.
+                // One token: line 1 col 0 len 3, variable.
+                let resp = serde_json::json!({
+                    "jsonrpc": "2.0",
+                    "id": idv,
+                    "result": { "resultId": "rid-range", "data": [1, 0, 3, 2, 0] }
+                });
+                write_frame(&mut stdout, &resp);
+            }
+            ("textDocument/semanticTokens/full/delta", Some(idv)) => {
+                // T M4.5: a `SemanticTokensDelta` over the /full data
+                // `[0,0,4,1,1, 0,5,3,2,0, 2,2,7,0,2]` — replace the
+                // last 5-int group (idx 10..15) with [3,0,9,1,0], so
+                // token 3 becomes line 3 col 0 len 9, function.
+                let resp = serde_json::json!({
+                    "jsonrpc": "2.0",
+                    "id": idv,
+                    "result": {
+                        "resultId": "rid-2",
+                        "edits": [
+                            { "start": 10, "deleteCount": 5, "data": [3, 0, 9, 1, 0] }
+                        ]
+                    }
+                });
+                write_frame(&mut stdout, &resp);
+            }
             ("textDocument/inlayHint", Some(idv)) => {
                 // T M4.5: a type hint (string label, kind 1) and a
                 // parameter hint (label *parts*, kind 2) so both

src/lsp.rs

@@ -837,10 +837,14 @@ enum ResponseRoute {
     /// Absorb a `textDocument/inlayHint` response into
     /// [`crate::inlay_hint::InlayHintStore`] at `(server, uri)`.
     InlayHint { uri: String },
-    /// Absorb a `textDocument/semanticTokens/full` response into
-    /// [`crate::semantic_tokens::SemanticTokenStore`] at `(server,
-    /// uri)`.
+    /// Absorb a `textDocument/semanticTokens/full` (or `/range`)
+    /// response into [`crate::semantic_tokens::SemanticTokenStore`]
+    /// at `(server, uri)`.
     SemanticTokens { uri: String },
+    /// Absorb a `textDocument/semanticTokens/full/delta` response —
+    /// spliced against the store's retained raw int stream at
+    /// `(server, uri)` — back into that same entry.
+    SemanticTokensDelta { uri: String },
     /// Absorb a Location-shaped nav response (references / declaration
     /// / typeDefinition / implementation) into
     /// [`crate::locations::LocationsStore`] at `(server, uri, kind)`.
@@ -886,6 +890,7 @@ impl ResponseRoute {
             | ResponseRoute::CodeAction { uri }
             | ResponseRoute::InlayHint { uri }
             | ResponseRoute::SemanticTokens { uri }
+            | ResponseRoute::SemanticTokensDelta { uri }
             | ResponseRoute::Locations { uri, .. }
             | ResponseRoute::DocumentSymbol { uri }
             | ResponseRoute::DocumentHighlight { uri } => uri,
@@ -1871,9 +1876,8 @@ impl LspManager {
 
     /// Send `textDocument/semanticTokens/full` for `uri`. The response
     /// (the relative-encoded `data` array) is decoded and absorbed
-    /// into the semantic-token store at `(sid, uri)`. v1 is full-only
-    /// — no `/range` or `/full/delta`. Returns the async-runtime
-    /// [`JobId`] the response will settle.
+    /// into the semantic-token store at `(sid, uri)`. Returns the
+    /// async-runtime [`JobId`] the response will settle.
     pub fn request_semantic_tokens(
         &mut self,
         sid: LspServerId,
@@ -1888,6 +1892,59 @@ impl LspManager {
         Ok(job_id)
     }
 
+    /// Send `textDocument/semanticTokens/range` for the `[start, end]`
+    /// slice of `uri` (the visible viewport, for large files). The
+    /// response shape is identical to `/full` and shares the same
+    /// store entry / route.
+    #[allow(clippy::too_many_arguments)]
+    pub fn request_semantic_tokens_range(
+        &mut self,
+        sid: LspServerId,
+        uri: impl Into<String>,
+        start_line: u32,
+        start_col: u32,
+        end_line: u32,
+        end_col: u32,
+    ) -> Result<JobId, String> {
+        let uri = uri.into();
+        let params = json!({
+            "textDocument": { "uri": uri.clone() },
+            "range": {
+                "start": { "line": start_line, "character": start_col },
+                "end":   { "line": end_line,   "character": end_col   },
+            },
+        });
+        let req_id = self.send_request(sid, "textDocument/semanticTokens/range", params)?;
+        let job_id = self.register_awaiter(sid, req_id, "textDocument/semanticTokens/range", &uri);
+        self.pending_routes
+            .insert((sid, req_id), ResponseRoute::SemanticTokens { uri });
+        Ok(job_id)
+    }
+
+    /// Send `textDocument/semanticTokens/full/delta` for `uri`,
+    /// passing the `previous_result_id` the last full/delta response
+    /// carried. The response (a `SemanticTokensDelta`, or a full
+    /// `SemanticTokens` if the server declined a delta) is spliced
+    /// against the store's retained raw int stream.
+    pub fn request_semantic_tokens_delta(
+        &mut self,
+        sid: LspServerId,
+        uri: impl Into<String>,
+        previous_result_id: impl Into<String>,
+    ) -> Result<JobId, String> {
+        let uri = uri.into();
+        let params = json!({
+            "textDocument": { "uri": uri.clone() },
+            "previousResultId": previous_result_id.into(),
+        });
+        let req_id = self.send_request(sid, "textDocument/semanticTokens/full/delta", params)?;
+        let job_id =
+            self.register_awaiter(sid, req_id, "textDocument/semanticTokens/full/delta", &uri);
+        self.pending_routes
+            .insert((sid, req_id), ResponseRoute::SemanticTokensDelta { uri });
+        Ok(job_id)
+    }
+
     /// Send `workspace/executeCommand`. No response route is
     /// registered: the command result is usually `null` and the real
     /// effect arrives as a server→client `workspace/applyEdit` (the
@@ -2524,6 +2581,21 @@ impl LspManager {
                     .expect("semantic token store mutex poisoned");
                 guard.set(key, resp);
             }
+            ResponseRoute::SemanticTokensDelta { uri } => {
+                let key = crate::semantic_tokens::SemanticTokenKey::new(server_key, uri.clone());
+                let mut guard = self
+                    .semantic_token_store
+                    .lock()
+                    .expect("semantic token store mutex poisoned");
+                // Splice against whatever raw stream the previous
+                // full/delta left here; empty if none (the server
+                // should then have answered full, which apply_delta
+                // detects and parses).
+                let prev_raw = guard.get(&key).map(|r| r.raw.clone()).unwrap_or_default();
+                let resp =
+                    crate::semantic_tokens::SemanticTokensResponse::apply_delta(&prev_raw, result);
+                guard.set(key, resp);
+            }
         }
     }
 
@@ -2983,16 +3055,16 @@ fn default_capabilities() -> Value {
             // `workspace/inlayHint/refresh`; on-demand re-query is
             // the v1 model.
             "inlayHint": { "dynamicRegistration": false },
-            // T M4.5 semantic tokens. v1 requests `full` only (no
-            // `/range`, no `/full/delta`). `formats: ["relative"]` is
-            // the only encoding LSP defines; the tokenTypes/
-            // tokenModifiers lists are the LSP-standard legend the
-            // client understands — the server intersects its legend
-            // with these and reports the agreed legend back via
-            // `semanticTokensProvider.legend`.
+            // T M4.5 semantic tokens. We support `/full`, the
+            // `/full/delta` follow-up, and the `/range` viewport
+            // request. `formats: ["relative"]` is the only encoding
+            // LSP defines; the tokenTypes/tokenModifiers lists are
+            // the LSP-standard legend the client understands — the
+            // server intersects its legend with these and reports the
+            // agreed legend back via `semanticTokensProvider.legend`.
             "semanticTokens": {
                 "dynamicRegistration": false,
-                "requests": { "full": true, "range": false },
+                "requests": { "full": { "delta": true }, "range": true },
                 "formats": ["relative"],
                 "tokenTypes": [
                     "namespace", "type", "class", "enum", "interface",

src/lua_bindings.rs

@@ -7567,6 +7567,46 @@ pub fn install_lsp(lua: &Lua, manager: &SharedLspManager) -> mlua::Result<()> {
         )?;
     }
 
+    {
+        let m = manager.clone();
+        lsp_mod.set(
+            "_request_semantic_tokens_range_raw",
+            lua.create_function(
+                move |_,
+                      (id, uri, sl, sc, el, ec): (
+                    LspServerIdLua,
+                    String,
+                    u32,
+                    u32,
+                    u32,
+                    u32,
+                )| {
+                    let job_id = m
+                        .borrow_mut()
+                        .request_semantic_tokens_range(id.0, uri, sl, sc, el, ec)
+                        .map_err(mlua::Error::external)?;
+                    Ok(job_id)
+                },
+            )?,
+        )?;
+    }
+
+    {
+        let m = manager.clone();
+        lsp_mod.set(
+            "_request_semantic_tokens_delta_raw",
+            lua.create_function(
+                move |_, (id, uri, prev): (LspServerIdLua, String, String)| {
+                    let job_id = m
+                        .borrow_mut()
+                        .request_semantic_tokens_delta(id.0, uri, prev)
+                        .map_err(mlua::Error::external)?;
+                    Ok(job_id)
+                },
+            )?,
+        )?;
+    }
+
     {
         let m = manager.clone();
         lsp_mod.set(
@@ -9668,6 +9708,24 @@ pub fn install_semantic_tokens(lua: &Lua, manager: &SharedLspManager) -> mlua::R
         )?;
     }
 
+    {
+        // The opaque server cursor from the last full/delta response,
+        // or nil. Pass it as the `previousResultId` of the next
+        // `/full/delta` request.
+        let mgr = manager.clone();
+        m.set(
+            "result_id",
+            lua.create_function(move |_, (id, uri): (LspServerIdLua, String)| {
+                let store_handle = mgr.borrow().semantic_token_store();
+                let guard = store_handle
+                    .lock()
+                    .expect("semantic token store mutex poisoned");
+                let key = SemanticTokenKey::new(id.0.raw().to_string(), uri);
+                Ok(guard.get(&key).and_then(|r| r.result_id.clone()))
+            })?,
+        )?;
+    }
+
     {
         let mgr = manager.clone();
         m.set(

src/semantic_tokens.rs

@@ -45,59 +45,116 @@ pub struct SemanticToken {
     pub token_modifiers: u32,
 }
 
-/// Parsed `textDocument/semanticTokens` response.
+/// Parsed `textDocument/semanticTokens` (`/full`, `/range`, or a
+/// delta-applied `/full/delta`) response.
 #[derive(Clone, Debug, Default)]
 pub struct SemanticTokensResponse {
     /// Tokens in document order (decoded from the relative encoding).
     pub tokens: Vec<SemanticToken>,
-    /// Opaque server cursor for a future delta request (unused by the
-    /// v1 full-only path; surfaced for completeness).
+    /// Opaque server cursor. Pass back as `previousResultId` on the
+    /// next `/full/delta` request.
     pub result_id: Option<String>,
+    /// The raw relative-encoded int stream this response decoded
+    /// from. Retained so a subsequent `/full/delta` can splice its
+    /// edits against it (the delta is expressed over the *previous
+    /// data array*, not the decoded tokens).
+    pub raw: Vec<u32>,
+}
+
+/// Decode the flat relative-encoded int stream into absolute tokens.
+/// A trailing partial (<5) group is ignored rather than panicking.
+fn decode(ints: &[u32]) -> Vec<SemanticToken> {
+    let mut tokens = Vec::with_capacity(ints.len() / 5);
+    let mut line = 0u32;
+    let mut start = 0u32;
+    for chunk in ints.chunks_exact(5) {
+        let (d_line, d_start, length, tt, tm) = (chunk[0], chunk[1], chunk[2], chunk[3], chunk[4]);
+        // deltaLine is relative to the previous token's line;
+        // deltaStartChar is relative to the previous token's start
+        // *iff* on the same line, else absolute from col 0.
+        line += d_line;
+        start = if d_line == 0 {
+            start + d_start
+        } else {
+            d_start
+        };
+        tokens.push(SemanticToken {
+            line,
+            start,
+            length,
+            token_type: tt,
+            token_modifiers: tm,
+        });
+    }
+    tokens
+}
+
+fn ints_of(v: &Value, key: &str) -> Vec<u32> {
+    v.get(key)
+        .and_then(Value::as_array)
+        .map(|a| a.iter().map(|n| n.as_u64().unwrap_or(0) as u32).collect())
+        .unwrap_or_default()
 }
 
 impl SemanticTokensResponse {
-    /// Parse `SemanticTokens | null`.
-    ///
-    /// `data` must be a flat array whose length is a multiple of 5; a
-    /// trailing partial group (malformed server) is ignored rather
-    /// than panicking. A `null` / shapeless result yields no tokens.
+    /// Parse a `SemanticTokens | null` (the `/full` and `/range`
+    /// shape). A `null` / shapeless result yields no tokens.
     #[must_use]
     pub fn from_lsp_value(v: &Value) -> Self {
         let result_id = v.get("resultId").and_then(Value::as_str).map(str::to_owned);
-        let Some(data) = v.get("data").and_then(Value::as_array) else {
+        if v.get("data").and_then(Value::as_array).is_none() {
             return Self {
                 tokens: Vec::new(),
                 result_id,
+                raw: Vec::new(),
             };
-        };
-        let ints: Vec<u32> = data
-            .iter()
-            .map(|n| n.as_u64().unwrap_or(0) as u32)
-            .collect();
-        let mut tokens = Vec::with_capacity(ints.len() / 5);
-        let mut line = 0u32;
-        let mut start = 0u32;
-        for chunk in ints.chunks_exact(5) {
-            let (d_line, d_start, length, tt, tm) =
-                (chunk[0], chunk[1], chunk[2], chunk[3], chunk[4]);
-            // deltaLine is relative to the previous token's line;
-            // deltaStartChar is relative to the previous token's
-            // start *iff* on the same line, else absolute from col 0.
-            line += d_line;
-            start = if d_line == 0 {
-                start + d_start
-            } else {
-                d_start
-            };
-            tokens.push(SemanticToken {
-                line,
-                start,
-                length,
-                token_type: tt,
-                token_modifiers: tm,
-            });
         }
-        Self { tokens, result_id }
+        let raw = ints_of(v, "data");
+        Self {
+            tokens: decode(&raw),
+            result_id,
+            raw,
+        }
+    }
+
+    /// Apply a `/full/delta` response against the previous raw int
+    /// stream. The server is allowed by the spec to answer a delta
+    /// request with a *full* `SemanticTokens` instead — detected by a
+    /// `data` array and handled by [`Self::from_lsp_value`].
+    ///
+    /// A `SemanticTokensDelta` is `{ resultId?, edits: [{ start,
+    /// deleteCount, data? }] }`, each edit a splice over the previous
+    /// data array. Edits are applied in descending `start` order so
+    /// earlier indices stay valid regardless of server ordering, and
+    /// bounds are clamped defensively.
+    #[must_use]
+    pub fn apply_delta(prev_raw: &[u32], v: &Value) -> Self {
+        if v.get("data").and_then(Value::as_array).is_some() {
+            return Self::from_lsp_value(v);
+        }
+        let result_id = v.get("resultId").and_then(Value::as_str).map(str::to_owned);
+        let mut data = prev_raw.to_vec();
+        if let Some(edits) = v.get("edits").and_then(Value::as_array) {
+            let mut parsed: Vec<(usize, usize, Vec<u32>)> = edits
+                .iter()
+                .filter_map(|e| {
+                    let start = e.get("start")?.as_u64()? as usize;
+                    let delete = e.get("deleteCount")?.as_u64()? as usize;
+                    Some((start, delete, ints_of(e, "data")))
+                })
+                .collect();
+            parsed.sort_by_key(|e| std::cmp::Reverse(e.0));
+            for (start, delete, ins) in parsed {
+                let s = start.min(data.len());
+                let end = start.saturating_add(delete).min(data.len());
+                data.splice(s..end, ins);
+            }
+        }
+        Self {
+            tokens: decode(&data),
+            result_id,
+            raw: data,
+        }
     }
 
     /// True iff the server returned no tokens.
@@ -327,10 +384,71 @@ mod tests {
                     token_modifiers: 0,
                 }],
                 result_id: None,
+                raw: vec![0, 0, 1, 0, 0],
             },
         );
         assert_eq!(s.get(&key).unwrap().tokens.len(), 1);
         s.clear(&key);
         assert!(s.get(&key).is_none());
     }
+
+    #[test]
+    fn from_lsp_value_retains_raw() {
+        let v = json!({ "resultId": "r1", "data": [0, 0, 4, 1, 1, 0, 5, 3, 2, 0] });
+        let r = SemanticTokensResponse::from_lsp_value(&v);
+        assert_eq!(r.raw, vec![0, 0, 4, 1, 1, 0, 5, 3, 2, 0]);
+        assert_eq!(r.tokens.len(), 2);
+        assert_eq!(r.result_id.as_deref(), Some("r1"));
+    }
+
+    #[test]
+    fn apply_delta_splices_previous_raw() {
+        // prev: two tokens. Delta replaces the 2nd group (indices
+        // 5..10) with a different 5-int group and bumps resultId.
+        let prev = [0u32, 0, 4, 1, 1, 0, 5, 3, 2, 0];
+        let delta = json!({
+            "resultId": "r2",
+            "edits": [{ "start": 5, "deleteCount": 5, "data": [1, 2, 6, 0, 0] }]
+        });
+        let r = SemanticTokensResponse::apply_delta(&prev, &delta);
+        assert_eq!(r.result_id.as_deref(), Some("r2"));
+        assert_eq!(r.raw, vec![0, 0, 4, 1, 1, 1, 2, 6, 0, 0]);
+        // 2nd token: deltaLine 1 ⇒ line 1, start absolute 2, len 6.
+        assert_eq!(
+            r.tokens[1],
+            SemanticToken {
+                line: 1,
+                start: 2,
+                length: 6,
+                token_type: 0,
+                token_modifiers: 0
+            }
+        );
+    }
+
+    #[test]
+    fn apply_delta_multi_edit_descending_safe() {
+        // Two edits given in ascending order; applying ascending
+        // would invalidate the second's indices. Delete first group,
+        // insert a group after the (original) second.
+        let prev = [0u32, 0, 1, 0, 0, 0, 1, 1, 0, 0];
+        let delta = json!({ "edits": [
+            { "start": 0, "deleteCount": 5, "data": [] },
+            { "start": 10, "deleteCount": 0, "data": [2, 0, 3, 0, 0] }
+        ]});
+        let r = SemanticTokensResponse::apply_delta(&prev, &delta);
+        assert_eq!(r.raw, vec![0, 1, 1, 0, 0, 2, 0, 3, 0, 0]);
+    }
+
+    #[test]
+    fn apply_delta_accepts_full_fallback() {
+        // Server answered a delta request with a full result.
+        let r = SemanticTokensResponse::apply_delta(
+            &[9, 9, 9, 9, 9],
+            &json!({ "resultId": "f", "data": [0, 0, 2, 1, 0] }),
+        );
+        assert_eq!(r.raw, vec![0, 0, 2, 1, 0]);
+        assert_eq!(r.tokens.len(), 1);
+        assert_eq!(r.result_id.as_deref(), Some("f"));
+    }
 }

tests/m4_acceptance.rs

@@ -3973,6 +3973,120 @@ fn m4_19_semantic_tokens_refresh_repulls_via_server_request() {
     );
 }
 
+/// T M4.5 — `textDocument/semanticTokens/range` through the Lua
+/// surface. Same decode path as `/full`, scoped to a range; the
+/// fake returns one token.
+#[test]
+fn m4_20_semantic_tokens_range() {
+    let mut s = pmacs::editor::EditorState::new();
+    spawn_lsp_and_init(&mut s, None);
+
+    let uri = "file:///tmp/m4_20_sem.rs";
+    s.lua_host
+        .lua()
+        .load(format!(
+            "pmacs.lsp.did_open(_G._lsp, '{uri}', 1, 'let x = 1\\nlet y = 2\\n')
+             pmacs.lsp.request_semantic_tokens_range(_G._lsp, '{uri}', 0, 0, 5, 0)"
+        ))
+        .exec()
+        .expect("kick off semantic tokens range request");
+
+    assert!(
+        pump_lua_flag(
+            &mut s,
+            &format!("#pmacs.semantic_tokens.tokens(_G._lsp, '{uri}') > 0"),
+            5,
+        ),
+        "range response did not land in the store"
+    );
+
+    let (count, tok): (usize, Vec<u32>) = s
+        .lua_host
+        .lua()
+        .load(format!(
+            "local t = pmacs.semantic_tokens.tokens(_G._lsp, '{uri}')
+             local x = t[1]
+             return #t, {{ x.line, x.start, x.length, x.token_type,
+               x.token_modifiers }}"
+        ))
+        .eval()
+        .expect("read range tokens back");
+    assert_eq!(count, 1);
+    assert_eq!(tok, vec![1, 0, 3, 2, 0]);
+}
+
+/// T M4.5 — `/full` then `/full/delta`. The first pull seeds the
+/// store (3 tokens, `resultId` "rid-1"); the delta request (driven
+/// with that previous id) splices the server's edit over the
+/// retained raw stream, yielding the updated 3rd token and the new
+/// `resultId` "rid-2".
+#[test]
+fn m4_21_semantic_tokens_full_then_delta() {
+    let mut s = pmacs::editor::EditorState::new();
+    spawn_lsp_and_init(&mut s, None);
+
+    let uri = "file:///tmp/m4_21_sem.rs";
+    s.lua_host
+        .lua()
+        .load(format!(
+            "pmacs.lsp.did_open(_G._lsp, '{uri}', 1, 'fn a() {{}}\\n')
+             pmacs.lsp.request_semantic_tokens(_G._lsp, '{uri}')"
+        ))
+        .exec()
+        .expect("kick off full request");
+    assert!(
+        pump_lua_flag(
+            &mut s,
+            &format!("pmacs.semantic_tokens.result_id(_G._lsp, '{uri}') == 'rid-1'"),
+            5,
+        ),
+        "full response did not seed the store"
+    );
+    let third_full: Vec<u32> = s
+        .lua_host
+        .lua()
+        .load(format!(
+            "local t = pmacs.semantic_tokens.tokens(_G._lsp, '{uri}')
+             local x = t[3]
+             return {{ x.line, x.start, x.length, x.token_type,
+               x.token_modifiers }}"
+        ))
+        .eval()
+        .expect("read full token 3");
+    assert_eq!(third_full, vec![2, 2, 7, 0, 2]);
+
+    // Delta against the seeded result id.
+    s.lua_host
+        .lua()
+        .load(format!(
+            "pmacs.lsp.request_semantic_tokens_delta(_G._lsp, '{uri}', 'rid-1')"
+        ))
+        .exec()
+        .expect("kick off delta request");
+    assert!(
+        pump_lua_flag(
+            &mut s,
+            &format!("pmacs.semantic_tokens.result_id(_G._lsp, '{uri}') == 'rid-2'"),
+            5,
+        ),
+        "delta response did not update the store"
+    );
+    let (count, third_delta): (usize, Vec<u32>) = s
+        .lua_host
+        .lua()
+        .load(format!(
+            "local t = pmacs.semantic_tokens.tokens(_G._lsp, '{uri}')
+             local x = t[3]
+             return #t, {{ x.line, x.start, x.length, x.token_type,
+               x.token_modifiers }}"
+        ))
+        .eval()
+        .expect("read delta token 3");
+    assert_eq!(count, 3, "delta replaced one group, still 3 tokens");
+    // [3,0,9,1,0] spliced as the 3rd group: line 0+0+3, abs col 0.
+    assert_eq!(third_delta, vec![3, 0, 9, 1, 0]);
+}
+
 /// Default LSP bundle (`builtin/runtime/lsp.lua`) is wired in: the
 /// hooks are defined, the namespace tables exist, the user-facing
 /// commands are registered with the command registry, and the default