diff --git a/builtin/runtime/lsp.lua b/builtin/runtime/lsp.lua index 85adba5..c098eed 100644 --- a/builtin/runtime/lsp.lua +++ b/builtin/runtime/lsp.lua @@ -6,7 +6,7 @@ -- * `pmacs.lsp.go_to_definition` / `pmacs.lsp.format_buffer` / -- `pmacs.lsp.hover_at_cursor` / `pmacs.lsp.signature_help_at_cursor` -- / `pmacs.lsp.rename` / `pmacs.lsp.code_actions` / --- `pmacs.lsp.inlay_hints`, bound to +-- `pmacs.lsp.inlay_hints` / `pmacs.lsp.semantic_tokens`, bound to -- default chords below. -- -- Scope: one server per language across all buffers; async-await @@ -15,9 +15,9 @@ -- (L2), code actions + `workspace/executeCommand` + server→client -- `workspace/applyEdit` (L3), ordered resource-op edits -- (create/rename/delete file) with buffer-registry reconciliation --- (L4), and inlay hints (data + modeline; inline virtual-text --- rendering is a later milestone). Semantic tokens and file-watch --- capability registration are later layers. +-- (L4), inlay hints, and semantic tokens (each data + modeline; +-- wiring them into rendering is a separate rendering milestone). +-- File-watch capability registration is a later layer. pmacs.lsp = pmacs.lsp or {} pmacs.lsp.config = pmacs.lsp.config or {} @@ -273,6 +273,7 @@ pmacs.lsp.request_rename = wrap_request(pmacs.lsp._request_rename_raw) pmacs.lsp.request_code_action = wrap_request(pmacs.lsp._request_code_action_raw) pmacs.lsp.request_execute_command = wrap_request(pmacs.lsp._request_execute_command_raw) pmacs.lsp.request_inlay_hint = wrap_request(pmacs.lsp._request_inlay_hint_raw) +pmacs.lsp.request_semantic_tokens = wrap_request(pmacs.lsp._request_semantic_tokens_raw) -- Render an `:await()` failure into a modeline-friendly reason. -- `Handle:await()` raises `{ tag = "cancelled", ... }` when the @@ -664,6 +665,47 @@ function pmacs.lsp.inlay_hints() end) end +-- T M4.5 — semantic tokens for the whole buffer. Requests +-- `textDocument/semanticTokens/full`, stores the decoded absolute +-- tokens, and surfaces a modeline summary (count + first token's +-- type, resolved through the server's legend). Data only: wiring +-- LSP tokens into styling (a second authority alongside tree-sitter) +-- is a separate rendering milestone — a render layer subscribes to +-- the same `pmacs.semantic_tokens` store when it lands. +function pmacs.lsp.semantic_tokens() + local rec = attached_for_active() + if not rec then + pmacs.editor.set_status("LSP: no server for active buffer") + return + end + pmacs.semantic_tokens.clear(rec.server, rec.uri) + pmacs.async(function() + local ok, err = pcall(function() + pmacs.lsp.request_semantic_tokens(rec.server, rec.uri):await() + end) + if not ok then + pmacs.editor.set_status("LSP: " .. lsp_await_error(err)) + return + end + local toks = pmacs.semantic_tokens.tokens(rec.server, rec.uri) + if not toks or #toks == 0 then + pmacs.editor.set_status("LSP: no semantic tokens") + return + end + local first = toks[1] + -- Resolve the type index through the legend (0-based index -> + -- 1-based Lua array); fall back to the raw index if no legend. + local legend = pmacs.semantic_tokens.legend(rec.server) + local tname = legend and legend.token_types + and legend.token_types[first.token_type + 1] + or tostring(first.token_type) + pmacs.editor.set_status(string.format( + "LSP: %d semantic token%s; first '%s' at %d:%d", + #toks, (#toks == 1 and "" or "s"), + tname, first.line + 1, first.start + 1)) + end) +end + function pmacs.lsp.format_buffer() local rec = attached_for_active() if not rec then @@ -921,6 +963,12 @@ pmacs.command.define { fn = pmacs.lsp.inlay_hints, } +pmacs.command.define { + name = "lsp.semantic-tokens", + description = "Fetch semantic tokens (type-aware classification) for the buffer (LSP).", + fn = pmacs.lsp.semantic_tokens, +} + -- T M4.5 L1 — unwind the cross-file jump ring. Pairs with the -- `pmacs.editor.push_jump()` every navigation action records before -- it moves the cursor. @@ -945,6 +993,7 @@ pmacs.keymap.bind { scope = "global", sequence = "C-c o", command = "lsp.documen pmacs.keymap.bind { scope = "global", sequence = "C-c r", command = "lsp.rename" } pmacs.keymap.bind { scope = "global", sequence = "C-c a", command = "lsp.code-actions" } pmacs.keymap.bind { scope = "global", sequence = "C-c i", command = "lsp.inlay-hints" } +pmacs.keymap.bind { scope = "global", sequence = "C-c y", command = "lsp.semantic-tokens" } pmacs.keymap.bind { scope = "global", sequence = "C-c h", command = "lsp.hover" } pmacs.keymap.bind { scope = "global", sequence = "C-c s", command = "lsp.signature-help" } pmacs.keymap.bind { scope = "global", sequence = "C-c f", command = "lsp.format-buffer" } diff --git a/src/bin/pmacs_fake_lsp.rs b/src/bin/pmacs_fake_lsp.rs index f5bffbb..cc3bd90 100644 --- a/src/bin/pmacs_fake_lsp.rs +++ b/src/bin/pmacs_fake_lsp.rs @@ -121,7 +121,14 @@ fn main() { "completionProvider": { "triggerCharacters": ["."] }, "definitionProvider": true, "documentFormattingProvider": true, - "diagnosticProvider": { "interFileDependencies": false, "workspaceDiagnostics": false } + "diagnosticProvider": { "interFileDependencies": false, "workspaceDiagnostics": false }, + "semanticTokensProvider": { + "legend": { + "tokenTypes": ["namespace", "function", "variable"], + "tokenModifiers": ["declaration", "readonly"] + }, + "full": true + } }, "serverInfo": { "name": "pmacs-fake-lsp", "version": "0.1.0" } } @@ -584,6 +591,21 @@ fn main() { }); write_frame(&mut stdout, &resp); } + ("textDocument/semanticTokens/full", Some(idv)) => { + // T M4.5: relative-encoded `data`. Three tokens: + // [0,0,4,1,1] line 0 col 0 len 4, function, decl + // [0,5,3,2,0] same line col 5 len 3, variable + // [2,2,7,0,2] +2 lines col 2 len 7, namespace, ro + let resp = serde_json::json!({ + "jsonrpc": "2.0", + "id": idv, + "result": { + "resultId": "rid-1", + "data": [0, 0, 4, 1, 1, 0, 5, 3, 2, 0, 2, 2, 7, 0, 2] + } + }); + write_frame(&mut stdout, &resp); + } ("textDocument/inlayHint", Some(idv)) => { // T M4.5: a type hint (string label, kind 1) and a // parameter hint (label *parts*, kind 2) so both diff --git a/src/lib.rs b/src/lib.rs index 685b954..d30c2d0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -102,6 +102,7 @@ pub mod rope; #[cfg(feature = "crdt")] pub mod semantic_client; pub mod semantic_render; +pub mod semantic_tokens; pub mod signature; pub mod socket_path; pub mod symbol; diff --git a/src/lsp.rs b/src/lsp.rs index eff73ae..fd8e72c 100644 --- a/src/lsp.rs +++ b/src/lsp.rs @@ -767,6 +767,10 @@ pub struct LspManager { /// `textDocument/inlayHint` response lands, keyed by `(server, /// uri)`. inlay_hint_store: crate::inlay_hint::SharedInlayHintStore, + /// T M4.5 semantic-token store. Populated when a + /// `textDocument/semanticTokens/full` response lands, keyed by + /// `(server, uri)`. + semantic_token_store: crate::semantic_tokens::SharedSemanticTokenStore, /// Per-server request id → response routing target. /// `request_completion` etc. record an entry here; `handle_response` /// consumes it to absorb the response into the correct store. @@ -833,6 +837,10 @@ enum ResponseRoute { /// Absorb a `textDocument/inlayHint` response into /// [`crate::inlay_hint::InlayHintStore`] at `(server, uri)`. InlayHint { uri: String }, + /// Absorb a `textDocument/semanticTokens/full` response into + /// [`crate::semantic_tokens::SemanticTokenStore`] at `(server, + /// uri)`. + SemanticTokens { uri: String }, /// Absorb a Location-shaped nav response (references / declaration /// / typeDefinition / implementation) into /// [`crate::locations::LocationsStore`] at `(server, uri, kind)`. @@ -877,6 +885,7 @@ impl ResponseRoute { | ResponseRoute::Rename { uri } | ResponseRoute::CodeAction { uri } | ResponseRoute::InlayHint { uri } + | ResponseRoute::SemanticTokens { uri } | ResponseRoute::Locations { uri, .. } | ResponseRoute::DocumentSymbol { uri } | ResponseRoute::DocumentHighlight { uri } => uri, @@ -961,6 +970,7 @@ impl LspManager { rename_store: crate::rename::make_shared_store(), code_action_store: crate::code_action::make_shared_store(), inlay_hint_store: crate::inlay_hint::make_shared_store(), + semantic_token_store: crate::semantic_tokens::make_shared_store(), pending_routes: HashMap::new(), status_tracker: crate::lsp_status::LspStatusTracker::new(), project_servers: HashMap::new(), @@ -1045,6 +1055,12 @@ impl LspManager { self.inlay_hint_store.clone() } + /// Shared semantic-token store (T M4.5). + #[must_use] + pub fn semantic_token_store(&self) -> crate::semantic_tokens::SharedSemanticTokenStore { + self.semantic_token_store.clone() + } + /// T M4.8: per-server status snapshot, derived from the LSP event /// stream. The modeline reads its label from this. #[must_use] @@ -1853,6 +1869,25 @@ impl LspManager { Ok(job_id) } + /// Send `textDocument/semanticTokens/full` for `uri`. The response + /// (the relative-encoded `data` array) is decoded and absorbed + /// into the semantic-token store at `(sid, uri)`. v1 is full-only + /// — no `/range` or `/full/delta`. Returns the async-runtime + /// [`JobId`] the response will settle. + pub fn request_semantic_tokens( + &mut self, + sid: LspServerId, + uri: impl Into, + ) -> Result { + let uri = uri.into(); + let params = json!({ "textDocument": { "uri": uri.clone() } }); + let req_id = self.send_request(sid, "textDocument/semanticTokens/full", params)?; + let job_id = self.register_awaiter(sid, req_id, "textDocument/semanticTokens/full", &uri); + self.pending_routes + .insert((sid, req_id), ResponseRoute::SemanticTokens { uri }); + Ok(job_id) + } + /// Send `workspace/executeCommand`. No response route is /// registered: the command result is usually `null` and the real /// effect arrives as a server→client `workspace/applyEdit` (the @@ -2480,6 +2515,15 @@ impl LspManager { .expect("inlay hint store mutex poisoned"); guard.set(key, resp); } + ResponseRoute::SemanticTokens { uri } => { + let resp = crate::semantic_tokens::SemanticTokensResponse::from_lsp_value(result); + let key = crate::semantic_tokens::SemanticTokenKey::new(server_key, uri.clone()); + let mut guard = self + .semantic_token_store + .lock() + .expect("semantic token store mutex poisoned"); + guard.set(key, resp); + } } } @@ -2931,6 +2975,30 @@ fn default_capabilities() -> Value { // `workspace/inlayHint/refresh`; on-demand re-query is // the v1 model. "inlayHint": { "dynamicRegistration": false }, + // T M4.5 semantic tokens. v1 requests `full` only (no + // `/range`, no `/full/delta`). `formats: ["relative"]` is + // the only encoding LSP defines; the tokenTypes/ + // tokenModifiers lists are the LSP-standard legend the + // client understands — the server intersects its legend + // with these and reports the agreed legend back via + // `semanticTokensProvider.legend`. + "semanticTokens": { + "dynamicRegistration": false, + "requests": { "full": true, "range": false }, + "formats": ["relative"], + "tokenTypes": [ + "namespace", "type", "class", "enum", "interface", + "struct", "typeParameter", "parameter", "variable", + "property", "enumMember", "event", "function", + "method", "macro", "keyword", "modifier", "comment", + "string", "number", "regexp", "operator", "decorator" + ], + "tokenModifiers": [ + "declaration", "definition", "readonly", "static", + "deprecated", "abstract", "async", "modification", + "documentation", "defaultLibrary" + ], + }, "publishDiagnostics": { "relatedInformation": true }, }, }) diff --git a/src/lua_bindings.rs b/src/lua_bindings.rs index 8672b94..1ce6b55 100644 --- a/src/lua_bindings.rs +++ b/src/lua_bindings.rs @@ -7553,6 +7553,20 @@ pub fn install_lsp(lua: &Lua, manager: &SharedLspManager) -> mlua::Result<()> { )?; } + { + let m = manager.clone(); + lsp_mod.set( + "_request_semantic_tokens_raw", + lua.create_function(move |_, (id, uri): (LspServerIdLua, String)| { + let job_id = m + .borrow_mut() + .request_semantic_tokens(id.0, uri) + .map_err(mlua::Error::external)?; + Ok(job_id) + })?, + )?; + } + { let m = manager.clone(); lsp_mod.set( @@ -7829,6 +7843,7 @@ pub fn make_lsp_manager( install_rename(lua, &manager)?; install_code_action(lua, &manager)?; install_inlay_hint(lua, &manager)?; + install_semantic_tokens(lua, &manager)?; Ok(manager) } @@ -8632,6 +8647,9 @@ use crate::hover::{Hover, HoverKey}; use crate::inlay_hint::{InlayHint as LspInlayHint, InlayHintKey}; use crate::locations::{LocationKind, LocationsKey}; use crate::rename::{RenameKey, WorkspaceEditResponse, WorkspaceOp}; +use crate::semantic_tokens::{ + SemanticToken as LspSemanticToken, SemanticTokenKey, SemanticTokensLegend, +}; use crate::signature::{Signature, SignatureHelp, SignatureKey, SignatureParameter}; use crate::symbol::{Symbol as LspSymbol, SymbolKey}; @@ -9574,6 +9592,101 @@ pub fn install_inlay_hint(lua: &Lua, manager: &SharedLspManager) -> mlua::Result Ok(()) } +fn semantic_token_to_lua(lua: &Lua, t: &LspSemanticToken) -> mlua::Result { + let out = lua.create_table_with_capacity(0, 5)?; + out.set("line", t.line)?; + out.set("start", t.start)?; + out.set("length", t.length)?; + out.set("token_type", t.token_type)?; + out.set("token_modifiers", t.token_modifiers)?; + Ok(out) +} + +/// Install `pmacs.semantic_tokens.*` (T M4.5). `tokens(sid, uri)` +/// returns the decoded absolute tokens `{ { line, start, length, +/// token_type, token_modifiers }, … }`; `legend(sid)` returns +/// `{ token_types = {…}, token_modifiers = {…} }` from the server's +/// advertised `semanticTokensProvider.legend` (or nil), so callers +/// resolve the `token_type` index / `token_modifiers` bitset; +/// `clear(sid, uri)` drops the entry. No renderer here — wiring LSP +/// tokens into styling is a separate rendering milestone; this is the +/// data surface that work reads. +pub fn install_semantic_tokens(lua: &Lua, manager: &SharedLspManager) -> mlua::Result<()> { + let pmacs: Table = lua.globals().get("pmacs")?; + let m = lua.create_table()?; + + { + let mgr = manager.clone(); + m.set( + "tokens", + lua.create_function(move |lua, (id, uri): (LspServerIdLua, String)| { + let store_handle = mgr.borrow().semantic_token_store(); + let guard = store_handle + .lock() + .expect("semantic token store mutex poisoned"); + let key = SemanticTokenKey::new(id.0.raw().to_string(), uri); + let out = lua.create_table()?; + if let Some(r) = guard.get(&key) { + for (i, t) in r.tokens.iter().enumerate() { + out.set(i + 1, semantic_token_to_lua(lua, t)?)?; + } + } + Ok(Value::Table(out)) + })?, + )?; + } + + { + let mgr = manager.clone(); + m.set( + "legend", + lua.create_function(move |lua, id: LspServerIdLua| { + // Parse out an owned legend inside the borrow, then + // build the Lua table once the manager borrow is + // dropped. + let parsed = { + let guard = mgr.borrow(); + guard + .capabilities(id.0) + .and_then(SemanticTokensLegend::from_capabilities) + }; + let Some(legend) = parsed else { + return Ok(Value::Nil); + }; + let to_arr = |names: &[String]| -> mlua::Result
{ + let t = lua.create_table_with_capacity(names.len(), 0)?; + for (i, n) in names.iter().enumerate() { + t.set(i + 1, n.as_str())?; + } + Ok(t) + }; + let out = lua.create_table_with_capacity(0, 2)?; + out.set("token_types", to_arr(&legend.token_types)?)?; + out.set("token_modifiers", to_arr(&legend.token_modifiers)?)?; + Ok(Value::Table(out)) + })?, + )?; + } + + { + let mgr = manager.clone(); + m.set( + "clear", + lua.create_function(move |_, (id, uri): (LspServerIdLua, String)| { + let store_handle = mgr.borrow().semantic_token_store(); + let mut guard = store_handle + .lock() + .expect("semantic token store mutex poisoned"); + guard.clear(&SemanticTokenKey::new(id.0.raw().to_string(), uri)); + Ok(()) + })?, + )?; + } + + pmacs.set("semantic_tokens", m)?; + Ok(()) +} + // --------------------------------------------------------------------------- // pmacs.project: project model (T M4.9) // --------------------------------------------------------------------------- diff --git a/src/semantic_tokens.rs b/src/semantic_tokens.rs new file mode 100644 index 0000000..030abaa --- /dev/null +++ b/src/semantic_tokens.rs @@ -0,0 +1,336 @@ +// semantic_tokens.rs --- T M4.5 LSP semantic tokens. + +//! `textDocument/semanticTokens/full` response state. +//! +//! Semantic tokens are the server's type-aware classification of every +//! token in a document (this identifier is a *mutable* `variable`, +//! that one a `function.defaultLibrary`, …). The wire format is a flat +//! `data: number[]` whose every 5 ints describe one token **relative** +//! to the previous one: +//! +//! ```text +//! [deltaLine, deltaStartChar, length, tokenType, tokenModifiers] +//! ``` +//! +//! This module decodes that into a flat list of *absolute* +//! [`SemanticToken`]s, and parses the server's +//! `semanticTokensProvider.legend` so callers can resolve the +//! `token_type` / `token_modifiers` indices to names. +//! +//! Scope: this is the **LSP data layer only**. It is deliberately +//! independent of the M11 semantic-render protocol +//! ([`crate::semantic_render`] / [`crate::semantic_client`]), which +//! projects tree-sitter highlighting into the frontend wire families. +//! Wiring LSP tokens into rendering (a second styling authority, +//! priority vs. tree-sitter) is a separate rendering milestone; like +//! the other LSP features, nothing here paints — Lua reads the store. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use serde_json::Value; + +/// One decoded, **absolute**-positioned semantic token. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SemanticToken { + /// Zero-based line. + pub line: u32, + /// Zero-based start column (UTF-16 code units, per LSP). + pub start: u32, + /// Token length in UTF-16 code units. + pub length: u32, + /// Index into the legend's `token_types`. + pub token_type: u32, + /// Bitset; bit `i` set ⇒ legend's `token_modifiers[i]` applies. + pub token_modifiers: u32, +} + +/// Parsed `textDocument/semanticTokens` response. +#[derive(Clone, Debug, Default)] +pub struct SemanticTokensResponse { + /// Tokens in document order (decoded from the relative encoding). + pub tokens: Vec, + /// Opaque server cursor for a future delta request (unused by the + /// v1 full-only path; surfaced for completeness). + pub result_id: Option, +} + +impl SemanticTokensResponse { + /// Parse `SemanticTokens | null`. + /// + /// `data` must be a flat array whose length is a multiple of 5; a + /// trailing partial group (malformed server) is ignored rather + /// than panicking. A `null` / shapeless result yields no tokens. + #[must_use] + pub fn from_lsp_value(v: &Value) -> Self { + let result_id = v.get("resultId").and_then(Value::as_str).map(str::to_owned); + let Some(data) = v.get("data").and_then(Value::as_array) else { + return Self { + tokens: Vec::new(), + result_id, + }; + }; + let ints: Vec = data + .iter() + .map(|n| n.as_u64().unwrap_or(0) as u32) + .collect(); + let mut tokens = Vec::with_capacity(ints.len() / 5); + let mut line = 0u32; + let mut start = 0u32; + for chunk in ints.chunks_exact(5) { + let (d_line, d_start, length, tt, tm) = + (chunk[0], chunk[1], chunk[2], chunk[3], chunk[4]); + // deltaLine is relative to the previous token's line; + // deltaStartChar is relative to the previous token's + // start *iff* on the same line, else absolute from col 0. + line += d_line; + start = if d_line == 0 { + start + d_start + } else { + d_start + }; + tokens.push(SemanticToken { + line, + start, + length, + token_type: tt, + token_modifiers: tm, + }); + } + Self { tokens, result_id } + } + + /// True iff the server returned no tokens. + #[must_use] + pub fn is_empty(&self) -> bool { + self.tokens.is_empty() + } +} + +/// The server's `semanticTokensProvider.legend`: the ordered name +/// tables the `token_type` index and `token_modifiers` bits map into. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct SemanticTokensLegend { + /// `token_type` index → name. + pub token_types: Vec, + /// Modifier bit position → name. + pub token_modifiers: Vec, +} + +impl SemanticTokensLegend { + /// Pull the legend out of an `initialize` `ServerCapabilities` + /// JSON value. Returns `None` if the server advertises no + /// `semanticTokensProvider` (or it carries no `legend`). + #[must_use] + pub fn from_capabilities(caps: &Value) -> Option { + let legend = caps.get("semanticTokensProvider")?.get("legend")?; + let pull = |key: &str| -> Vec { + legend + .get(key) + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(|v| v.as_str().map(str::to_owned)) + .collect() + }) + .unwrap_or_default() + }; + Some(Self { + token_types: pull("tokenTypes"), + token_modifiers: pull("tokenModifiers"), + }) + } + + /// Resolve a `token_type` index to its legend name. + #[must_use] + pub fn type_name(&self, index: u32) -> Option<&str> { + self.token_types.get(index as usize).map(String::as_str) + } + + /// Resolve a `token_modifiers` bitset to the set legend names, + /// low bit first. + #[must_use] + pub fn modifier_names(&self, bits: u32) -> Vec<&str> { + (0..self.token_modifiers.len()) + .filter(|i| bits & (1 << i) != 0) + .map(|i| self.token_modifiers[i].as_str()) + .collect() + } +} + +/// Per-server, per-uri semantic-token state. +#[derive(Default)] +pub struct SemanticTokenStore { + by_key: HashMap, +} + +/// Key into [`SemanticTokenStore`]. +#[derive(Clone, Eq, PartialEq, Hash, Debug)] +pub struct SemanticTokenKey { + /// Decimal LSP server id. + pub server: String, + /// Document URI the request was made on. + pub uri: String, +} + +impl SemanticTokenKey { + /// Construct a key. + #[must_use] + pub fn new(server: impl Into, uri: impl Into) -> Self { + Self { + server: server.into(), + uri: uri.into(), + } + } +} + +impl SemanticTokenStore { + /// Empty store. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Replace the response at `key`. + pub fn set(&mut self, key: SemanticTokenKey, response: SemanticTokensResponse) { + self.by_key.insert(key, response); + } + + /// Drop the entry at `key`. + pub fn clear(&mut self, key: &SemanticTokenKey) { + self.by_key.remove(key); + } + + /// Look up the entry at `key`. + #[must_use] + pub fn get(&self, key: &SemanticTokenKey) -> Option<&SemanticTokensResponse> { + self.by_key.get(key) + } +} + +/// Cheaply-cloneable shared handle. +pub type SharedSemanticTokenStore = Arc>; + +/// Build a fresh shared store. +#[must_use] +pub fn make_shared_store() -> SharedSemanticTokenStore { + Arc::new(Mutex::new(SemanticTokenStore::new())) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn decodes_relative_encoding_across_lines() { + // Three tokens: + // - line 0, char 0, len 3, type 1, mods 0 + // - same line, +5 chars → char 5, len 2, type 2, mods 0 + // - +2 lines, char 4 (absolute, deltaLine!=0), len 6, type 0, + // mods 0b101 + let v = json!({ + "resultId": "1", + "data": [ + 0, 0, 3, 1, 0, + 0, 5, 2, 2, 0, + 2, 4, 6, 0, 5 + ] + }); + let r = SemanticTokensResponse::from_lsp_value(&v); + assert_eq!(r.result_id.as_deref(), Some("1")); + assert_eq!(r.tokens.len(), 3); + assert_eq!( + r.tokens[0], + SemanticToken { + line: 0, + start: 0, + length: 3, + token_type: 1, + token_modifiers: 0 + } + ); + assert_eq!( + r.tokens[1], + SemanticToken { + line: 0, + start: 5, + length: 2, + token_type: 2, + token_modifiers: 0 + } + ); + assert_eq!( + r.tokens[2], + SemanticToken { + line: 2, + start: 4, + length: 6, + token_type: 0, + token_modifiers: 5 + } + ); + } + + #[test] + fn trailing_partial_group_is_ignored() { + let v = json!({ "data": [0, 0, 3, 1, 0, 9, 9] }); + let r = SemanticTokensResponse::from_lsp_value(&v); + assert_eq!(r.tokens.len(), 1); + } + + #[test] + fn null_response_is_empty() { + let r = SemanticTokensResponse::from_lsp_value(&Value::Null); + assert!(r.is_empty()); + assert!(r.result_id.is_none()); + } + + #[test] + fn legend_parses_and_resolves() { + let caps = json!({ + "semanticTokensProvider": { + "legend": { + "tokenTypes": ["namespace", "type", "function"], + "tokenModifiers": ["declaration", "readonly", "static"] + }, + "full": true + } + }); + let legend = SemanticTokensLegend::from_capabilities(&caps).unwrap(); + assert_eq!(legend.type_name(2), Some("function")); + assert_eq!(legend.type_name(99), None); + // bits 0b101 = declaration + static + assert_eq!(legend.modifier_names(0b101), vec!["declaration", "static"]); + assert_eq!(legend.modifier_names(0), Vec::<&str>::new()); + } + + #[test] + fn no_provider_yields_no_legend() { + assert!( + SemanticTokensLegend::from_capabilities(&json!({ "hoverProvider": true })).is_none() + ); + } + + #[test] + fn store_set_get_clear() { + let mut s = SemanticTokenStore::new(); + let key = SemanticTokenKey::new("1", "file:///a"); + s.set( + key.clone(), + SemanticTokensResponse { + tokens: vec![SemanticToken { + line: 0, + start: 0, + length: 1, + token_type: 0, + token_modifiers: 0, + }], + result_id: None, + }, + ); + assert_eq!(s.get(&key).unwrap().tokens.len(), 1); + s.clear(&key); + assert!(s.get(&key).is_none()); + } +} diff --git a/tests/m4_acceptance.rs b/tests/m4_acceptance.rs index 3a5822e..6d06f03 100644 --- a/tests/m4_acceptance.rs +++ b/tests/m4_acceptance.rs @@ -3804,6 +3804,70 @@ fn m4_16_lua_surface_drives_inlay_hints() { assert!(pad1, "second hint requested paddingRight"); } +/// T M4.5 — semantic tokens through the Lua surface. Drives +/// `pmacs.lsp.request_semantic_tokens` against the fake and asserts +/// the relative `data` encoding decoded to absolute tokens (incl. the +/// multi-line delta where `deltaStartChar` becomes absolute), and +/// that `pmacs.semantic_tokens.legend` exposes the server's legend so +/// the `token_type` index resolves to a name. +#[test] +fn m4_17_lua_surface_drives_semantic_tokens() { + let mut s = pmacs::editor::EditorState::new(); + spawn_lsp_and_init(&mut s, None); + + let uri = "file:///tmp/m4_17_sem.rs"; + s.lua_host + .lua() + .load(format!( + "pmacs.lsp.did_open(_G._lsp, '{uri}', 1, 'fn a() {{}}\\n\\nlet b = 1\\n') + pmacs.lsp.request_semantic_tokens(_G._lsp, '{uri}')" + )) + .exec() + .expect("kick off semantic tokens request"); + + assert!( + pump_lua_flag( + &mut s, + &format!("#pmacs.semantic_tokens.tokens(_G._lsp, '{uri}') > 0"), + 5, + ), + "semantic tokens response did not land in the store" + ); + + // data = [0,0,4,1,1, 0,5,3,2,0, 2,2,7,0,2] + // t1: line 0 start 0 len 4 type 1 mods 1 + // t2: line 0 start 5 len 3 type 2 mods 0 (same-line delta) + // t3: line 2 start 2 len 7 type 0 mods 2 (deltaLine!=0 ⇒ + // startChar absolute) + let (count, t1, t2, t3, type1_name, type0_name): ( + usize, + Vec, + Vec, + Vec, + String, + String, + ) = s + .lua_host + .lua() + .load(format!( + "local t = pmacs.semantic_tokens.tokens(_G._lsp, '{uri}') + local lg = pmacs.semantic_tokens.legend(_G._lsp) + local function tup(x) return {{ x.line, x.start, x.length, + x.token_type, x.token_modifiers }} end + return #t, tup(t[1]), tup(t[2]), tup(t[3]), + lg.token_types[2], lg.token_types[1]" + )) + .eval() + .expect("read semantic tokens + legend back"); + assert_eq!(count, 3); + assert_eq!(t1, vec![0, 0, 4, 1, 1]); + assert_eq!(t2, vec![0, 5, 3, 2, 0]); + assert_eq!(t3, vec![2, 2, 7, 0, 2]); + // Legend resolves the type index (0-based) → name (1-based Lua). + assert_eq!(type1_name, "function"); + assert_eq!(type0_name, "namespace"); +} + /// Default LSP bundle (`builtin/runtime/lsp.lua`) is wired in: the /// hooks are defined, the namespace tables exist, the user-facing /// commands are registered with the command registry, and the default