T M4.5: semantic tokens (textDocument/semanticTokens/full)
LSP data layer only — independent of the M11 semantic-render
protocol (semantic_render.rs / semantic_client.rs, tree-sitter →
frontend wire families). No InstanceMessage family added; wiring
LSP tokens into styling is a separate rendering milestone. Same
shape as every sibling LSP feature: typed store + async request
+ Lua surface + command + modeline summary.
- src/semantic_tokens.rs: decode the 5-int relative encoding
(deltaLine, deltaStartChar, length, tokenType, tokenModifiers)
into absolute SemanticToken{line,start,length,token_type,
token_modifiers}, with the same-line-vs-new-line deltaStartChar
rule and defensive truncation of a malformed trailing group.
SemanticTokensLegend::from_capabilities parses
semanticTokensProvider.legend and resolves type index / modifier
bitset to names. Store keyed (server, uri). 6 unit tests.
- src/lsp.rs: store + accessor, ResponseRoute::SemanticTokens +
absorb, request_semantic_tokens (/full; v1 no range/delta),
textDocument.semanticTokens client capability (full-only,
formats=[relative], standard LSP legend).
- src/lua_bindings.rs: _request_semantic_tokens_raw,
pmacs.semantic_tokens.{tokens, legend, clear} (legend reads the
per-server initialize capabilities).
- pmacs_fake_lsp.rs: semanticTokensProvider.legend in initialize;
textDocument/semanticTokens/full arm with relative-encoded data.
- builtin/runtime/lsp.lua: pmacs.lsp.semantic_tokens() requests
full, stores, modeline summary (first token's type resolved via
legend); lsp.semantic-tokens command + C-c y.
- tests/m4_acceptance.rs: m4_17 drives the request via the Lua
surface, asserts decoded absolute tokens (incl. deltaLine!=0 ⇒
absolute startChar) and legend index→name resolution.
Gates: lib 1285/0, m4 72/0, m8_1 10/0, m8_9 26/0, m8_10 19/0,
m9_1 18/0, m5_8 5/0, m11_5 (--features crdt) 2/0 (confirms no
collision with the M11 render protocol); fmt + clippy clean.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
parent
bdc9d48005
commit
cf2947f7f9
|
|
@ -6,7 +6,7 @@
|
|||
-- * `pmacs.lsp.go_to_definition` / `pmacs.lsp.format_buffer` /
|
||||
-- `pmacs.lsp.hover_at_cursor` / `pmacs.lsp.signature_help_at_cursor`
|
||||
-- / `pmacs.lsp.rename` / `pmacs.lsp.code_actions` /
|
||||
-- `pmacs.lsp.inlay_hints`, bound to
|
||||
-- `pmacs.lsp.inlay_hints` / `pmacs.lsp.semantic_tokens`, bound to
|
||||
-- default chords below.
|
||||
--
|
||||
-- Scope: one server per language across all buffers; async-await
|
||||
|
|
@ -15,9 +15,9 @@
|
|||
-- (L2), code actions + `workspace/executeCommand` + server→client
|
||||
-- `workspace/applyEdit` (L3), ordered resource-op edits
|
||||
-- (create/rename/delete file) with buffer-registry reconciliation
|
||||
-- (L4), and inlay hints (data + modeline; inline virtual-text
|
||||
-- rendering is a later milestone). Semantic tokens and file-watch
|
||||
-- capability registration are later layers.
|
||||
-- (L4), inlay hints, and semantic tokens (each data + modeline;
|
||||
-- wiring them into rendering is a separate rendering milestone).
|
||||
-- File-watch capability registration is a later layer.
|
||||
|
||||
pmacs.lsp = pmacs.lsp or {}
|
||||
pmacs.lsp.config = pmacs.lsp.config or {}
|
||||
|
|
@ -273,6 +273,7 @@ pmacs.lsp.request_rename = wrap_request(pmacs.lsp._request_rename_raw)
|
|||
pmacs.lsp.request_code_action = wrap_request(pmacs.lsp._request_code_action_raw)
|
||||
pmacs.lsp.request_execute_command = wrap_request(pmacs.lsp._request_execute_command_raw)
|
||||
pmacs.lsp.request_inlay_hint = wrap_request(pmacs.lsp._request_inlay_hint_raw)
|
||||
pmacs.lsp.request_semantic_tokens = wrap_request(pmacs.lsp._request_semantic_tokens_raw)
|
||||
|
||||
-- Render an `:await()` failure into a modeline-friendly reason.
|
||||
-- `Handle:await()` raises `{ tag = "cancelled", ... }` when the
|
||||
|
|
@ -664,6 +665,47 @@ function pmacs.lsp.inlay_hints()
|
|||
end)
|
||||
end
|
||||
|
||||
-- T M4.5 — semantic tokens for the whole buffer. Requests
|
||||
-- `textDocument/semanticTokens/full`, stores the decoded absolute
|
||||
-- tokens, and surfaces a modeline summary (count + first token's
|
||||
-- type, resolved through the server's legend). Data only: wiring
|
||||
-- LSP tokens into styling (a second authority alongside tree-sitter)
|
||||
-- is a separate rendering milestone — a render layer subscribes to
|
||||
-- the same `pmacs.semantic_tokens` store when it lands.
|
||||
function pmacs.lsp.semantic_tokens()
|
||||
local rec = attached_for_active()
|
||||
if not rec then
|
||||
pmacs.editor.set_status("LSP: no server for active buffer")
|
||||
return
|
||||
end
|
||||
pmacs.semantic_tokens.clear(rec.server, rec.uri)
|
||||
pmacs.async(function()
|
||||
local ok, err = pcall(function()
|
||||
pmacs.lsp.request_semantic_tokens(rec.server, rec.uri):await()
|
||||
end)
|
||||
if not ok then
|
||||
pmacs.editor.set_status("LSP: " .. lsp_await_error(err))
|
||||
return
|
||||
end
|
||||
local toks = pmacs.semantic_tokens.tokens(rec.server, rec.uri)
|
||||
if not toks or #toks == 0 then
|
||||
pmacs.editor.set_status("LSP: no semantic tokens")
|
||||
return
|
||||
end
|
||||
local first = toks[1]
|
||||
-- Resolve the type index through the legend (0-based index ->
|
||||
-- 1-based Lua array); fall back to the raw index if no legend.
|
||||
local legend = pmacs.semantic_tokens.legend(rec.server)
|
||||
local tname = legend and legend.token_types
|
||||
and legend.token_types[first.token_type + 1]
|
||||
or tostring(first.token_type)
|
||||
pmacs.editor.set_status(string.format(
|
||||
"LSP: %d semantic token%s; first '%s' at %d:%d",
|
||||
#toks, (#toks == 1 and "" or "s"),
|
||||
tname, first.line + 1, first.start + 1))
|
||||
end)
|
||||
end
|
||||
|
||||
function pmacs.lsp.format_buffer()
|
||||
local rec = attached_for_active()
|
||||
if not rec then
|
||||
|
|
@ -921,6 +963,12 @@ pmacs.command.define {
|
|||
fn = pmacs.lsp.inlay_hints,
|
||||
}
|
||||
|
||||
pmacs.command.define {
|
||||
name = "lsp.semantic-tokens",
|
||||
description = "Fetch semantic tokens (type-aware classification) for the buffer (LSP).",
|
||||
fn = pmacs.lsp.semantic_tokens,
|
||||
}
|
||||
|
||||
-- T M4.5 L1 — unwind the cross-file jump ring. Pairs with the
|
||||
-- `pmacs.editor.push_jump()` every navigation action records before
|
||||
-- it moves the cursor.
|
||||
|
|
@ -945,6 +993,7 @@ pmacs.keymap.bind { scope = "global", sequence = "C-c o", command = "lsp.documen
|
|||
pmacs.keymap.bind { scope = "global", sequence = "C-c r", command = "lsp.rename" }
|
||||
pmacs.keymap.bind { scope = "global", sequence = "C-c a", command = "lsp.code-actions" }
|
||||
pmacs.keymap.bind { scope = "global", sequence = "C-c i", command = "lsp.inlay-hints" }
|
||||
pmacs.keymap.bind { scope = "global", sequence = "C-c y", command = "lsp.semantic-tokens" }
|
||||
pmacs.keymap.bind { scope = "global", sequence = "C-c h", command = "lsp.hover" }
|
||||
pmacs.keymap.bind { scope = "global", sequence = "C-c s", command = "lsp.signature-help" }
|
||||
pmacs.keymap.bind { scope = "global", sequence = "C-c f", command = "lsp.format-buffer" }
|
||||
|
|
|
|||
|
|
@ -121,7 +121,14 @@ fn main() {
|
|||
"completionProvider": { "triggerCharacters": ["."] },
|
||||
"definitionProvider": true,
|
||||
"documentFormattingProvider": true,
|
||||
"diagnosticProvider": { "interFileDependencies": false, "workspaceDiagnostics": false }
|
||||
"diagnosticProvider": { "interFileDependencies": false, "workspaceDiagnostics": false },
|
||||
"semanticTokensProvider": {
|
||||
"legend": {
|
||||
"tokenTypes": ["namespace", "function", "variable"],
|
||||
"tokenModifiers": ["declaration", "readonly"]
|
||||
},
|
||||
"full": true
|
||||
}
|
||||
},
|
||||
"serverInfo": { "name": "pmacs-fake-lsp", "version": "0.1.0" }
|
||||
}
|
||||
|
|
@ -584,6 +591,21 @@ fn main() {
|
|||
});
|
||||
write_frame(&mut stdout, &resp);
|
||||
}
|
||||
("textDocument/semanticTokens/full", Some(idv)) => {
|
||||
// T M4.5: relative-encoded `data`. Three tokens:
|
||||
// [0,0,4,1,1] line 0 col 0 len 4, function, decl
|
||||
// [0,5,3,2,0] same line col 5 len 3, variable
|
||||
// [2,2,7,0,2] +2 lines col 2 len 7, namespace, ro
|
||||
let resp = serde_json::json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": idv,
|
||||
"result": {
|
||||
"resultId": "rid-1",
|
||||
"data": [0, 0, 4, 1, 1, 0, 5, 3, 2, 0, 2, 2, 7, 0, 2]
|
||||
}
|
||||
});
|
||||
write_frame(&mut stdout, &resp);
|
||||
}
|
||||
("textDocument/inlayHint", Some(idv)) => {
|
||||
// T M4.5: a type hint (string label, kind 1) and a
|
||||
// parameter hint (label *parts*, kind 2) so both
|
||||
|
|
|
|||
|
|
@ -102,6 +102,7 @@ pub mod rope;
|
|||
#[cfg(feature = "crdt")]
|
||||
pub mod semantic_client;
|
||||
pub mod semantic_render;
|
||||
pub mod semantic_tokens;
|
||||
pub mod signature;
|
||||
pub mod socket_path;
|
||||
pub mod symbol;
|
||||
|
|
|
|||
68
src/lsp.rs
68
src/lsp.rs
|
|
@ -767,6 +767,10 @@ pub struct LspManager {
|
|||
/// `textDocument/inlayHint` response lands, keyed by `(server,
|
||||
/// uri)`.
|
||||
inlay_hint_store: crate::inlay_hint::SharedInlayHintStore,
|
||||
/// T M4.5 semantic-token store. Populated when a
|
||||
/// `textDocument/semanticTokens/full` response lands, keyed by
|
||||
/// `(server, uri)`.
|
||||
semantic_token_store: crate::semantic_tokens::SharedSemanticTokenStore,
|
||||
/// Per-server request id → response routing target.
|
||||
/// `request_completion` etc. record an entry here; `handle_response`
|
||||
/// consumes it to absorb the response into the correct store.
|
||||
|
|
@ -833,6 +837,10 @@ enum ResponseRoute {
|
|||
/// Absorb a `textDocument/inlayHint` response into
|
||||
/// [`crate::inlay_hint::InlayHintStore`] at `(server, uri)`.
|
||||
InlayHint { uri: String },
|
||||
/// Absorb a `textDocument/semanticTokens/full` response into
|
||||
/// [`crate::semantic_tokens::SemanticTokenStore`] at `(server,
|
||||
/// uri)`.
|
||||
SemanticTokens { uri: String },
|
||||
/// Absorb a Location-shaped nav response (references / declaration
|
||||
/// / typeDefinition / implementation) into
|
||||
/// [`crate::locations::LocationsStore`] at `(server, uri, kind)`.
|
||||
|
|
@ -877,6 +885,7 @@ impl ResponseRoute {
|
|||
| ResponseRoute::Rename { uri }
|
||||
| ResponseRoute::CodeAction { uri }
|
||||
| ResponseRoute::InlayHint { uri }
|
||||
| ResponseRoute::SemanticTokens { uri }
|
||||
| ResponseRoute::Locations { uri, .. }
|
||||
| ResponseRoute::DocumentSymbol { uri }
|
||||
| ResponseRoute::DocumentHighlight { uri } => uri,
|
||||
|
|
@ -961,6 +970,7 @@ impl LspManager {
|
|||
rename_store: crate::rename::make_shared_store(),
|
||||
code_action_store: crate::code_action::make_shared_store(),
|
||||
inlay_hint_store: crate::inlay_hint::make_shared_store(),
|
||||
semantic_token_store: crate::semantic_tokens::make_shared_store(),
|
||||
pending_routes: HashMap::new(),
|
||||
status_tracker: crate::lsp_status::LspStatusTracker::new(),
|
||||
project_servers: HashMap::new(),
|
||||
|
|
@ -1045,6 +1055,12 @@ impl LspManager {
|
|||
self.inlay_hint_store.clone()
|
||||
}
|
||||
|
||||
/// Shared semantic-token store (T M4.5).
|
||||
#[must_use]
|
||||
pub fn semantic_token_store(&self) -> crate::semantic_tokens::SharedSemanticTokenStore {
|
||||
self.semantic_token_store.clone()
|
||||
}
|
||||
|
||||
/// T M4.8: per-server status snapshot, derived from the LSP event
|
||||
/// stream. The modeline reads its label from this.
|
||||
#[must_use]
|
||||
|
|
@ -1853,6 +1869,25 @@ impl LspManager {
|
|||
Ok(job_id)
|
||||
}
|
||||
|
||||
/// Send `textDocument/semanticTokens/full` for `uri`. The response
|
||||
/// (the relative-encoded `data` array) is decoded and absorbed
|
||||
/// into the semantic-token store at `(sid, uri)`. v1 is full-only
|
||||
/// — no `/range` or `/full/delta`. Returns the async-runtime
|
||||
/// [`JobId`] the response will settle.
|
||||
pub fn request_semantic_tokens(
|
||||
&mut self,
|
||||
sid: LspServerId,
|
||||
uri: impl Into<String>,
|
||||
) -> Result<JobId, String> {
|
||||
let uri = uri.into();
|
||||
let params = json!({ "textDocument": { "uri": uri.clone() } });
|
||||
let req_id = self.send_request(sid, "textDocument/semanticTokens/full", params)?;
|
||||
let job_id = self.register_awaiter(sid, req_id, "textDocument/semanticTokens/full", &uri);
|
||||
self.pending_routes
|
||||
.insert((sid, req_id), ResponseRoute::SemanticTokens { uri });
|
||||
Ok(job_id)
|
||||
}
|
||||
|
||||
/// Send `workspace/executeCommand`. No response route is
|
||||
/// registered: the command result is usually `null` and the real
|
||||
/// effect arrives as a server→client `workspace/applyEdit` (the
|
||||
|
|
@ -2480,6 +2515,15 @@ impl LspManager {
|
|||
.expect("inlay hint store mutex poisoned");
|
||||
guard.set(key, resp);
|
||||
}
|
||||
ResponseRoute::SemanticTokens { uri } => {
|
||||
let resp = crate::semantic_tokens::SemanticTokensResponse::from_lsp_value(result);
|
||||
let key = crate::semantic_tokens::SemanticTokenKey::new(server_key, uri.clone());
|
||||
let mut guard = self
|
||||
.semantic_token_store
|
||||
.lock()
|
||||
.expect("semantic token store mutex poisoned");
|
||||
guard.set(key, resp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2931,6 +2975,30 @@ fn default_capabilities() -> Value {
|
|||
// `workspace/inlayHint/refresh`; on-demand re-query is
|
||||
// the v1 model.
|
||||
"inlayHint": { "dynamicRegistration": false },
|
||||
// T M4.5 semantic tokens. v1 requests `full` only (no
|
||||
// `/range`, no `/full/delta`). `formats: ["relative"]` is
|
||||
// the only encoding LSP defines; the tokenTypes/
|
||||
// tokenModifiers lists are the LSP-standard legend the
|
||||
// client understands — the server intersects its legend
|
||||
// with these and reports the agreed legend back via
|
||||
// `semanticTokensProvider.legend`.
|
||||
"semanticTokens": {
|
||||
"dynamicRegistration": false,
|
||||
"requests": { "full": true, "range": false },
|
||||
"formats": ["relative"],
|
||||
"tokenTypes": [
|
||||
"namespace", "type", "class", "enum", "interface",
|
||||
"struct", "typeParameter", "parameter", "variable",
|
||||
"property", "enumMember", "event", "function",
|
||||
"method", "macro", "keyword", "modifier", "comment",
|
||||
"string", "number", "regexp", "operator", "decorator"
|
||||
],
|
||||
"tokenModifiers": [
|
||||
"declaration", "definition", "readonly", "static",
|
||||
"deprecated", "abstract", "async", "modification",
|
||||
"documentation", "defaultLibrary"
|
||||
],
|
||||
},
|
||||
"publishDiagnostics": { "relatedInformation": true },
|
||||
},
|
||||
})
|
||||
|
|
|
|||
|
|
@ -7553,6 +7553,20 @@ pub fn install_lsp(lua: &Lua, manager: &SharedLspManager) -> mlua::Result<()> {
|
|||
)?;
|
||||
}
|
||||
|
||||
{
|
||||
let m = manager.clone();
|
||||
lsp_mod.set(
|
||||
"_request_semantic_tokens_raw",
|
||||
lua.create_function(move |_, (id, uri): (LspServerIdLua, String)| {
|
||||
let job_id = m
|
||||
.borrow_mut()
|
||||
.request_semantic_tokens(id.0, uri)
|
||||
.map_err(mlua::Error::external)?;
|
||||
Ok(job_id)
|
||||
})?,
|
||||
)?;
|
||||
}
|
||||
|
||||
{
|
||||
let m = manager.clone();
|
||||
lsp_mod.set(
|
||||
|
|
@ -7829,6 +7843,7 @@ pub fn make_lsp_manager(
|
|||
install_rename(lua, &manager)?;
|
||||
install_code_action(lua, &manager)?;
|
||||
install_inlay_hint(lua, &manager)?;
|
||||
install_semantic_tokens(lua, &manager)?;
|
||||
Ok(manager)
|
||||
}
|
||||
|
||||
|
|
@ -8632,6 +8647,9 @@ use crate::hover::{Hover, HoverKey};
|
|||
use crate::inlay_hint::{InlayHint as LspInlayHint, InlayHintKey};
|
||||
use crate::locations::{LocationKind, LocationsKey};
|
||||
use crate::rename::{RenameKey, WorkspaceEditResponse, WorkspaceOp};
|
||||
use crate::semantic_tokens::{
|
||||
SemanticToken as LspSemanticToken, SemanticTokenKey, SemanticTokensLegend,
|
||||
};
|
||||
use crate::signature::{Signature, SignatureHelp, SignatureKey, SignatureParameter};
|
||||
use crate::symbol::{Symbol as LspSymbol, SymbolKey};
|
||||
|
||||
|
|
@ -9574,6 +9592,101 @@ pub fn install_inlay_hint(lua: &Lua, manager: &SharedLspManager) -> mlua::Result
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn semantic_token_to_lua(lua: &Lua, t: &LspSemanticToken) -> mlua::Result<Table> {
|
||||
let out = lua.create_table_with_capacity(0, 5)?;
|
||||
out.set("line", t.line)?;
|
||||
out.set("start", t.start)?;
|
||||
out.set("length", t.length)?;
|
||||
out.set("token_type", t.token_type)?;
|
||||
out.set("token_modifiers", t.token_modifiers)?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Install `pmacs.semantic_tokens.*` (T M4.5). `tokens(sid, uri)`
|
||||
/// returns the decoded absolute tokens `{ { line, start, length,
|
||||
/// token_type, token_modifiers }, … }`; `legend(sid)` returns
|
||||
/// `{ token_types = {…}, token_modifiers = {…} }` from the server's
|
||||
/// advertised `semanticTokensProvider.legend` (or nil), so callers
|
||||
/// resolve the `token_type` index / `token_modifiers` bitset;
|
||||
/// `clear(sid, uri)` drops the entry. No renderer here — wiring LSP
|
||||
/// tokens into styling is a separate rendering milestone; this is the
|
||||
/// data surface that work reads.
|
||||
pub fn install_semantic_tokens(lua: &Lua, manager: &SharedLspManager) -> mlua::Result<()> {
|
||||
let pmacs: Table = lua.globals().get("pmacs")?;
|
||||
let m = lua.create_table()?;
|
||||
|
||||
{
|
||||
let mgr = manager.clone();
|
||||
m.set(
|
||||
"tokens",
|
||||
lua.create_function(move |lua, (id, uri): (LspServerIdLua, String)| {
|
||||
let store_handle = mgr.borrow().semantic_token_store();
|
||||
let guard = store_handle
|
||||
.lock()
|
||||
.expect("semantic token store mutex poisoned");
|
||||
let key = SemanticTokenKey::new(id.0.raw().to_string(), uri);
|
||||
let out = lua.create_table()?;
|
||||
if let Some(r) = guard.get(&key) {
|
||||
for (i, t) in r.tokens.iter().enumerate() {
|
||||
out.set(i + 1, semantic_token_to_lua(lua, t)?)?;
|
||||
}
|
||||
}
|
||||
Ok(Value::Table(out))
|
||||
})?,
|
||||
)?;
|
||||
}
|
||||
|
||||
{
|
||||
let mgr = manager.clone();
|
||||
m.set(
|
||||
"legend",
|
||||
lua.create_function(move |lua, id: LspServerIdLua| {
|
||||
// Parse out an owned legend inside the borrow, then
|
||||
// build the Lua table once the manager borrow is
|
||||
// dropped.
|
||||
let parsed = {
|
||||
let guard = mgr.borrow();
|
||||
guard
|
||||
.capabilities(id.0)
|
||||
.and_then(SemanticTokensLegend::from_capabilities)
|
||||
};
|
||||
let Some(legend) = parsed else {
|
||||
return Ok(Value::Nil);
|
||||
};
|
||||
let to_arr = |names: &[String]| -> mlua::Result<Table> {
|
||||
let t = lua.create_table_with_capacity(names.len(), 0)?;
|
||||
for (i, n) in names.iter().enumerate() {
|
||||
t.set(i + 1, n.as_str())?;
|
||||
}
|
||||
Ok(t)
|
||||
};
|
||||
let out = lua.create_table_with_capacity(0, 2)?;
|
||||
out.set("token_types", to_arr(&legend.token_types)?)?;
|
||||
out.set("token_modifiers", to_arr(&legend.token_modifiers)?)?;
|
||||
Ok(Value::Table(out))
|
||||
})?,
|
||||
)?;
|
||||
}
|
||||
|
||||
{
|
||||
let mgr = manager.clone();
|
||||
m.set(
|
||||
"clear",
|
||||
lua.create_function(move |_, (id, uri): (LspServerIdLua, String)| {
|
||||
let store_handle = mgr.borrow().semantic_token_store();
|
||||
let mut guard = store_handle
|
||||
.lock()
|
||||
.expect("semantic token store mutex poisoned");
|
||||
guard.clear(&SemanticTokenKey::new(id.0.raw().to_string(), uri));
|
||||
Ok(())
|
||||
})?,
|
||||
)?;
|
||||
}
|
||||
|
||||
pmacs.set("semantic_tokens", m)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// pmacs.project: project model (T M4.9)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -0,0 +1,336 @@
|
|||
// semantic_tokens.rs --- T M4.5 LSP semantic tokens.
|
||||
|
||||
//! `textDocument/semanticTokens/full` response state.
|
||||
//!
|
||||
//! Semantic tokens are the server's type-aware classification of every
|
||||
//! token in a document (this identifier is a *mutable* `variable`,
|
||||
//! that one a `function.defaultLibrary`, …). The wire format is a flat
|
||||
//! `data: number[]` whose every 5 ints describe one token **relative**
|
||||
//! to the previous one:
|
||||
//!
|
||||
//! ```text
|
||||
//! [deltaLine, deltaStartChar, length, tokenType, tokenModifiers]
|
||||
//! ```
|
||||
//!
|
||||
//! This module decodes that into a flat list of *absolute*
|
||||
//! [`SemanticToken`]s, and parses the server's
|
||||
//! `semanticTokensProvider.legend` so callers can resolve the
|
||||
//! `token_type` / `token_modifiers` indices to names.
|
||||
//!
|
||||
//! Scope: this is the **LSP data layer only**. It is deliberately
|
||||
//! independent of the M11 semantic-render protocol
|
||||
//! ([`crate::semantic_render`] / [`crate::semantic_client`]), which
|
||||
//! projects tree-sitter highlighting into the frontend wire families.
|
||||
//! Wiring LSP tokens into rendering (a second styling authority,
|
||||
//! priority vs. tree-sitter) is a separate rendering milestone; like
|
||||
//! the other LSP features, nothing here paints — Lua reads the store.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
/// One decoded, **absolute**-positioned semantic token.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub struct SemanticToken {
|
||||
/// Zero-based line.
|
||||
pub line: u32,
|
||||
/// Zero-based start column (UTF-16 code units, per LSP).
|
||||
pub start: u32,
|
||||
/// Token length in UTF-16 code units.
|
||||
pub length: u32,
|
||||
/// Index into the legend's `token_types`.
|
||||
pub token_type: u32,
|
||||
/// Bitset; bit `i` set ⇒ legend's `token_modifiers[i]` applies.
|
||||
pub token_modifiers: u32,
|
||||
}
|
||||
|
||||
/// Parsed `textDocument/semanticTokens` response.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct SemanticTokensResponse {
|
||||
/// Tokens in document order (decoded from the relative encoding).
|
||||
pub tokens: Vec<SemanticToken>,
|
||||
/// Opaque server cursor for a future delta request (unused by the
|
||||
/// v1 full-only path; surfaced for completeness).
|
||||
pub result_id: Option<String>,
|
||||
}
|
||||
|
||||
impl SemanticTokensResponse {
|
||||
/// Parse `SemanticTokens | null`.
|
||||
///
|
||||
/// `data` must be a flat array whose length is a multiple of 5; a
|
||||
/// trailing partial group (malformed server) is ignored rather
|
||||
/// than panicking. A `null` / shapeless result yields no tokens.
|
||||
#[must_use]
|
||||
pub fn from_lsp_value(v: &Value) -> Self {
|
||||
let result_id = v.get("resultId").and_then(Value::as_str).map(str::to_owned);
|
||||
let Some(data) = v.get("data").and_then(Value::as_array) else {
|
||||
return Self {
|
||||
tokens: Vec::new(),
|
||||
result_id,
|
||||
};
|
||||
};
|
||||
let ints: Vec<u32> = data
|
||||
.iter()
|
||||
.map(|n| n.as_u64().unwrap_or(0) as u32)
|
||||
.collect();
|
||||
let mut tokens = Vec::with_capacity(ints.len() / 5);
|
||||
let mut line = 0u32;
|
||||
let mut start = 0u32;
|
||||
for chunk in ints.chunks_exact(5) {
|
||||
let (d_line, d_start, length, tt, tm) =
|
||||
(chunk[0], chunk[1], chunk[2], chunk[3], chunk[4]);
|
||||
// deltaLine is relative to the previous token's line;
|
||||
// deltaStartChar is relative to the previous token's
|
||||
// start *iff* on the same line, else absolute from col 0.
|
||||
line += d_line;
|
||||
start = if d_line == 0 {
|
||||
start + d_start
|
||||
} else {
|
||||
d_start
|
||||
};
|
||||
tokens.push(SemanticToken {
|
||||
line,
|
||||
start,
|
||||
length,
|
||||
token_type: tt,
|
||||
token_modifiers: tm,
|
||||
});
|
||||
}
|
||||
Self { tokens, result_id }
|
||||
}
|
||||
|
||||
/// True iff the server returned no tokens.
|
||||
#[must_use]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.tokens.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// The server's `semanticTokensProvider.legend`: the ordered name
|
||||
/// tables the `token_type` index and `token_modifiers` bits map into.
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq)]
|
||||
pub struct SemanticTokensLegend {
|
||||
/// `token_type` index → name.
|
||||
pub token_types: Vec<String>,
|
||||
/// Modifier bit position → name.
|
||||
pub token_modifiers: Vec<String>,
|
||||
}
|
||||
|
||||
impl SemanticTokensLegend {
|
||||
/// Pull the legend out of an `initialize` `ServerCapabilities`
|
||||
/// JSON value. Returns `None` if the server advertises no
|
||||
/// `semanticTokensProvider` (or it carries no `legend`).
|
||||
#[must_use]
|
||||
pub fn from_capabilities(caps: &Value) -> Option<Self> {
|
||||
let legend = caps.get("semanticTokensProvider")?.get("legend")?;
|
||||
let pull = |key: &str| -> Vec<String> {
|
||||
legend
|
||||
.get(key)
|
||||
.and_then(Value::as_array)
|
||||
.map(|a| {
|
||||
a.iter()
|
||||
.filter_map(|v| v.as_str().map(str::to_owned))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
};
|
||||
Some(Self {
|
||||
token_types: pull("tokenTypes"),
|
||||
token_modifiers: pull("tokenModifiers"),
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolve a `token_type` index to its legend name.
|
||||
#[must_use]
|
||||
pub fn type_name(&self, index: u32) -> Option<&str> {
|
||||
self.token_types.get(index as usize).map(String::as_str)
|
||||
}
|
||||
|
||||
/// Resolve a `token_modifiers` bitset to the set legend names,
|
||||
/// low bit first.
|
||||
#[must_use]
|
||||
pub fn modifier_names(&self, bits: u32) -> Vec<&str> {
|
||||
(0..self.token_modifiers.len())
|
||||
.filter(|i| bits & (1 << i) != 0)
|
||||
.map(|i| self.token_modifiers[i].as_str())
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-server, per-uri semantic-token state.
|
||||
#[derive(Default)]
|
||||
pub struct SemanticTokenStore {
|
||||
by_key: HashMap<SemanticTokenKey, SemanticTokensResponse>,
|
||||
}
|
||||
|
||||
/// Key into [`SemanticTokenStore`].
|
||||
#[derive(Clone, Eq, PartialEq, Hash, Debug)]
|
||||
pub struct SemanticTokenKey {
|
||||
/// Decimal LSP server id.
|
||||
pub server: String,
|
||||
/// Document URI the request was made on.
|
||||
pub uri: String,
|
||||
}
|
||||
|
||||
impl SemanticTokenKey {
|
||||
/// Construct a key.
|
||||
#[must_use]
|
||||
pub fn new(server: impl Into<String>, uri: impl Into<String>) -> Self {
|
||||
Self {
|
||||
server: server.into(),
|
||||
uri: uri.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SemanticTokenStore {
|
||||
/// Empty store.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Replace the response at `key`.
|
||||
pub fn set(&mut self, key: SemanticTokenKey, response: SemanticTokensResponse) {
|
||||
self.by_key.insert(key, response);
|
||||
}
|
||||
|
||||
/// Drop the entry at `key`.
|
||||
pub fn clear(&mut self, key: &SemanticTokenKey) {
|
||||
self.by_key.remove(key);
|
||||
}
|
||||
|
||||
/// Look up the entry at `key`.
|
||||
#[must_use]
|
||||
pub fn get(&self, key: &SemanticTokenKey) -> Option<&SemanticTokensResponse> {
|
||||
self.by_key.get(key)
|
||||
}
|
||||
}
|
||||
|
||||
/// Cheaply-cloneable shared handle.
|
||||
pub type SharedSemanticTokenStore = Arc<Mutex<SemanticTokenStore>>;
|
||||
|
||||
/// Build a fresh shared store.
|
||||
#[must_use]
|
||||
pub fn make_shared_store() -> SharedSemanticTokenStore {
|
||||
Arc::new(Mutex::new(SemanticTokenStore::new()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn decodes_relative_encoding_across_lines() {
|
||||
// Three tokens:
|
||||
// - line 0, char 0, len 3, type 1, mods 0
|
||||
// - same line, +5 chars → char 5, len 2, type 2, mods 0
|
||||
// - +2 lines, char 4 (absolute, deltaLine!=0), len 6, type 0,
|
||||
// mods 0b101
|
||||
let v = json!({
|
||||
"resultId": "1",
|
||||
"data": [
|
||||
0, 0, 3, 1, 0,
|
||||
0, 5, 2, 2, 0,
|
||||
2, 4, 6, 0, 5
|
||||
]
|
||||
});
|
||||
let r = SemanticTokensResponse::from_lsp_value(&v);
|
||||
assert_eq!(r.result_id.as_deref(), Some("1"));
|
||||
assert_eq!(r.tokens.len(), 3);
|
||||
assert_eq!(
|
||||
r.tokens[0],
|
||||
SemanticToken {
|
||||
line: 0,
|
||||
start: 0,
|
||||
length: 3,
|
||||
token_type: 1,
|
||||
token_modifiers: 0
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
r.tokens[1],
|
||||
SemanticToken {
|
||||
line: 0,
|
||||
start: 5,
|
||||
length: 2,
|
||||
token_type: 2,
|
||||
token_modifiers: 0
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
r.tokens[2],
|
||||
SemanticToken {
|
||||
line: 2,
|
||||
start: 4,
|
||||
length: 6,
|
||||
token_type: 0,
|
||||
token_modifiers: 5
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trailing_partial_group_is_ignored() {
|
||||
let v = json!({ "data": [0, 0, 3, 1, 0, 9, 9] });
|
||||
let r = SemanticTokensResponse::from_lsp_value(&v);
|
||||
assert_eq!(r.tokens.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn null_response_is_empty() {
|
||||
let r = SemanticTokensResponse::from_lsp_value(&Value::Null);
|
||||
assert!(r.is_empty());
|
||||
assert!(r.result_id.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legend_parses_and_resolves() {
|
||||
let caps = json!({
|
||||
"semanticTokensProvider": {
|
||||
"legend": {
|
||||
"tokenTypes": ["namespace", "type", "function"],
|
||||
"tokenModifiers": ["declaration", "readonly", "static"]
|
||||
},
|
||||
"full": true
|
||||
}
|
||||
});
|
||||
let legend = SemanticTokensLegend::from_capabilities(&caps).unwrap();
|
||||
assert_eq!(legend.type_name(2), Some("function"));
|
||||
assert_eq!(legend.type_name(99), None);
|
||||
// bits 0b101 = declaration + static
|
||||
assert_eq!(legend.modifier_names(0b101), vec!["declaration", "static"]);
|
||||
assert_eq!(legend.modifier_names(0), Vec::<&str>::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_provider_yields_no_legend() {
|
||||
assert!(
|
||||
SemanticTokensLegend::from_capabilities(&json!({ "hoverProvider": true })).is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn store_set_get_clear() {
|
||||
let mut s = SemanticTokenStore::new();
|
||||
let key = SemanticTokenKey::new("1", "file:///a");
|
||||
s.set(
|
||||
key.clone(),
|
||||
SemanticTokensResponse {
|
||||
tokens: vec![SemanticToken {
|
||||
line: 0,
|
||||
start: 0,
|
||||
length: 1,
|
||||
token_type: 0,
|
||||
token_modifiers: 0,
|
||||
}],
|
||||
result_id: None,
|
||||
},
|
||||
);
|
||||
assert_eq!(s.get(&key).unwrap().tokens.len(), 1);
|
||||
s.clear(&key);
|
||||
assert!(s.get(&key).is_none());
|
||||
}
|
||||
}
|
||||
|
|
@ -3804,6 +3804,70 @@ fn m4_16_lua_surface_drives_inlay_hints() {
|
|||
assert!(pad1, "second hint requested paddingRight");
|
||||
}
|
||||
|
||||
/// T M4.5 — semantic tokens through the Lua surface. Drives
|
||||
/// `pmacs.lsp.request_semantic_tokens` against the fake and asserts
|
||||
/// the relative `data` encoding decoded to absolute tokens (incl. the
|
||||
/// multi-line delta where `deltaStartChar` becomes absolute), and
|
||||
/// that `pmacs.semantic_tokens.legend` exposes the server's legend so
|
||||
/// the `token_type` index resolves to a name.
|
||||
#[test]
|
||||
fn m4_17_lua_surface_drives_semantic_tokens() {
|
||||
let mut s = pmacs::editor::EditorState::new();
|
||||
spawn_lsp_and_init(&mut s, None);
|
||||
|
||||
let uri = "file:///tmp/m4_17_sem.rs";
|
||||
s.lua_host
|
||||
.lua()
|
||||
.load(format!(
|
||||
"pmacs.lsp.did_open(_G._lsp, '{uri}', 1, 'fn a() {{}}\\n\\nlet b = 1\\n')
|
||||
pmacs.lsp.request_semantic_tokens(_G._lsp, '{uri}')"
|
||||
))
|
||||
.exec()
|
||||
.expect("kick off semantic tokens request");
|
||||
|
||||
assert!(
|
||||
pump_lua_flag(
|
||||
&mut s,
|
||||
&format!("#pmacs.semantic_tokens.tokens(_G._lsp, '{uri}') > 0"),
|
||||
5,
|
||||
),
|
||||
"semantic tokens response did not land in the store"
|
||||
);
|
||||
|
||||
// data = [0,0,4,1,1, 0,5,3,2,0, 2,2,7,0,2]
|
||||
// t1: line 0 start 0 len 4 type 1 mods 1
|
||||
// t2: line 0 start 5 len 3 type 2 mods 0 (same-line delta)
|
||||
// t3: line 2 start 2 len 7 type 0 mods 2 (deltaLine!=0 ⇒
|
||||
// startChar absolute)
|
||||
let (count, t1, t2, t3, type1_name, type0_name): (
|
||||
usize,
|
||||
Vec<u32>,
|
||||
Vec<u32>,
|
||||
Vec<u32>,
|
||||
String,
|
||||
String,
|
||||
) = s
|
||||
.lua_host
|
||||
.lua()
|
||||
.load(format!(
|
||||
"local t = pmacs.semantic_tokens.tokens(_G._lsp, '{uri}')
|
||||
local lg = pmacs.semantic_tokens.legend(_G._lsp)
|
||||
local function tup(x) return {{ x.line, x.start, x.length,
|
||||
x.token_type, x.token_modifiers }} end
|
||||
return #t, tup(t[1]), tup(t[2]), tup(t[3]),
|
||||
lg.token_types[2], lg.token_types[1]"
|
||||
))
|
||||
.eval()
|
||||
.expect("read semantic tokens + legend back");
|
||||
assert_eq!(count, 3);
|
||||
assert_eq!(t1, vec![0, 0, 4, 1, 1]);
|
||||
assert_eq!(t2, vec![0, 5, 3, 2, 0]);
|
||||
assert_eq!(t3, vec![2, 2, 7, 0, 2]);
|
||||
// Legend resolves the type index (0-based) → name (1-based Lua).
|
||||
assert_eq!(type1_name, "function");
|
||||
assert_eq!(type0_name, "namespace");
|
||||
}
|
||||
|
||||
/// Default LSP bundle (`builtin/runtime/lsp.lua`) is wired in: the
|
||||
/// hooks are defined, the namespace tables exist, the user-facing
|
||||
/// commands are registered with the command registry, and the default
|
||||
|
|
|
|||
Loading…
Reference in New Issue