diff --git a/builtin/runtime/editops.lua b/builtin/runtime/editops.lua index 7d1f061..13fc1a6 100644 --- a/builtin/runtime/editops.lua +++ b/builtin/runtime/editops.lua @@ -110,28 +110,49 @@ local function is_cont_byte(b) return b >= 0x80 and b <= 0xBF end -local function cp_len(lead) - if lead < 0x80 then return 1 end - if lead >= 0xC2 and lead <= 0xDF then return 2 end - if lead >= 0xE0 and lead <= 0xEF then return 3 end - if lead >= 0xF0 and lead <= 0xF4 then return 4 end +-- Byte-length of the VALID UTF-8 scalar starting at index `i` of +-- `s`, or nil. Enforces the full second-byte constraint table — +-- overlong encodings (C0/C1, E0 80..9F, F0 80..8F), surrogates +-- (ED A0..BF), and > U+10FFFF (F4 90..BF, F5..FF) all fail, not +-- just bad lead/continuation ranges. Every consumer of "one +-- codepoint" routes through here so malformed bytes fail closed. +local function scalar_len(s, i) + local b1 = s:byte(i) + if not b1 then return nil end + if b1 < 0x80 then return 1 end + local b2 = s:byte(i + 1) + if not b2 or not is_cont_byte(b2) then return nil end + if b1 >= 0xC2 and b1 <= 0xDF then + return 2 + end + if b1 >= 0xE0 and b1 <= 0xEF then + if b1 == 0xE0 and b2 < 0xA0 then return nil end + if b1 == 0xED and b2 > 0x9F then return nil end + local b3 = s:byte(i + 2) + if not b3 or not is_cont_byte(b3) then return nil end + return 3 + end + if b1 >= 0xF0 and b1 <= 0xF4 then + if b1 == 0xF0 and b2 < 0x90 then return nil end + if b1 == 0xF4 and b2 > 0x8F then return nil end + local b3 = s:byte(i + 2) + if not b3 or not is_cont_byte(b3) then return nil end + local b4 = s:byte(i + 3) + if not b4 or not is_cont_byte(b4) then return nil end + return 4 + end return nil end local function single_codepoint(s) - if #s == 0 then return false end - local n = cp_len(s:byte(1)) - if not n or n ~= #s then return false end - for i = 2, #s do - if not is_cont_byte(s:byte(i)) then return false end - end - return true + return #s > 0 and scalar_len(s, 1) == #s end -- Start of the codepoint ending just before `pos`. Returns nil at -- BOB; nil, "malformed" when the preceding bytes are not one valid --- UTF-8 codepoint (fail closed — goto_byte guarantees no boundary --- alignment). +-- UTF-8 scalar (fail closed — goto_byte guarantees no boundary +-- alignment, and a length-consistent overlong/surrogate span must +-- not be treated as a character). local function prev_cp_start(buf, pos) if pos <= 0 then return nil end local q = pos - 1 @@ -142,9 +163,8 @@ local function prev_cp_start(buf, pos) q = q - 1 steps = steps + 1 end - local lead = buf:slice(q, q + 1):byte(1) - local n = cp_len(lead) - if not n or q + n ~= pos then return nil, "malformed" end + local span = buf:slice(q, pos) + if scalar_len(span, 1) ~= pos - q then return nil, "malformed" end return q end @@ -162,25 +182,29 @@ local function ascii_lower(s) end)) end --- First word byte upcased when it is a letter; every other letter --- downcased (single-span semantics, Q#EC4). +-- Per-word capitalize across the span (Emacs capitalize-region +-- parity, verified against Emacs 30.2: "hello WORLD" -> "Hello +-- World", "9abc" -> "9abc", "a9bc" -> "A9bc"): each word's first +-- byte is upcased when it is a letter, every other letter is +-- downcased. Words per the pack's ASCII class — `_` and digits are +-- word constituents here, so "foo_bar" gives "Foo_bar" where +-- Emacs's symbol-syntax `_` gives "Foo_Bar" (named deviation, +-- Q#EC4). local function ascii_capitalize(s) - local first = nil + local out = {} + local in_word = false for i = 1, #s do - if is_word_byte(s:byte(i)) then - first = i - break + local b = s:byte(i) + local w = is_word_byte(b) + if w and not in_word then + if b >= 97 and b <= 122 then b = b - 32 end + elseif w then + if b >= 65 and b <= 90 then b = b + 32 end end + in_word = w + out[#out + 1] = string.char(b) end - local lowered = ascii_lower(s) - if not first then return lowered end - local b = lowered:byte(first) - if b >= 97 and b <= 122 then - return lowered:sub(1, first - 1) - .. string.char(b - 32) - .. lowered:sub(first + 1) - end - return lowered + return table.concat(out) end -- ---- Q#EC2 shared discipline --------------------------------------- @@ -353,13 +377,17 @@ pmacs.command.define { or "transpose-chars: not enough characters") return end - local n2 = cp_len(at_b) - if not n2 or cursor + n2 > len then + -- Validate the WHOLE scalar at the cursor, trailing bytes + -- included — a valid lead followed by non-continuation bytes + -- must fail closed, not ride along as "one character". + local head = buf:slice(cursor, math.min(cursor + 4, len)) + local n2 = scalar_len(head, 1) + if not n2 then ed.set_status("transpose-chars: malformed UTF-8 at the cursor") return end local cp1 = buf:slice(s1, cursor) - local cp2 = buf:slice(cursor, cursor + n2) + local cp2 = head:sub(1, n2) guarded_replace("transpose-chars", buf, s1, cursor + n2, cp2 .. cp1, cursor + n2) end @@ -824,13 +852,26 @@ end -- loads before saveplace.lua (the loader ordering contract, Q#EC9). pmacs.hook.add("buffer.before-save", function() -- Outer pcall: a raised error in a short-circuit hook vetoes the - -- save; trim failure must report via status, never block a save. - pcall(function() + -- save; trim failure must never block a save. But it must not be + -- silently discarded either — an unexpected error is a bug, and + -- swallowing it hides the bug. Report on both channels the + -- autosave sweep uses: the status line (visible when the save + -- itself fails or is vetoed; a successful save overwrites it with + -- "saved ...") and the *errors* buffer via pmacs.error (durable + -- either way). Both reports are pcall'd — a broken reporting + -- channel must not resurrect the veto. + local ok, err = pcall(function() if trim_enabled then trim_active("delete-trailing-whitespace (on save)") end end) - -- nil return: never a veto. + if not ok then + local msg = "delete-trailing-whitespace (on save) failed: " + .. tostring(err) + pcall(ed.set_status, msg) + if pmacs.error then pcall(pmacs.error, msg) end + end + -- nil return either way: never a veto. end) -- ---- bindings (Q#EC1: all verified free across builtin bind sites) -- diff --git a/docs/editing-conveniences-framing.md b/docs/editing-conveniences-framing.md index cc91aa9..a39e2d1 100644 --- a/docs/editing-conveniences-framing.md +++ b/docs/editing-conveniences-framing.md @@ -58,6 +58,17 @@ Revision 5 (post-approval, R3's optional hardening adopted): zap fails closed — no kill, chain broken — when public Lua consumed the marker while the prompt was open. +Revision 6 (PR #111 round 1): codepoint recognition is full UTF-8 +scalar validation (second-byte constraint table; transpose +validates the cursor scalar trailing-bytes-included, and a +length-consistent overlong/surrogate span behind the cursor fails +closed); capitalize is per-word across the span (Emacs +capitalize-region parity, empirically verified, with the pack's +`_`-is-a-word-constituent deviation named); trim-on-save reports +unexpected errors on the status line AND the `*errors*` buffer via +`pmacs.error` instead of silently discarding them, still never +vetoing. + ## Ground truth (as of `7e127ab`) - **The taken-chord registry is wider than `builtin/keymaps/ @@ -337,7 +348,16 @@ range gsub with a byte map — NOT `string.upper/lower` and NOT backend (ground truth); this also keeps Lua 5.4 and LuaJIT identical. Non-ASCII bytes pass through untouched — pinned in acceptance (an `é` in the span is byte-identical after the op). -Capitalize = first word char upcased, rest of span downcased. + +Capitalize is PER-WORD across the span — Emacs capitalize-region +parity (PR #111 R1 finding 2; empirical, Emacs 30.2 `-Q --batch`: +`"hello WORLD"` → `"Hello World"`, `"9abc a9bc"` → `"9abc A9bc"`): +each word's first byte is upcased when it is a letter, every other +letter downcased; a digit-led word keeps its letters lowercase. One +named deviation remains: `_` is a word constituent in this pack's +class (the `word_at_cursor` precedent) but symbol-syntax in Emacs, +so `foo_bar` capitalizes as `Foo_bar` here versus Emacs's +`Foo_Bar`. ### Q#EC5 — Transpose: codepoint-aware chars, Emacs-verified word boundaries @@ -345,13 +365,24 @@ Capitalize = first word char upcased, rest of span downcased. cursor, cursor ends after both (Emacs drag-forward). At EOL (next char is `\n` or EOF) with ≥2 preceding codepoints: swap the two before the cursor (Emacs special case). Fewer than two reachable -codepoints → status, no edit. Codepoint boundaries by UTF-8 -lead-byte scan on a small slice around the cursor; **a continuation -byte at the cursor position fails closed** (status, no edit) — -`goto_byte` does not guarantee boundary alignment, and "never split -a codepoint" must hold against a misaligned start, not assume it -away. Newlines participate (transpose across lines works). One -replace spanning exactly the two codepoints. +codepoints → status, no edit. Codepoint recognition is FULL scalar +validation, not lead/continuation range checks (PR #111 R1 finding +1): a shared validator enforces the UTF-8 second-byte constraint +table — overlongs (`C0`/`C1`, `E0 80..9F`, `F0 80..8F`), surrogates +(`ED A0..BF`), and beyond-`U+10FFFF` (`F4 90..BF`, `F5..FF`) all +fail — and the scalar AT the cursor is validated trailing bytes +included (a valid lead followed by non-continuation bytes must not +ride along as "one character"). Failures fail closed: a +continuation byte at the cursor, a malformed scalar at the cursor, +and a length-consistent-but-invalid span behind it each report and +leave the buffer untouched — `goto_byte` does not guarantee +boundary alignment, and buffers are byte-clean, so malformed input +is reachable. (Zap's single-codepoint input check uses the same +validator as defense-in-depth; minibuffer contents arrive as +Rust-side UTF-8 — `set_contents` is `String`-typed — so the +buffer-facing checks are the load-bearing ones.) Newlines +participate (transpose across lines works). One replace spanning +exactly the two codepoints. `edit.transpose-words` (M-t), specified against the Emacs 30.2 table in Ground truth: @@ -581,9 +612,15 @@ saveplace's cursor-record within the before-save fan-out (recorded places see post-trim text). The ENTIRE callback body is wrapped in pcall with a `nil` return on both paths (the saveplace pattern): returning `nil` never vetoes, but a raised error in a -short-circuit hook vetoes immediately (`src/hook.rs:299`) — trim -failure of any kind must report via status and never block the -save. +short-circuit hook vetoes immediately (`src/hook.rs:299`). An +unexpected error caught by that pcall is NOT silently discarded +(PR #111 R1 finding 3) — it reports on both channels the autosave +sweep uses: the status line (visible when the save fails or is +vetoed; a successful save overwrites it with "saved ...") and the +`*errors*` buffer via `pmacs.error` (durable either way; the +async/mcp/syntax/autosave convention). Both reports are +themselves pcall'd so a broken reporting channel cannot resurrect +the veto. ### Q#EC10 — Cut from the pack: recenter @@ -658,12 +695,21 @@ harness the kill-ring suite already uses. separators skips forward to the next word; no word forward → no edit; `é` in the span is byte-identical while ASCII neighbors flip — and stays byte-identical regardless of process locale - (explicit-range pin). + (explicit-range pin); capitalize: region `"hello WORLD"` → + `"Hello World"` (per-word, the Emacs parity row), `"9abc a9bc"` → + `"9abc A9bc"` (digit-led word keeps letters lowercase), and + `"foo_bar baz"` → `"Foo_bar Baz"` (the named `_` deviation, + pinned). - Transpose-chars: mid-line swap + cursor advance; EOL two-before swap; BOB/single-char no-op; multi-byte: swapping `é` and `x` yields intact UTF-8 both orders; across-newline swap; **cursor parked on a continuation byte → status, no edit** (fail-closed - pin). Undo restores the original in ONE step (grain pin). + pin); **malformed-scalar pins**: a valid lead with a + non-continuation trailing byte at the cursor (`a\xC3xb`), an + overlong span behind the cursor (`\xE0\x80\x80b`), and a + beyond-`U+10FFFF` span behind it (`\xF4\x90\x80\x80b`) each → + status, buffer byte-identical. Undo restores the original in ONE + step (grain pin). - Transpose-words: the full nine-position Emacs table from Ground truth, byte-for-byte including final cursor positions for the seven mutating rows; the two no-successor rows assert NO edit and @@ -751,7 +797,11 @@ harness the kill-ring suite already uses. (default) → save writes bytes untouched; **veto-immunity pin**: a rejecting intercept during on-save trim → the save still proceeds with a status report; another before-save callback's - veto still vetoes (trim's `nil` return masks nothing). + veto still vetoes (trim's `nil` return masks nothing); + **unexpected-error pin**: an error raised inside the on-save trim + (beyond the per-edit pcalls) → the save still proceeds AND the + failure lands in the `pmacs.error` log (stubbed, the m9_6 + pattern) — never silently discarded. No CRDT-specific suite: every editops edit is a daemon-peer edit on the dispatch or minibuffer-accept path with no optimistic-classifier diff --git a/tests/editops_acceptance.rs b/tests/editops_acceptance.rs index 5452495..0a30f95 100644 --- a/tests/editops_acceptance.rs +++ b/tests/editops_acceptance.rs @@ -306,11 +306,29 @@ fn capitalize_word_and_region() { alt(&mut s, 'c'); assert_eq!( buffer_text(&s), - "Hello world", - "single-span capitalize: first word char up, rest down" + "Hello World", + "per-word capitalize across the region (Emacs capitalize-region)" ); } +#[test] +fn capitalize_is_per_word_with_the_packs_word_class() { + // Emacs 30.2 parity rows: a digit-led word keeps its letters + // lowercase; a letter after a digit is not a word start. + let mut s = editor_with("9abc a9bc"); + exec(&s, "pmacs.editor.begin_selection(0)"); + exec(&s, "pmacs.editor.goto_byte(9)"); + alt(&mut s, 'c'); + assert_eq!(buffer_text(&s), "9abc A9bc"); + // The named deviation: `_` is a word constituent in this pack's + // ASCII class (Emacs's symbol-syntax `_` would give "Foo_Bar"). + let mut s = editor_with("foo_bar baz"); + exec(&s, "pmacs.editor.begin_selection(0)"); + exec(&s, "pmacs.editor.goto_byte(11)"); + alt(&mut s, 'c'); + assert_eq!(buffer_text(&s), "Foo_bar Baz"); +} + #[test] fn case_ops_report_when_no_word_follows() { let mut s = editor_with("foo "); @@ -406,6 +424,55 @@ fn transpose_chars_fails_closed_on_a_continuation_byte() { assert!(status(&s).contains("multi-byte")); } +/// Seed raw (possibly malformed) bytes via Lua \x escapes and compare +/// byte-identically Lua-side (`from_utf8_lossy` would mask differences). +fn seed_raw(s: &EditorState, lua_bytes: &str) { + exec( + s, + &format!("local b = pmacs.window.buffer(); b:replace(0, b:len(), \"{lua_bytes}\")"), + ); + exec(s, "pmacs.editor.goto_byte(0)"); +} + +fn raw_equals(s: &EditorState, lua_bytes: &str) -> bool { + eval( + s, + &format!("local b = pmacs.window.buffer(); return b:slice(0, b:len()) == \"{lua_bytes}\""), + ) +} + +#[test] +fn transpose_chars_fails_closed_on_invalid_trailing_bytes() { + // A valid lead (C3) followed by a non-continuation byte: the + // scalar at the cursor must be validated whole, not length-only. + let mut s = editor_with(""); + seed_raw(&s, "a\\xC3xb"); + exec(&s, "pmacs.editor.goto_byte(1)"); + ctrl(&mut s, 't'); + assert!(raw_equals(&s, "a\\xC3xb"), "no edit on a malformed scalar"); + assert!(status(&s).contains("malformed UTF-8 at the cursor")); +} + +#[test] +fn transpose_chars_fails_closed_on_invalid_scalars_behind_the_cursor() { + // Length-consistent but scalar-invalid spans behind the cursor: + // an overlong three-byte encoding and a beyond-U+10FFFF four-byte + // encoding. Both scan back cleanly (lead + continuations) and + // must still fail closed. + let cases = [("\\xE0\\x80\\x80b", 3i64), ("\\xF4\\x90\\x80\\x80b", 4)]; + for (bytes, cursor_pos) in cases { + let mut s = editor_with(""); + seed_raw(&s, bytes); + exec(&s, &format!("pmacs.editor.goto_byte({cursor_pos})")); + ctrl(&mut s, 't'); + assert!(raw_equals(&s, bytes), "no edit for {bytes}"); + assert!( + status(&s).contains("malformed UTF-8 before the cursor"), + "fail-closed status for {bytes}" + ); + } +} + // --------------------------------------------------------------------------- // Transpose-words: the nine-position Emacs 30.2 table (Q#EC5) // --------------------------------------------------------------------------- @@ -1175,6 +1242,53 @@ fn trim_on_save_failure_never_vetoes_the_save() { let _ = std::fs::remove_file(&path); } +#[test] +fn trim_on_save_unexpected_error_reports_and_still_saves() { + let path = temp_path("unexpected.txt"); + std::fs::write(&path, "x \n").unwrap(); + let mut s = EditorState::new(); + exec(&s, "pmacs.editops.trim_on_save(true)"); + // Capture the pmacs.error log (the m9_6 stub pattern — the + // `if pmacs.error` branch is a no-op without it). + exec( + &s, + r" + PMACS_ERROR_LOG = {} + pmacs.error = function(msg) + PMACS_ERROR_LOG[#PMACS_ERROR_LOG + 1] = msg + end + ", + ); + exec( + &s, + &format!( + "pmacs.buffer.from_file({})", + lua_str(path.to_str().unwrap()) + ), + ); + // Force an error INSIDE trim, past the per-edit pcalls: trim's + // context snapshot reads pmacs.window.current(), which nothing + // else on the save path touches. + exec( + &s, + "TRIM_ORIG_WC = pmacs.window.current; pmacs.window.current = function() error('boom') end", + ); + ctrl(&mut s, 'x'); + ctrl(&mut s, 's'); + exec(&s, "pmacs.window.current = TRIM_ORIG_WC"); + assert_eq!( + std::fs::read_to_string(&path).unwrap(), + "x \n", + "the save proceeded (trim aborted before any edit)" + ); + let logged: String = eval(&s, "return PMACS_ERROR_LOG[1] or ''"); + assert!( + logged.contains("delete-trailing-whitespace (on save) failed:"), + "the unexpected error reached the pmacs.error log, got: {logged}" + ); + let _ = std::fs::remove_file(&path); +} + #[test] fn another_callbacks_veto_is_not_masked_by_trim() { let path = temp_path("veto.txt");