fix(edit): PR #111 round 1 — scalar-valid UTF-8, per-word capitalize, trim error reporting

Finding 1: codepoint recognition is now full UTF-8 scalar validation
(shared second-byte constraint table: overlongs, surrogates, and
beyond-U+10FFFF all fail), and transpose validates the scalar AT the
cursor trailing-bytes-included — a valid lead with non-continuation
trailing bytes fails closed, as does a length-consistent overlong or
out-of-range span behind the cursor. Zap's single-codepoint check
uses the same validator as defense-in-depth (minibuffer contents
arrive as Rust-side UTF-8; the buffer-facing checks are the
load-bearing ones).

Finding 2: capitalize is per-word across the span — Emacs
capitalize-region parity, verified against Emacs 30.2 ("hello WORLD"
-> "Hello World", "9abc a9bc" -> "9abc A9bc"); the one remaining
deviation is named and pinned: `_` is a word constituent in this
pack's ASCII class, so "foo_bar" -> "Foo_bar" versus Emacs's
"Foo_Bar".

Finding 3: an unexpected error caught by the trim-on-save outer
pcall is no longer discarded — it reports on the status line AND the
*errors* buffer via pmacs.error (the autosave sweep convention),
both pcall'd, still never vetoing the save.

All three fixes bite-verified: the five new/updated acceptance cases
fail against the pre-fix editops.lua (72 total now). Framing at
revision 6.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018vF4gQVozBWi38y1SJiGfQ

builtin/runtime/editops.lua

@@ -110,28 +110,49 @@ local function is_cont_byte(b)
   return b >= 0x80 and b <= 0xBF
 end
 
-local function cp_len(lead)
+-- Byte-length of the VALID UTF-8 scalar starting at index `i` of
-  if lead < 0x80 then return 1 end
+-- `s`, or nil. Enforces the full second-byte constraint table —
-  if lead >= 0xC2 and lead <= 0xDF then return 2 end
+-- overlong encodings (C0/C1, E0 80..9F, F0 80..8F), surrogates
-  if lead >= 0xE0 and lead <= 0xEF then return 3 end
+-- (ED A0..BF), and > U+10FFFF (F4 90..BF, F5..FF) all fail, not
-  if lead >= 0xF0 and lead <= 0xF4 then return 4 end
+-- just bad lead/continuation ranges. Every consumer of "one
+-- codepoint" routes through here so malformed bytes fail closed.
+local function scalar_len(s, i)
+  local b1 = s:byte(i)
+  if not b1 then return nil end
+  if b1 < 0x80 then return 1 end
+  local b2 = s:byte(i + 1)
+  if not b2 or not is_cont_byte(b2) then return nil end
+  if b1 >= 0xC2 and b1 <= 0xDF then
+    return 2
+  end
+  if b1 >= 0xE0 and b1 <= 0xEF then
+    if b1 == 0xE0 and b2 < 0xA0 then return nil end
+    if b1 == 0xED and b2 > 0x9F then return nil end
+    local b3 = s:byte(i + 2)
+    if not b3 or not is_cont_byte(b3) then return nil end
+    return 3
+  end
+  if b1 >= 0xF0 and b1 <= 0xF4 then
+    if b1 == 0xF0 and b2 < 0x90 then return nil end
+    if b1 == 0xF4 and b2 > 0x8F then return nil end
+    local b3 = s:byte(i + 2)
+    if not b3 or not is_cont_byte(b3) then return nil end
+    local b4 = s:byte(i + 3)
+    if not b4 or not is_cont_byte(b4) then return nil end
+    return 4
+  end
   return nil
 end
 
 local function single_codepoint(s)
-  if #s == 0 then return false end
+  return #s > 0 and scalar_len(s, 1) == #s
-  local n = cp_len(s:byte(1))
-  if not n or n ~= #s then return false end
-  for i = 2, #s do
-    if not is_cont_byte(s:byte(i)) then return false end
-  end
-  return true
 end
 
 -- Start of the codepoint ending just before `pos`. Returns nil at
 -- BOB; nil, "malformed" when the preceding bytes are not one valid
--- UTF-8 codepoint (fail closed — goto_byte guarantees no boundary
+-- UTF-8 scalar (fail closed — goto_byte guarantees no boundary
--- alignment).
+-- alignment, and a length-consistent overlong/surrogate span must
+-- not be treated as a character).
 local function prev_cp_start(buf, pos)
   if pos <= 0 then return nil end
   local q = pos - 1
@@ -142,9 +163,8 @@ local function prev_cp_start(buf, pos)
     q = q - 1
     steps = steps + 1
   end
-  local lead = buf:slice(q, q + 1):byte(1)
+  local span = buf:slice(q, pos)
-  local n = cp_len(lead)
+  if scalar_len(span, 1) ~= pos - q then return nil, "malformed" end
-  if not n or q + n ~= pos then return nil, "malformed" end
   return q
 end
 
@@ -162,25 +182,29 @@ local function ascii_lower(s)
   end))
 end
 
--- First word byte upcased when it is a letter; every other letter
+-- Per-word capitalize across the span (Emacs capitalize-region
--- downcased (single-span semantics, Q#EC4).
+-- parity, verified against Emacs 30.2: "hello WORLD" -> "Hello
+-- World", "9abc" -> "9abc", "a9bc" -> "A9bc"): each word's first
+-- byte is upcased when it is a letter, every other letter is
+-- downcased. Words per the pack's ASCII class — `_` and digits are
+-- word constituents here, so "foo_bar" gives "Foo_bar" where
+-- Emacs's symbol-syntax `_` gives "Foo_Bar" (named deviation,
+-- Q#EC4).
 local function ascii_capitalize(s)
-  local first = nil
+  local out = {}
+  local in_word = false
   for i = 1, #s do
-    if is_word_byte(s:byte(i)) then
+    local b = s:byte(i)
-      first = i
+    local w = is_word_byte(b)
-      break
+    if w and not in_word then
+      if b >= 97 and b <= 122 then b = b - 32 end
+    elseif w then
+      if b >= 65 and b <= 90 then b = b + 32 end
     end
+    in_word = w
+    out[#out + 1] = string.char(b)
   end
-  local lowered = ascii_lower(s)
+  return table.concat(out)
-  if not first then return lowered end
-  local b = lowered:byte(first)
-  if b >= 97 and b <= 122 then
-    return lowered:sub(1, first - 1)
-      .. string.char(b - 32)
-      .. lowered:sub(first + 1)
-  end
-  return lowered
 end
 
 -- ---- Q#EC2 shared discipline ---------------------------------------
@@ -353,13 +377,17 @@ pmacs.command.define {
           or "transpose-chars: not enough characters")
         return
       end
-      local n2 = cp_len(at_b)
+      -- Validate the WHOLE scalar at the cursor, trailing bytes
-      if not n2 or cursor + n2 > len then
+      -- included — a valid lead followed by non-continuation bytes
+      -- must fail closed, not ride along as "one character".
+      local head = buf:slice(cursor, math.min(cursor + 4, len))
+      local n2 = scalar_len(head, 1)
+      if not n2 then
         ed.set_status("transpose-chars: malformed UTF-8 at the cursor")
         return
       end
       local cp1 = buf:slice(s1, cursor)
-      local cp2 = buf:slice(cursor, cursor + n2)
+      local cp2 = head:sub(1, n2)
       guarded_replace("transpose-chars", buf, s1, cursor + n2, cp2 .. cp1,
         cursor + n2)
     end
@@ -824,13 +852,26 @@ end
 -- loads before saveplace.lua (the loader ordering contract, Q#EC9).
 pmacs.hook.add("buffer.before-save", function()
   -- Outer pcall: a raised error in a short-circuit hook vetoes the
-  -- save; trim failure must report via status, never block a save.
+  -- save; trim failure must never block a save. But it must not be
-  pcall(function()
+  -- silently discarded either — an unexpected error is a bug, and
+  -- swallowing it hides the bug. Report on both channels the
+  -- autosave sweep uses: the status line (visible when the save
+  -- itself fails or is vetoed; a successful save overwrites it with
+  -- "saved ...") and the *errors* buffer via pmacs.error (durable
+  -- either way). Both reports are pcall'd — a broken reporting
+  -- channel must not resurrect the veto.
+  local ok, err = pcall(function()
     if trim_enabled then
       trim_active("delete-trailing-whitespace (on save)")
     end
   end)
-  -- nil return: never a veto.
+  if not ok then
+    local msg = "delete-trailing-whitespace (on save) failed: "
+      .. tostring(err)
+    pcall(ed.set_status, msg)
+    if pmacs.error then pcall(pmacs.error, msg) end
+  end
+  -- nil return either way: never a veto.
 end)
 
 -- ---- bindings (Q#EC1: all verified free across builtin bind sites) --

docs/editing-conveniences-framing.md

@@ -58,6 +58,17 @@ Revision 5 (post-approval, R3's optional hardening adopted):
 zap fails closed — no kill, chain broken — when public Lua consumed
 the marker while the prompt was open.
 
+Revision 6 (PR #111 round 1): codepoint recognition is full UTF-8
+scalar validation (second-byte constraint table; transpose
+validates the cursor scalar trailing-bytes-included, and a
+length-consistent overlong/surrogate span behind the cursor fails
+closed); capitalize is per-word across the span (Emacs
+capitalize-region parity, empirically verified, with the pack's
+`_`-is-a-word-constituent deviation named); trim-on-save reports
+unexpected errors on the status line AND the `*errors*` buffer via
+`pmacs.error` instead of silently discarding them, still never
+vetoing.
+
 ## Ground truth (as of `7e127ab`)
 
 - **The taken-chord registry is wider than `builtin/keymaps/
@@ -337,7 +348,16 @@ range gsub with a byte map — NOT `string.upper/lower` and NOT
 backend (ground truth); this also keeps Lua 5.4 and LuaJIT
 identical. Non-ASCII bytes pass through untouched — pinned in
 acceptance (an `é` in the span is byte-identical after the op).
-Capitalize = first word char upcased, rest of span downcased.
+
+Capitalize is PER-WORD across the span — Emacs capitalize-region
+parity (PR #111 R1 finding 2; empirical, Emacs 30.2 `-Q --batch`:
+`"hello WORLD"` → `"Hello World"`, `"9abc a9bc"` → `"9abc A9bc"`):
+each word's first byte is upcased when it is a letter, every other
+letter downcased; a digit-led word keeps its letters lowercase. One
+named deviation remains: `_` is a word constituent in this pack's
+class (the `word_at_cursor` precedent) but symbol-syntax in Emacs,
+so `foo_bar` capitalizes as `Foo_bar` here versus Emacs's
+`Foo_Bar`.
 
 ### Q#EC5 — Transpose: codepoint-aware chars, Emacs-verified word boundaries
 
@@ -345,13 +365,24 @@ Capitalize = first word char upcased, rest of span downcased.
 cursor, cursor ends after both (Emacs drag-forward). At EOL (next
 char is `\n` or EOF) with ≥2 preceding codepoints: swap the two
 before the cursor (Emacs special case). Fewer than two reachable
-codepoints → status, no edit. Codepoint boundaries by UTF-8
+codepoints → status, no edit. Codepoint recognition is FULL scalar
-lead-byte scan on a small slice around the cursor; **a continuation
+validation, not lead/continuation range checks (PR #111 R1 finding
-byte at the cursor position fails closed** (status, no edit) —
+1): a shared validator enforces the UTF-8 second-byte constraint
-`goto_byte` does not guarantee boundary alignment, and "never split
+table — overlongs (`C0`/`C1`, `E0 80..9F`, `F0 80..8F`), surrogates
-a codepoint" must hold against a misaligned start, not assume it
+(`ED A0..BF`), and beyond-`U+10FFFF` (`F4 90..BF`, `F5..FF`) all
-away. Newlines participate (transpose across lines works). One
+fail — and the scalar AT the cursor is validated trailing bytes
-replace spanning exactly the two codepoints.
+included (a valid lead followed by non-continuation bytes must not
+ride along as "one character"). Failures fail closed: a
+continuation byte at the cursor, a malformed scalar at the cursor,
+and a length-consistent-but-invalid span behind it each report and
+leave the buffer untouched — `goto_byte` does not guarantee
+boundary alignment, and buffers are byte-clean, so malformed input
+is reachable. (Zap's single-codepoint input check uses the same
+validator as defense-in-depth; minibuffer contents arrive as
+Rust-side UTF-8 — `set_contents` is `String`-typed — so the
+buffer-facing checks are the load-bearing ones.) Newlines
+participate (transpose across lines works). One replace spanning
+exactly the two codepoints.
 
 `edit.transpose-words` (M-t), specified against the Emacs 30.2
 table in Ground truth:
@@ -581,9 +612,15 @@ saveplace's cursor-record within the before-save fan-out (recorded
 places see post-trim text). The ENTIRE callback body is wrapped in
 pcall with a `nil` return on both paths (the saveplace pattern):
 returning `nil` never vetoes, but a raised error in a
-short-circuit hook vetoes immediately (`src/hook.rs:299`) — trim
+short-circuit hook vetoes immediately (`src/hook.rs:299`). An
-failure of any kind must report via status and never block the
+unexpected error caught by that pcall is NOT silently discarded
-save.
+(PR #111 R1 finding 3) — it reports on both channels the autosave
+sweep uses: the status line (visible when the save fails or is
+vetoed; a successful save overwrites it with "saved ...") and the
+`*errors*` buffer via `pmacs.error` (durable either way; the
+async/mcp/syntax/autosave convention). Both reports are
+themselves pcall'd so a broken reporting channel cannot resurrect
+the veto.
 
 ### Q#EC10 — Cut from the pack: recenter
 
@@ -658,12 +695,21 @@ harness the kill-ring suite already uses.
   separators skips forward to the next word; no word forward → no
   edit; `é` in the span is byte-identical while ASCII neighbors
   flip — and stays byte-identical regardless of process locale
-  (explicit-range pin).
+  (explicit-range pin); capitalize: region `"hello WORLD"` →
+  `"Hello World"` (per-word, the Emacs parity row), `"9abc a9bc"` →
+  `"9abc A9bc"` (digit-led word keeps letters lowercase), and
+  `"foo_bar baz"` → `"Foo_bar Baz"` (the named `_` deviation,
+  pinned).
 - Transpose-chars: mid-line swap + cursor advance; EOL two-before
   swap; BOB/single-char no-op; multi-byte: swapping `é` and `x`
   yields intact UTF-8 both orders; across-newline swap; **cursor
   parked on a continuation byte → status, no edit** (fail-closed
-  pin). Undo restores the original in ONE step (grain pin).
+  pin); **malformed-scalar pins**: a valid lead with a
+  non-continuation trailing byte at the cursor (`a\xC3xb`), an
+  overlong span behind the cursor (`\xE0\x80\x80b`), and a
+  beyond-`U+10FFFF` span behind it (`\xF4\x90\x80\x80b`) each →
+  status, buffer byte-identical. Undo restores the original in ONE
+  step (grain pin).
 - Transpose-words: the full nine-position Emacs table from Ground
   truth, byte-for-byte including final cursor positions for the
   seven mutating rows; the two no-successor rows assert NO edit and
@@ -751,7 +797,11 @@ harness the kill-ring suite already uses.
   (default) → save writes bytes untouched; **veto-immunity pin**: a
   rejecting intercept during on-save trim → the save still
   proceeds with a status report; another before-save callback's
-  veto still vetoes (trim's `nil` return masks nothing).
+  veto still vetoes (trim's `nil` return masks nothing);
+  **unexpected-error pin**: an error raised inside the on-save trim
+  (beyond the per-edit pcalls) → the save still proceeds AND the
+  failure lands in the `pmacs.error` log (stubbed, the m9_6
+  pattern) — never silently discarded.
 
 No CRDT-specific suite: every editops edit is a daemon-peer edit on
 the dispatch or minibuffer-accept path with no optimistic-classifier

tests/editops_acceptance.rs

@@ -306,11 +306,29 @@ fn capitalize_word_and_region() {
     alt(&mut s, 'c');
     assert_eq!(
         buffer_text(&s),
-        "Hello world",
+        "Hello World",
-        "single-span capitalize: first word char up, rest down"
+        "per-word capitalize across the region (Emacs capitalize-region)"
     );
 }
 
+#[test]
+fn capitalize_is_per_word_with_the_packs_word_class() {
+    // Emacs 30.2 parity rows: a digit-led word keeps its letters
+    // lowercase; a letter after a digit is not a word start.
+    let mut s = editor_with("9abc a9bc");
+    exec(&s, "pmacs.editor.begin_selection(0)");
+    exec(&s, "pmacs.editor.goto_byte(9)");
+    alt(&mut s, 'c');
+    assert_eq!(buffer_text(&s), "9abc A9bc");
+    // The named deviation: `_` is a word constituent in this pack's
+    // ASCII class (Emacs's symbol-syntax `_` would give "Foo_Bar").
+    let mut s = editor_with("foo_bar baz");
+    exec(&s, "pmacs.editor.begin_selection(0)");
+    exec(&s, "pmacs.editor.goto_byte(11)");
+    alt(&mut s, 'c');
+    assert_eq!(buffer_text(&s), "Foo_bar Baz");
+}
+
 #[test]
 fn case_ops_report_when_no_word_follows() {
     let mut s = editor_with("foo   ");
@@ -406,6 +424,55 @@ fn transpose_chars_fails_closed_on_a_continuation_byte() {
     assert!(status(&s).contains("multi-byte"));
 }
 
+/// Seed raw (possibly malformed) bytes via Lua \x escapes and compare
+/// byte-identically Lua-side (`from_utf8_lossy` would mask differences).
+fn seed_raw(s: &EditorState, lua_bytes: &str) {
+    exec(
+        s,
+        &format!("local b = pmacs.window.buffer(); b:replace(0, b:len(), \"{lua_bytes}\")"),
+    );
+    exec(s, "pmacs.editor.goto_byte(0)");
+}
+
+fn raw_equals(s: &EditorState, lua_bytes: &str) -> bool {
+    eval(
+        s,
+        &format!("local b = pmacs.window.buffer(); return b:slice(0, b:len()) == \"{lua_bytes}\""),
+    )
+}
+
+#[test]
+fn transpose_chars_fails_closed_on_invalid_trailing_bytes() {
+    // A valid lead (C3) followed by a non-continuation byte: the
+    // scalar at the cursor must be validated whole, not length-only.
+    let mut s = editor_with("");
+    seed_raw(&s, "a\\xC3xb");
+    exec(&s, "pmacs.editor.goto_byte(1)");
+    ctrl(&mut s, 't');
+    assert!(raw_equals(&s, "a\\xC3xb"), "no edit on a malformed scalar");
+    assert!(status(&s).contains("malformed UTF-8 at the cursor"));
+}
+
+#[test]
+fn transpose_chars_fails_closed_on_invalid_scalars_behind_the_cursor() {
+    // Length-consistent but scalar-invalid spans behind the cursor:
+    // an overlong three-byte encoding and a beyond-U+10FFFF four-byte
+    // encoding. Both scan back cleanly (lead + continuations) and
+    // must still fail closed.
+    let cases = [("\\xE0\\x80\\x80b", 3i64), ("\\xF4\\x90\\x80\\x80b", 4)];
+    for (bytes, cursor_pos) in cases {
+        let mut s = editor_with("");
+        seed_raw(&s, bytes);
+        exec(&s, &format!("pmacs.editor.goto_byte({cursor_pos})"));
+        ctrl(&mut s, 't');
+        assert!(raw_equals(&s, bytes), "no edit for {bytes}");
+        assert!(
+            status(&s).contains("malformed UTF-8 before the cursor"),
+            "fail-closed status for {bytes}"
+        );
+    }
+}
+
 // ---------------------------------------------------------------------------
 // Transpose-words: the nine-position Emacs 30.2 table (Q#EC5)
 // ---------------------------------------------------------------------------
@@ -1175,6 +1242,53 @@ fn trim_on_save_failure_never_vetoes_the_save() {
     let _ = std::fs::remove_file(&path);
 }
 
+#[test]
+fn trim_on_save_unexpected_error_reports_and_still_saves() {
+    let path = temp_path("unexpected.txt");
+    std::fs::write(&path, "x  \n").unwrap();
+    let mut s = EditorState::new();
+    exec(&s, "pmacs.editops.trim_on_save(true)");
+    // Capture the pmacs.error log (the m9_6 stub pattern — the
+    // `if pmacs.error` branch is a no-op without it).
+    exec(
+        &s,
+        r"
+        PMACS_ERROR_LOG = {}
+        pmacs.error = function(msg)
+            PMACS_ERROR_LOG[#PMACS_ERROR_LOG + 1] = msg
+        end
+        ",
+    );
+    exec(
+        &s,
+        &format!(
+            "pmacs.buffer.from_file({})",
+            lua_str(path.to_str().unwrap())
+        ),
+    );
+    // Force an error INSIDE trim, past the per-edit pcalls: trim's
+    // context snapshot reads pmacs.window.current(), which nothing
+    // else on the save path touches.
+    exec(
+        &s,
+        "TRIM_ORIG_WC = pmacs.window.current; pmacs.window.current = function() error('boom') end",
+    );
+    ctrl(&mut s, 'x');
+    ctrl(&mut s, 's');
+    exec(&s, "pmacs.window.current = TRIM_ORIG_WC");
+    assert_eq!(
+        std::fs::read_to_string(&path).unwrap(),
+        "x  \n",
+        "the save proceeded (trim aborted before any edit)"
+    );
+    let logged: String = eval(&s, "return PMACS_ERROR_LOG[1] or ''");
+    assert!(
+        logged.contains("delete-trailing-whitespace (on save) failed:"),
+        "the unexpected error reached the pmacs.error log, got: {logged}"
+    );
+    let _ = std::fs::remove_file(&path);
+}
+
 #[test]
 fn another_callbacks_veto_is_not_masked_by_trim() {
     let path = temp_path("veto.txt");