diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 307f734..0e8a05c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,6 +105,17 @@ jobs: - name: Run conformance suite run: cargo run --release -p epiphany-testkit --example conformance_suite 1 + # The Chapter 10 performance-budget gates (Phase 2 worklist F1), in + # release with reduced sampling (EPIPHANY_BENCH_QUICK): the 1K and 10K + # reduction points and both bundle budgets must pass; known-pending + # points are documented xfails inside the benches ("F surfaces, K + # fixes"). The heaviest (50K) reduction point runs only in the nightly + # soak below. + - name: Performance budget gates (quick) + env: + EPIPHANY_BENCH_QUICK: "1" + run: cargo bench -p epiphany-testkit + # Track A, Agent H's merge gate (Phase 2). A discrete job so a spelling / # decomposition pre-pass regression is attributable to H, not buried in the # workspace test run. Asserts H's PHASE2_QUICKSTART acceptance criterion: the @@ -138,3 +149,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Run conformance soak run: cargo run --release -p epiphany-testkit --example conformance_suite 10 + # The full budget-gate run, including the minute-scale 50K reduction + # point (a documented xfail until the O(n²) reducer fix lands). + - name: Performance budget gates (full) + run: cargo bench -p epiphany-testkit diff --git a/Cargo.lock b/Cargo.lock index 72a3b56..49ebbd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -127,6 +127,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + [[package]] name = "android-activity" version = "0.6.1" @@ -161,6 +170,18 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + [[package]] name = "arboard" version = "3.6.1" @@ -583,6 +604,12 @@ dependencies = [ "wayland-client", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.65" @@ -622,6 +649,58 @@ dependencies = [ "libc", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.5.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" + [[package]] name = "clipboard-win" version = "5.4.1" @@ -789,12 +868,52 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -997,6 +1116,12 @@ dependencies = [ "winit", ] +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + [[package]] name = "emath" version = "0.29.1" @@ -1151,6 +1276,7 @@ dependencies = [ name = "epiphany-testkit" version = "0.0.0" dependencies = [ + "criterion", "epiphany-bundle", "epiphany-core", "epiphany-determinism", @@ -1593,6 +1719,16 @@ dependencies = [ "bitflags 2.13.0", ] +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -1792,6 +1928,32 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + [[package]] name = "jni" version = "0.22.4" @@ -2466,6 +2628,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "orbclient" version = "0.3.55" @@ -2794,6 +2962,35 @@ dependencies = [ "bitflags 2.13.0", ] +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + [[package]] name = "renderdoc-sys" version = "1.1.0" @@ -2973,6 +3170,19 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "serde_repr" version = "0.1.20" @@ -3312,6 +3522,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.11.0" @@ -4541,6 +4761,12 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index 446ab7d..4c4ae81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -88,6 +88,13 @@ unicode-normalization = "0.1" # produce compressed fixtures. Default features (legacy formats, dictionary # building) are off — none are needed to decode standard frames. zstd = { version = "0.13", default-features = false } +# Criterion drives the Chapter 10 performance benches (Phase 2 worklist F1) in +# epiphany-testkit/benches (see that crate's DECISIONS.md, F0/F1). Pinned to +# the 0.5 line: its MSRV (1.70) fits the workspace's pinned 1.77, while +# criterion 0.6+ requires 1.80. Default features are off — the plotting stack +# (plotters) and rayon are dead weight for budget gates — keeping only +# `cargo_bench_support` so plain `cargo bench` remains the entry point. +criterion = { version = "0.5", default-features = false, features = ["cargo_bench_support"] } # Determinism-sensitive: never enable fast-math-style codegen on canonical # numerical paths (Appendix D, "Rounding and CPU Behavior"). The default diff --git a/crates/epiphany-bundle/DECISIONS.md b/crates/epiphany-bundle/DECISIONS.md index 893053b..586aec3 100644 --- a/crates/epiphany-bundle/DECISIONS.md +++ b/crates/epiphany-bundle/DECISIONS.md @@ -351,6 +351,14 @@ When that companion lands, reconcile this crate's `header`, `superblock`, round-trip would be the trigger, per the QUICKSTART process notes). This is the file-format analogue of `epiphany-core`'s P11-4. +> **Ratified (2026-07-02):** `spec/binary_format.tex` v0.1.0 Chapter 7 pins +> this crate's header (64-byte table), superblock (256-byte table), chunk +> framing and hash preimages, `ChunkRef`, block framing, the manifest body +> order with its sort/dedup rules, and the operation-index payload (P12-D1, +> `req:binfmt:opindex`) exactly as implemented and golden-locked here. The +> reconciliation trigger never fired: the companion was transcribed from this +> crate. + ### P11-D3 — Blob hashing shape is ambiguous Chapter 8 §"Blobs" says blobs are *"content-addressed identically to chunks diff --git a/crates/epiphany-core/DECISIONS.md b/crates/epiphany-core/DECISIONS.md index 6a1256b..9c881a7 100644 --- a/crates/epiphany-core/DECISIONS.md +++ b/crates/epiphany-core/DECISIONS.md @@ -139,6 +139,14 @@ companion lands, reconcile this crate's `CanonicalEncode`/`CanonicalDecode` and the whole-score `codec` with it (a failing cross-crate round-trip test would be the trigger, per the QUICKSTART process notes). +> **Ratified (2026-07-02):** the Binary Format companion now exists +> (`spec/binary_format.tex`, v0.1.0). Its Chapter 5 ratifies this crate's +> whole-`Score` positional codec, the convention macros, every discriminant +> table, and the `CanonicalValue` seam as the schema-major-0 wire form, with +> the frozen-layout rule (a field-set change is a schema-major change). No +> reconciliation was needed: the companion was transcribed from this codec and +> its golden anchors, so the trigger never fired. + **Phase 2 — Agent K (Operation Catalog): the `CanonicalValue` seam.** Track B's Operation Catalog shifts `epiphany-ops` from identifier-only operation payloads to *value-typed* ones (an `InsertEvent` carrying the real `Event`, a diff --git a/crates/epiphany-core/src/prepass.rs b/crates/epiphany-core/src/prepass.rs index 79a294e..c437832 100644 --- a/crates/epiphany-core/src/prepass.rs +++ b/crates/epiphany-core/src/prepass.rs @@ -1186,7 +1186,7 @@ fn decomposition_source_rank(source: &DecompositionSource) -> usize { /// decompositions only "for events that lack a higher-precedence attachment", /// with the "same precedence machinery" as spelling, minus the axes the /// attachment does not carry (no analysis layers, no `priority` field). Among -/// competing authored attachments the lowest [`decomposition_source_rank`] +/// competing authored attachments the lowest `decomposition_source_rank` /// wins; a remaining tie keeps the first candidate in the score's /// `decomposition_attachments` order, which is canonical (codec-fixed), so the /// resolution is deterministic across replicas. diff --git a/crates/epiphany-core/src/tempo.rs b/crates/epiphany-core/src/tempo.rs index 0689ef7..f8f00e9 100644 --- a/crates/epiphany-core/src/tempo.rs +++ b/crates/epiphany-core/src/tempo.rs @@ -163,7 +163,7 @@ const INVERSION_TOLERANCE_WHOLE_NOTES: f64 = 1e-6; /// The typed absolute residual tolerance of the inverse conversion /// ([`TempoMap::wallclock_to_musical`]): a [`Tolerance`] of class /// [`ToleranceClass::TempoIntegration`] (Appendix D §"Tolerance Classes" — no -/// ad-hoc epsilons), absolute bound [`INVERSION_TOLERANCE_WHOLE_NOTES`] whole +/// ad-hoc epsilons), absolute bound `INVERSION_TOLERANCE_WHOLE_NOTES` whole /// notes, no relative bound, governing validation (it decides whether a /// continued-fraction candidate is accepted; conversion is advisory, never /// canonical state). diff --git a/crates/epiphany-editor-core/src/lib.rs b/crates/epiphany-editor-core/src/lib.rs index 29dc1ae..25c92de 100644 --- a/crates/epiphany-editor-core/src/lib.rs +++ b/crates/epiphany-editor-core/src/lib.rs @@ -1000,7 +1000,7 @@ impl EditorSession { /// replaying the log reconstructs the same atomic unit. This is the substrate for /// intents that must land several primitives together (e.g. a value change with a /// matching respelling); editor-level atomicity (commit only on full success) is - /// inherited from [`Self::commit`]. + /// inherited from the private `commit` path. pub fn apply_transaction( &mut self, label: &str, diff --git a/crates/epiphany-engrave/src/lib.rs b/crates/epiphany-engrave/src/lib.rs index f40ce0e..aa32924 100644 --- a/crates/epiphany-engrave/src/lib.rs +++ b/crates/epiphany-engrave/src/lib.rs @@ -4,7 +4,7 @@ //! Agent I's **engraving constraint solver** (spec **Chapter 9**, "Constraint- //! Solver Interface"): it turns a [`ConstrainedLayoutIR`] into a //! [`ResolvedLayoutIR`] with real geometry. It is the production-side replacement -//! for `epiphany-layout-ir`'s interface-only [`StubSolver`] — the QUICKSTART puts +//! for `epiphany-layout-ir`'s interface-only `StubSolver` — the QUICKSTART puts //! the *interface* (`layout-ir`) and the *algorithm* (`engrave`) in separate //! crates so the core/product boundary stays sharp (`spec/PHASE2_QUICKSTART.md`, //! crate topology). @@ -12,7 +12,7 @@ //! ## Phase status — `Minimal` tier //! //! [`Engraver`] runs a genuine deterministic **horizontal spacing pass** (see -//! [`spacing`]) — placing each glyph-bearing slot left-to-right by a +//! the private `spacing` module) — placing each glyph-bearing slot left-to-right by a //! collision-aware advance (its preferred width floored by the real glyph //! bearings) — and **evaluates the IR's declared constraints** against the //! resolved geometry, routed by [`LayoutConstraint::strength`] (Chapter 9 diff --git a/crates/epiphany-layout-ir/DECISIONS.md b/crates/epiphany-layout-ir/DECISIONS.md index 47c82bb..7065de0 100644 --- a/crates/epiphany-layout-ir/DECISIONS.md +++ b/crates/epiphany-layout-ir/DECISIONS.md @@ -343,6 +343,12 @@ object is covered); the provenance-preservation contract itself is unchanged. declarations and gates `apply`/`apply_transaction` through `EditBarrier::prohibits_edit`. + > **Ratified (2026-07-02):** `spec/binary_format.tex` v0.1.0 Chapter 8 + > ratifies the blob byte form (P12-E1, `req:binfmt:ext-blobs`), pins + > `MAX_CONDITION_DEPTH = 64` as the normative recursion bound (P12-E2, + > `req:binfmt:condition-depth`), and adopts the open-value `ObjectKind` + > decode stance (P12-E3, `req:binfmt:object-kind-open`). + ## Pass 12 candidates (ambiguities for the spec, not resolved in code) 1. **Strength attachment to constraint instances.** Chapter 9 §"Strength Levels" diff --git a/crates/epiphany-layout-ir/src/hittest.rs b/crates/epiphany-layout-ir/src/hittest.rs index 87b81e9..e9de4e9 100644 --- a/crates/epiphany-layout-ir/src/hittest.rs +++ b/crates/epiphany-layout-ir/src/hittest.rs @@ -11,7 +11,7 @@ //! ## Coordinate frame //! //! Shapes are in **staff-space, y-up world** coordinates — the same frame as -//! [`RenderPrimitive::position`] and [`Stroke`] endpoints, *before* any +//! [`RenderPrimitive::position`] and stroke endpoints, *before* any //! renderer's world→screen transform. A GUI maps a screen point to this frame //! with the inverse of the same transform its renderer uses for display (for the //! SVG renderer, the inverse of its single `translate(-min_x, max_y) scale(1,-1)` diff --git a/crates/epiphany-ops/DECISIONS.md b/crates/epiphany-ops/DECISIONS.md index e63d142..dfd0d15 100644 --- a/crates/epiphany-ops/DECISIONS.md +++ b/crates/epiphany-ops/DECISIONS.md @@ -431,6 +431,15 @@ is deterministic and unambiguous but **provisional**: when the Binary Format companion lands, reconcile `encode.rs` and the per-type `CanonicalEncode` impls with it. A failing cross-crate round-trip test is the trigger. +> **Ratified (2026-07-02):** `spec/binary_format.tex` v0.1.0 Chapter 6 pins +> this crate's wire forms exactly as implemented — the envelope field order and +> its normative id-leads property, the `OperationPayload` (0..=3) and +> `OperationKind` (0..=23, append-only) discriminant tables, per-payload +> framing, `OperationKindTag`'s separate space, the 28-byte stamp, the DVV +> layout, and the full effects/conflict/anomaly/`MaterializedState` vocabulary. +> The reconciliation trigger never fired: the companion was transcribed from +> this crate and its golden anchors. + ## Spec-compliance audit follow-up (2026-07, Push 1) Four reduction-semantics fixes closing MUST-level gaps the six-agent spec audit @@ -708,3 +717,69 @@ equivocation fuzz plus the unchanged `run_equivocation_fuzz` gate this). no change to the determinism crate. No encoding changed and no discriminant was appended; `operation_kind_tag_decode_mirrors_encode_exactly` / `operation_kind_tag_decode_rejects_malformed_bytes` pin the contract. + +## Subquadratic `canonical_reduction_order` (2026-07, the F1 → K fix) + +- **The defect (F surfaces, K fixes).** The testkit's F1 bench + (`crates/epiphany-testkit/benches/reduction.rs`, Chapter 10: > 10,000 + envelopes/s cold) documented `canonical_reduction_order` as O(n²) twice + over: a literal double loop over all (predecessor, successor) pairs to + build indegrees, and a full ready-scan per emission. Measured pre-fix: + ~155K / ~12.5K / ~1.7K env/s at 1K / 10K / 50K envelopes (50K ≈ 29 s per + cold reduce — the documented xfail row). +- **The algorithm: term decomposition + monotone thresholds, never pairs.** + Materializing edges is inherently quadratic for the common chain shape + (every DVV floor covers the whole replica prefix, so covered *pairs* are + Θ(n²)); the rewrite therefore never enumerates pairs. Each causal-context + entry becomes one *requirement term* over the present set: + - *Vector floor `(r, n)`* — covers exactly the present envelopes of replica + `r` with counter `<= n` (zero-based floor, P11-C7): a **prefix** of the + replica's lane sorted by `(counter, slice index)`. The term is satisfied + when the lane's emission **frontier** (first unemitted slot, monotone) + passes the prefix; if the floor covers the envelope's own id, only the + self-pair is exempt, so that term instead reads "frontier at own slot and + **second frontier** past the prefix" (both monotone). Terms park in + per-lane `BTreeMap`s keyed by the threshold slot and are drained exactly + once as the frontiers advance. + - *Explicit dot* — covers exactly the present envelopes bearing that id; + satisfied when the id's unemitted multiplicity reaches 0 (1 for a + self-dot, which covers only duplicate-id twins). + An envelope is ready when its unsatisfied-term count reaches zero; ready + envelopes sit in a `BinaryHeap` keyed by `(reduction tuple, slice index)`. + A pre-sorted `(tuple, index)` list with a cursor supplies the malformed- + cycle fallback (heap empty, envelopes remain). Total work is + `O((n + Σ|context|) log n)`. +- **Why the order is byte-identical (the consensus argument).** The order is + *defined* by: edge `p → s` iff `p != s` and `s.context.covers(p.id)`; + ready iff every present covered predecessor emitted; emit the minimum + `(reduction tuple, slice index)` among ready (slice index reproduces + `min_by_key`'s first-minimum rule — reachable only under duplicate-id + tuple ties); if none is ready, the same minimum over all unemitted (cycle + break). The rewrite computes the *same readiness predicate*: each term is + satisfied iff all envelopes it covers are emitted, every covered + predecessor is covered by at least one term, and terms cover only covered + predecessors — so "all terms satisfied" ⇔ "all covered predecessors + emitted", including the exempt self-pair (a floor/dot naming the + envelope's own id) and envelopes covered by both a dot and a floor (the + conjunction needs no per-pair dedup; the redundant term is harmless). + Absent context entries (unknown replicas, floors below every present + counter, absent dots) yield no term, exactly as they yield no edge. Same + edge relation, same ready set, same total order on ready ⇒ the same + emission sequence, element for element. +- **The retained oracle + regression gate.** The pre-fix implementation is + kept verbatim as `canonical_reduction_order_reference` (`#[cfg(test)]`), + and three property tests assert element-for-element *pointer* equality of + both orders (the strictest check — it distinguishes byte-identical twins): + `canonical_order_matches_reference_on_fuzz_sets` (the crate's well-formed + fuzz generator, 250 seeds × 2 slice orders), + `..._on_adversarial_sets` (400 randomized hostile sets: self-covering + floors, dots to present/absent/own ids, duplicate ids with tied stamps, + `SYSTEM_DERIVED` replicas, stamps contradicting the causal edges, empty + contexts), and `..._on_directed_shapes` (a 2,000-envelope full-coverage + chain with descending stamps, self-covering and dot-only chains, dot + 2-/3-cycles, mutual-floor cycles, twin permutations). Mutation-checked: + breaking the self-exemption or the self-dot threshold fails the suite. +- **Measured post-fix (same bench, dev profile 2026-07):** ~674K / ~257K / + ~87K env/s at 1K / 10K / 50K — the 50K row cleared its budget by ~8.7x, + the gate printed its XPASS promotion notice, and the row was flipped to + `Pass` in the same change (see `epiphany-testkit/DECISIONS.md` F1). diff --git a/crates/epiphany-ops/src/payload.rs b/crates/epiphany-ops/src/payload.rs index 88c08c9..ebd5603 100644 --- a/crates/epiphany-ops/src/payload.rs +++ b/crates/epiphany-ops/src/payload.rs @@ -709,7 +709,7 @@ pub(crate) fn resolved_anchor_position(anchor: &TimeAnchor) -> MusicalPosition { impl SetUserSystemBreakOp { /// The anchor's resolved musical position — the canonical LWW bucketing key - /// (see [`resolved_anchor_position`]). + /// (see the private `resolved_anchor_position` helper). pub fn resolved_position(&self) -> MusicalPosition { resolved_anchor_position(&self.anchor) } @@ -1118,7 +1118,7 @@ pub struct SetUserPageBreakOp { impl SetUserPageBreakOp { /// The anchor's resolved musical position — the canonical LWW bucketing key - /// (see [`resolved_anchor_position`]). + /// (see the private `resolved_anchor_position` helper). pub fn resolved_position(&self) -> MusicalPosition { resolved_anchor_position(&self.anchor) } diff --git a/crates/epiphany-ops/src/reduce.rs b/crates/epiphany-ops/src/reduce.rs index db45d83..9e7e8be 100644 --- a/crates/epiphany-ops/src/reduce.rs +++ b/crates/epiphany-ops/src/reduce.rs @@ -28,7 +28,8 @@ //! deferred to the Operation Catalog (§6.11); see `DECISIONS.md` for the exact //! boundary. -use std::collections::{BTreeMap, BTreeSet}; +use std::cmp::Reverse; +use std::collections::{BTreeMap, BTreeSet, BinaryHeap}; use epiphany_core::{ canonical_pitch_bytes, derive_promoted_voice_id, AnchorOffset, AnnotationAnchor, @@ -59,6 +60,7 @@ use crate::payload::{ RespellPitchOp, SetMetadataOp, SetMetricGridOp, SetUserPageBreakOp, TransposeOp, TupletCompensation, }; +use crate::stamp::StampTuple; use crate::support::{ObjectKind, SerializedCanonicalInputs}; use crate::undo::{UndoPolicy, UndoTransactionPayload}; @@ -71,8 +73,297 @@ use crate::undo::{UndoPolicy, UndoTransactionPayload}; /// coverage, choosing the smallest HLC reduction tuple among ready operations. /// A malformed causal cycle has no valid topological order; the smallest HLC /// tuple deterministically breaks the cycle so every replica still converges. +/// +/// ## Subquadratic construction (worklist F1 → K fix) +/// +/// The order is *defined* pairwise: an edge `p → s` exists iff `p != s` and +/// `s.causal_context.covers(p.id)`; an envelope is *ready* when every present +/// covered predecessor has been emitted; the smallest reduction tuple among +/// ready envelopes emits next (slice position breaks the — duplicate-id-only — +/// tuple ties, matching `min_by_key`'s first-minimum rule, which the retained +/// test-only `canonical_reduction_order_reference` oracle implements literally +/// in O(n²)). Materializing the edges is inherently quadratic for the common +/// chain-context shape (every DVV floor covers the full replica prefix), so +/// this implementation never enumerates pairs. It decomposes each context into +/// *requirement terms* whose conjunction is exactly pairwise readiness: +/// +/// * A **vector floor** `(r, n)` covers precisely the present envelopes of +/// replica `r` with counter `<= n` (the zero-based DVV floor, P11-C7) — +/// a *prefix* of the replica's lane sorted by `(counter, slice index)`. +/// The term is satisfied when the lane's emission frontier (the first +/// unemitted lane slot) passes the prefix; a floor that covers the +/// envelope's *own* id exempts only the self-pair, so that term is instead +/// "frontier at own slot and *second* frontier past the prefix". Both +/// frontiers are monotone, so each term is woken exactly once from a +/// `BTreeMap` keyed by the threshold slot. +/// * An **explicit dot** covers precisely the present envelopes bearing that +/// id; the term is satisfied when the id's unemitted multiplicity reaches +/// zero (or one, for a dot naming the envelope's own id). A dot also lying +/// under one of the context's own floors yields a (redundant) second term — +/// harmless, because readiness is the conjunction of terms, not a +/// predecessor count, so no per-pair dedup is needed. +/// +/// Context entries covering no present envelope (absent replicas, floors below +/// every present counter, absent dots) yield no term, exactly as they yield no +/// edge pairwise. Total work is `O((n + Σ context entries) · log n)`. pub fn canonical_reduction_order<'a>( envelopes: &[&'a OperationEnvelope], +) -> Vec<&'a OperationEnvelope> { + let len = envelopes.len(); + let keys: Vec = envelopes + .iter() + .map(|env| env.stamp.reduction_tuple()) + .collect(); + + // Static indexes over the present set: per-replica lanes sorted by + // (counter, slice index), each envelope's (lane, slot) position, and the + // per-id unemitted multiplicity (> 1 only for duplicate ids, e.g. + // equivocation twins fed to this function directly). + let mut lane_of: BTreeMap = BTreeMap::new(); + let mut lanes: Vec = Vec::new(); + let mut position = vec![(0usize, 0usize); len]; + { + let mut sorted: Vec<(ReplicaId, u64, usize)> = envelopes + .iter() + .enumerate() + .map(|(index, env)| (env.id.replica, env.id.counter, index)) + .collect(); + sorted.sort_unstable(); + for (replica, counter, index) in sorted { + let lane_index = *lane_of.entry(replica).or_insert_with(|| { + lanes.push(OrderLane::default()); + lanes.len() - 1 + }); + let lane = &mut lanes[lane_index]; + position[index] = (lane_index, lane.slots.len()); + lane.slots.push(index); + lane.counters.push(counter); + } + for lane in &mut lanes { + // All slots start unemitted: frontier 0, second frontier 1 + // (clamped to the lane length, the "exhausted" sentinel). + lane.second = 1.min(lane.slots.len()); + } + } + let mut id_slots: BTreeMap = BTreeMap::new(); + for env in envelopes { + id_slots.entry(env.id).or_default().unemitted += 1; + } + + // One requirement term per covering context entry; `remaining` counts the + // currently-unsatisfied terms. Terms already satisfied here (they cover + // nothing, or nothing beyond the envelope itself) register no watcher. + let mut remaining = vec![0usize; len]; + for (index, env) in envelopes.iter().enumerate() { + for (&replica, &floor) in &env.causal_context.vector { + let Some(&lane_index) = lane_of.get(&replica) else { + continue; + }; + let lane = &mut lanes[lane_index]; + let prefix = lane.counters.partition_point(|&counter| counter <= floor); + if prefix == 0 { + continue; + } + if env.id.replica == replica && env.id.counter <= floor { + // The floor covers this envelope's own id; only the self-pair + // is exempt. Required: every other prefix slot emitted, i.e. + // frontier at the own slot *and* second frontier past the + // prefix (the own slot stays unemitted until emission). + let own_slot = position[index].1; + if lane.frontier < own_slot { + remaining[index] += 1; + lane.frontier_watchers.entry(own_slot).or_default().push( + FloorWatcher::ExceptSelf { + node: index, + prefix, + }, + ); + } else if lane.second < prefix { + remaining[index] += 1; + lane.second_watchers.entry(prefix).or_default().push(index); + } + } else if lane.frontier < prefix { + remaining[index] += 1; + lane.frontier_watchers + .entry(prefix) + .or_default() + .push(FloorWatcher::Whole { node: index }); + } + } + for dot in env.causal_context.dots() { + let Some(id_slot) = id_slots.get_mut(&dot) else { + continue; // absent id: covers nothing present, no edge + }; + if dot == env.id { + // A dot naming the envelope's own id covers only duplicates. + if id_slot.unemitted > 1 { + remaining[index] += 1; + id_slot.watch_one.push(index); + } + } else { + remaining[index] += 1; + id_slot.watch_zero.push(index); + } + } + } + + // Deterministic Kahn walk. The heap holds every envelope whose terms are + // all satisfied (pushed exactly at the transition; entries for envelopes + // already emitted through cycle-breaking are skipped lazily), keyed by + // (reduction tuple, slice index) — the reference's `min_by_key` order. + let mut heap: BinaryHeap> = (0..len) + .filter(|&index| remaining[index] == 0) + .map(|index| Reverse((keys[index], index))) + .collect(); + let mut by_key: Vec = (0..len).collect(); + by_key.sort_unstable_by_key(|&index| (keys[index], index)); + let mut cycle_cursor = 0usize; + + let mut emitted = vec![false; len]; + let mut ordered = Vec::with_capacity(len); + let mut woken: Vec = Vec::new(); + while ordered.len() < len { + let mut ready = None; + while let Some(Reverse((_, index))) = heap.pop() { + if !emitted[index] { + ready = Some(index); + break; + } + } + let next = match ready { + Some(index) => index, + None => { + // A cycle is malformed, but selecting by the canonical + // tie-breaker keeps reduction deterministic and unlocks the + // forced envelope's dependents. + while emitted[by_key[cycle_cursor]] { + cycle_cursor += 1; + } + by_key[cycle_cursor] + } + }; + emitted[next] = true; + ordered.push(envelopes[next]); + + // Dot wake-ups: the id's unemitted multiplicity dropped by one. + let id_slot = id_slots + .get_mut(&envelopes[next].id) + .expect("every present id has a slot"); + id_slot.unemitted -= 1; + if id_slot.unemitted <= 1 { + woken.append(&mut id_slot.watch_one); + } + if id_slot.unemitted == 0 { + woken.append(&mut id_slot.watch_zero); + } + + // Floor wake-ups: advance the lane frontiers (both point at unemitted + // slots — or the lane length — by invariant, so an emission below the + // second frontier is at one of them) and drain the passed watchers. + let (lane_index, slot) = position[next]; + let lane = &mut lanes[lane_index]; + if slot == lane.frontier { + lane.frontier = lane.second; + lane.second = next_unemitted(&lane.slots, &emitted, lane.frontier + 1); + } else if slot == lane.second { + lane.second = next_unemitted(&lane.slots, &emitted, slot + 1); + } + while let Some(entry) = lane.frontier_watchers.first_entry() { + if *entry.key() > lane.frontier { + break; + } + for watcher in entry.remove() { + match watcher { + FloorWatcher::Whole { node } => woken.push(node), + FloorWatcher::ExceptSelf { node, prefix } => { + if lane.second >= prefix { + woken.push(node); + } else { + lane.second_watchers.entry(prefix).or_default().push(node); + } + } + } + } + } + while let Some(entry) = lane.second_watchers.first_entry() { + if *entry.key() > lane.second { + break; + } + for node in entry.remove() { + woken.push(node); + } + } + + for node in woken.drain(..) { + remaining[node] -= 1; + if remaining[node] == 0 && !emitted[node] { + heap.push(Reverse((keys[node], node))); + } + } + } + ordered +} + +/// One replica's present envelopes in [`canonical_reduction_order`], sorted by +/// `(counter, slice index)`, with the two monotone emission frontiers and the +/// floor-term watchers keyed by the frontier slot they wait for. +#[derive(Default)] +struct OrderLane { + /// Envelope slice indexes, sorted by `(counter, slice index)`. + slots: Vec, + /// The slots' counters (parallel to `slots`, ascending). + counters: Vec, + /// First unemitted slot (== `slots.len()` once exhausted). + frontier: usize, + /// Second unemitted slot (>= `slots.len()` once fewer than two remain). + second: usize, + /// Floor terms waiting for `frontier >= key`. + frontier_watchers: BTreeMap>, + /// Self-exempt floor terms waiting for `second >= key`. + second_watchers: BTreeMap>, +} + +/// One present `OperationId`'s bookkeeping in [`canonical_reduction_order`]: +/// its unemitted multiplicity and the dot terms watching it. +#[derive(Default)] +struct IdSlot { + /// Present envelopes bearing this id that are not yet emitted. + unemitted: usize, + /// Dot terms satisfied when `unemitted` reaches zero. + watch_zero: Vec, + /// Self-dot terms (duplicate ids) satisfied when `unemitted` reaches one. + watch_one: Vec, +} + +/// A floor term parked in [`OrderLane::frontier_watchers`]. +enum FloorWatcher { + /// Satisfied outright when the frontier reaches its key (the prefix end). + Whole { node: usize }, + /// A floor covering the node's own id: when the frontier reaches the + /// node's own slot (its key), the term is satisfied if the second + /// frontier already passed `prefix`, else it re-parks on the second + /// frontier. + ExceptSelf { node: usize, prefix: usize }, +} + +/// The first unemitted slot position at or after `from` (== `slots.len()` when +/// exhausted). Frontier scans only ever move forward, so the total scan work +/// per lane is linear. +fn next_unemitted(slots: &[usize], emitted: &[bool], from: usize) -> usize { + let mut at = from.min(slots.len()); + while at < slots.len() && emitted[slots[at]] { + at += 1; + } + at +} + +/// The pre-F1 O(n²) implementation, retained verbatim as the property-test +/// oracle for [`canonical_reduction_order`]: it materializes every covered +/// `(predecessor, successor)` pair and re-scans the whole set per emission, +/// which *is* the order's pairwise definition, executed literally. +#[cfg(test)] +pub(crate) fn canonical_reduction_order_reference<'a>( + envelopes: &[&'a OperationEnvelope], ) -> Vec<&'a OperationEnvelope> { let len = envelopes.len(); let mut indegree = vec![0usize; len]; @@ -4839,6 +5130,346 @@ mod tests { } } + // --- Subquadratic canonical order vs. the retained O(n²) oracle. -------- + + /// A minimal envelope for pure ordering tests: payload content never + /// affects the canonical reduction order. + fn order_env( + replica: ReplicaId, + counter: u64, + physical: i64, + logical: u32, + ctx: CausalContext, + ) -> OperationEnvelope { + let id = OperationId::new(replica, counter); + OperationEnvelope { + id, + author: AuthorId(1), + stamp: OperationStamp::new( + HybridLogicalClock::new(WallClockTime(physical), logical), + id, + ), + causal_context: ctx, + transaction: None, + payload: OperationPayload::Primitive(OperationKind::DeleteEvent(DeleteEventOp { + event: EventId::new(ReplicaId(7), counter % 5), + tuplet_compensation: TupletCompensation::NotInTuplet, + })), + } + } + + /// Asserts the subquadratic order equals the reference oracle's + /// element-for-element — by slice element identity (pointer equality), the + /// strictest possible check: it distinguishes even byte-identical + /// duplicate envelopes, whose tuple ties both implementations must break + /// by slice position. + fn assert_order_matches_reference(envelopes: &[OperationEnvelope]) { + let refs: Vec<&OperationEnvelope> = envelopes.iter().collect(); + let fast = canonical_reduction_order(&refs); + let oracle = canonical_reduction_order_reference(&refs); + assert_eq!(fast.len(), oracle.len()); + for (at, (a, b)) in fast.iter().zip(&oracle).enumerate() { + assert!( + std::ptr::eq(*a, *b), + "canonical order diverges from the reference at position {at}: \ + {:?} vs {:?} (n = {})", + a.id, + b.id, + envelopes.len() + ); + } + } + + /// In-place Fisher–Yates driven by the seeded generator (slice order is an + /// input to the tuple tie-break, so permutations must be exercised too). + fn shuffle_envelopes( + envelopes: &mut [OperationEnvelope], + rng: &mut epiphany_determinism::fuzz::SplitMix64, + ) { + for i in (1..envelopes.len()).rev() { + let j = (rng.next_u64() % (i as u64 + 1)) as usize; + envelopes.swap(i, j); + } + } + + /// A hostile ordering input the crate's well-formed generators avoid: + /// floors that cover the envelope's own id or absent counters, dots to + /// present / absent / own ids, duplicate ids (byte-identical twins), + /// `SYSTEM_DERIVED` replicas, colliding stamps that contradict the causal + /// edges (so the topological pass and the cycle-breaker both engage), and + /// empty contexts. + fn adversarial_set( + rng: &mut epiphany_determinism::fuzz::SplitMix64, + n: usize, + ) -> Vec { + let replicas = 1 + rng.next_u64() % 4; + let mut next_counter: BTreeMap = BTreeMap::new(); + let mut envs: Vec = Vec::with_capacity(n); + for _ in 0..n { + if !envs.is_empty() && rng.next_u64() % 8 == 0 { + // A duplicate id — and a byte-identical stamp, so the + // reduction tuple genuinely ties. + let victim = envs[(rng.next_u64() % envs.len() as u64) as usize].clone(); + envs.push(victim); + continue; + } + let replica = if rng.next_u64() % 16 == 0 { + ReplicaId::SYSTEM_DERIVED + } else { + ReplicaId(1 + rng.next_u64() % replicas) + }; + let slot = next_counter.entry(replica).or_insert(0); + // Occasionally skip counters so floors assert absent predecessors. + let counter = *slot + rng.next_u64() % 2; + *slot = counter + 1; + let id = OperationId::new(replica, counter); + + let mut ctx = CausalContext::new(); + for _ in 0..rng.next_u64() % 3 { + let target = match rng.next_u64() % 5 { + 0 => replica, // may cover the envelope's own id + 1 => ReplicaId(9), // absent replica + 2 => ReplicaId::SYSTEM_DERIVED, + _ => ReplicaId(1 + rng.next_u64() % replicas), + }; + // May exceed every present counter (covering future authoring + // of that replica — a causal cycle) or fall below all of them. + ctx = ctx.with_seen(target, rng.next_u64() % 8); + } + for _ in 0..rng.next_u64() % 3 { + let dot = if !envs.is_empty() && rng.next_u64() % 2 == 0 { + envs[(rng.next_u64() % envs.len() as u64) as usize].id + } else if rng.next_u64() % 4 == 0 { + id // the envelope's own id + } else { + OperationId::new(ReplicaId(1 + rng.next_u64() % 5), rng.next_u64() % 10) + }; + ctx = ctx.with_dot(dot); + } + + // Tiny stamp ranges force heavy tuple collisions and stamps that + // contradict the causal edges. + envs.push(order_env( + replica, + counter, + (rng.next_u64() % 6) as i64, + (rng.next_u64() % 3) as u32, + ctx, + )); + } + envs + } + + #[test] + fn canonical_order_matches_reference_on_fuzz_sets() { + // The crate's own well-formed generator: multi-replica meshes of + // vector floors, occasional equivocation twins (duplicate ids) and + // HLC-monotonicity anomalies, empty contexts on counter-0 roots. + let mut rng = epiphany_determinism::fuzz::SplitMix64::new(0xF1_0DE2_0001); + for _ in 0..250 { + let n = 1 + (rng.next_u64() % 40) as usize; + let mut envs = crate::fuzz::gen_envelope_set(&mut rng, n); + assert_order_matches_reference(&envs); + shuffle_envelopes(&mut envs, &mut rng); + assert_order_matches_reference(&envs); + } + } + + #[test] + fn canonical_order_matches_reference_on_adversarial_sets() { + let mut rng = epiphany_determinism::fuzz::SplitMix64::new(0xADE5_A71A_0002); + assert_order_matches_reference(&[]); + for iteration in 0..400 { + let n = 1 + (rng.next_u64() % 60) as usize; + let mut envs = adversarial_set(&mut rng, n); + assert_order_matches_reference(&envs); + if iteration % 4 == 0 { + shuffle_envelopes(&mut envs, &mut rng); + assert_order_matches_reference(&envs); + } + } + } + + #[test] + fn canonical_order_matches_reference_on_directed_shapes() { + let r = ReplicaId(1); + + // A 2,000-envelope single-replica chain whose every DVV floor covers + // the full replica prefix — the inherently-quadratic-pairs shape the + // subquadratic construction exists for — with *descending* stamps, so + // the causal edges (not the HLC) decide every single emission. + let full_chain: Vec = (0..2_000) + .map(|c| { + let ctx = if c == 0 { + CausalContext::new() + } else { + CausalContext::new().with_seen(r, c - 1) + }; + order_env(r, c, 2_000 - c as i64, 0, ctx) + }) + .collect(); + assert_order_matches_reference(&full_chain); + + // A self-covering chain: every floor also covers the envelope's own + // id (the exempted self-pair). + let self_chain: Vec = (0..600) + .map(|c| { + order_env( + r, + c, + 600 - c as i64, + 0, + CausalContext::new().with_seen(r, c), + ) + }) + .collect(); + assert_order_matches_reference(&self_chain); + + // A dot-only chain (no vector floors at all). + let dot_chain: Vec = (0..600) + .map(|c| { + let ctx = if c == 0 { + CausalContext::new() + } else { + CausalContext::new().with_dot(OperationId::new(r, c - 1)) + }; + order_env(r, c, 600 - c as i64, 0, ctx) + }) + .collect(); + assert_order_matches_reference(&dot_chain); + + // Malformed dot cycles (2-cycle and 3-cycle) among bystanders with + // empty contexts and identical stamps. + let id = |rep: u64, c: u64| OperationId::new(ReplicaId(rep), c); + let cycles = vec![ + order_env( + ReplicaId(2), + 0, + 5, + 0, + CausalContext::new().with_dot(id(2, 1)), + ), + order_env( + ReplicaId(2), + 1, + 5, + 0, + CausalContext::new().with_dot(id(2, 0)), + ), + order_env( + ReplicaId(3), + 0, + 5, + 0, + CausalContext::new().with_dot(id(3, 2)), + ), + order_env( + ReplicaId(3), + 1, + 5, + 0, + CausalContext::new().with_dot(id(3, 0)), + ), + order_env( + ReplicaId(3), + 2, + 5, + 0, + CausalContext::new().with_dot(id(3, 1)), + ), + order_env(ReplicaId(4), 0, 5, 0, CausalContext::new()), + order_env(ReplicaId(5), 0, 5, 0, CausalContext::new()), + ]; + assert_order_matches_reference(&cycles); + + // Mutual full-coverage floors (a floor cycle where each envelope also + // covers itself), plus coverage of absent ids on an absent replica. + let floor_cycle = vec![ + order_env( + ReplicaId(6), + 0, + 9, + 0, + CausalContext::new().with_seen(ReplicaId(6), 1), + ), + order_env( + ReplicaId(6), + 1, + 8, + 0, + CausalContext::new().with_seen(ReplicaId(6), 1), + ), + order_env( + ReplicaId(6), + 2, + 7, + 0, + CausalContext::new() + .with_seen(ReplicaId(40), 12) + .with_dot(id(41, 3)), + ), + ]; + assert_order_matches_reference(&floor_cycle); + + // Byte-identical duplicate ids (tuple ties broken by slice position) + // in several slice orders, including a dot and a floor onto the + // duplicated id. + let twin = order_env(ReplicaId(2), 3, 1, 0, CausalContext::new()); + let mut twins = vec![ + twin.clone(), + twin.clone(), + order_env( + ReplicaId(2), + 4, + 0, + 0, + CausalContext::new().with_dot(id(2, 3)), + ), + order_env( + ReplicaId(3), + 0, + 0, + 0, + CausalContext::new().with_seen(ReplicaId(2), 3), + ), + // A twin that dots its own id: covers only its duplicate. + order_env( + ReplicaId(2), + 3, + 1, + 0, + CausalContext::new().with_dot(id(2, 3)), + ), + ]; + assert_order_matches_reference(&twins); + twins.reverse(); + assert_order_matches_reference(&twins); + twins.swap(0, 2); + assert_order_matches_reference(&twins); + + // SYSTEM_DERIVED authoring under floors and dots from user replicas. + let sys = ReplicaId::SYSTEM_DERIVED; + let system = vec![ + order_env(sys, 0, 3, 0, CausalContext::new()), + order_env(sys, 1, 2, 0, CausalContext::new().with_seen(sys, 0)), + order_env( + ReplicaId(2), + 0, + 1, + 0, + CausalContext::new().with_seen(sys, 1), + ), + order_env( + ReplicaId(2), + 1, + 0, + 0, + CausalContext::new().with_dot(id(u64::MAX, 0)), + ), + ]; + assert_order_matches_reference(&system); + } + #[test] fn reduction_is_permutation_invariant() { let envs = vec![ diff --git a/crates/epiphany-ops/src/valuegen.rs b/crates/epiphany-ops/src/valuegen.rs index 5ca35cc..5a4bb9a 100644 --- a/crates/epiphany-ops/src/valuegen.rs +++ b/crates/epiphany-ops/src/valuegen.rs @@ -269,7 +269,7 @@ pub fn region(id: RegionId) -> Region { } /// Score metadata with a `nth`-distinct title (M2d) — distinct `nth` give -/// distinct [`ScoreMetadata`] values so a harness can drive concurrent +/// distinct `ScoreMetadata` values so a harness can drive concurrent /// `SetMetadata`s, an advisory LWW field that resolves by canonical order with /// no conflict. pub fn score_metadata(nth: u8) -> epiphany_core::ScoreMetadata { diff --git a/crates/epiphany-render-svg/src/lib.rs b/crates/epiphany-render-svg/src/lib.rs index c9127c5..9374d26 100644 --- a/crates/epiphany-render-svg/src/lib.rs +++ b/crates/epiphany-render-svg/src/lib.rs @@ -3,7 +3,7 @@ //! //! Agent I's **SVG renderer** behind the Epiphany `RenderIR` interface (spec //! **Chapter 7** §"RenderIR"): it turns a -//! [`ResolvedLayoutIR`](epiphany_layout_ir::ResolvedLayoutIR) into well-formed +//! [`ResolvedLayoutIR`] into well-formed //! **SVG 1.1**, drawing each glyph from **genuine Bravura SMuFL** data: inline //! outline ``s by default, or `` set in an `@font-face`-embedded //! subset. It is the visible end of the v0 `Score → layout IR` pipeline: from a @@ -15,7 +15,7 @@ //! the renderer was golden-locked against the **stub solver's** output first, then //! against the real [`epiphany_engrave`](../epiphany_engrave/index.html) solver //! once real notation and re-spacing landed. The renderer consumes any solver's -//! [`ResolvedLayoutIR`](epiphany_layout_ir::ResolvedLayoutIR): it preserves the +//! [`ResolvedLayoutIR`]: it preserves the //! resolved geometry, provenance traces, XML validity, deterministic output, and //! glyph-mode choice without making engraving-semantic decisions. //! @@ -24,8 +24,9 @@ //! The bundled outlines are extracted from the official OFL `Bravura.otf` (see //! `tools/extract_bravura_outlines.py` and `tools/OFL.txt`) in staff-space, //! y-up coordinates. The renderer makes SVG-encoding choices only and never -//! engraving-semantic ones; see [`svg`] for the coordinate system, the -//! provenance-tracing contract, and the diagnostic-not-paper-over rule. +//! engraving-semantic ones; see the private `svg` module for the coordinate +//! system, the provenance-tracing contract, and the diagnostic-not-paper-over +//! rule. //! //! ## Font availability //! diff --git a/crates/epiphany-testkit/Cargo.toml b/crates/epiphany-testkit/Cargo.toml index 79c1f70..51d0243 100644 --- a/crates/epiphany-testkit/Cargo.toml +++ b/crates/epiphany-testkit/Cargo.toml @@ -16,7 +16,25 @@ epiphany-ops.workspace = true # now drives the real crate instead of an in-tree stub. epiphany-layout-ir.workspace = true +# The Chapter 10 performance benches (worklist F1) are the only dev-dependency +# user: the budget-gate logic itself lives in `src/budget.rs` on plain `std`, +# so the library builds without criterion. +[dev-dependencies] +criterion.workspace = true + # Drives the whole suite at scale outside the unit-test timeout — the analogue # of epiphany-determinism's `fuzz_roundtrip` and epiphany-bundle's `fuzz_crash`. [[example]] name = "conformance_suite" + +# The Chapter 10 performance-budget benches (worklist F1; DECISIONS.md F0/F1). +# `harness = false`: each bench owns `main()`, running criterion's measurements +# first and then the budget gate — the assertion layer criterion cannot express +# — exiting nonzero on an unexpected budget miss. +[[bench]] +name = "reduction" +harness = false + +[[bench]] +name = "bundle" +harness = false diff --git a/crates/epiphany-testkit/DECISIONS.md b/crates/epiphany-testkit/DECISIONS.md index 0210eec..3b2b4cc 100644 --- a/crates/epiphany-testkit/DECISIONS.md +++ b/crates/epiphany-testkit/DECISIONS.md @@ -48,7 +48,7 @@ Concretely: `[prepass-harness]` stage in the conformance suite. - **`benches/` lives in this crate** (not per-crate). The Chapter-10 budgets are - workspace-level, and the marquee bench (the reducer's `O(n²)` + workspace-level, and the marquee bench (the reducer's — at the time `O(n²)` — `canonical_reduction_order` at 10K+ envelopes, worklist F1) drives `epiphany-ops` *through* the testkit's envelope generators — exactly what this crate already does. A bench that lived in `epiphany-ops` could not reuse the @@ -70,6 +70,96 @@ mirrors how v0's `convergence`/`roundtrip` modules are shared between **Unblocks:** F1 (benches), F3 (corpus + taxonomy harness — done), F4 (per-agent harness skeletons — H done), F5 (integration skeleton). +## F1 — The Chapter 10 budget benches (`benches/reduction.rs`, `benches/bundle.rs`) + +*Worklist item F1: a criterion bench asserting Chapter-10 budgets at documented +scale points, known-pending points marked xfail with the numeric budget written +in the bench. Home per F0: this crate's `benches/`.* + +**Criterion version: `0.5` (locks to 0.5.1), default features off plus +`cargo_bench_support`.** The 0.5 line's MSRV (1.70) fits the workspace's pinned +`rust-version = "1.77"`; criterion 0.6+ requires 1.80. Disabling default +features drops the plotting stack (`plotters`) and `rayon` — dead weight for +budget gates — while `cargo_bench_support` keeps plain `cargo bench` as the +entry point. Two transitive pins in `Cargo.lock` keep the tree MSRV-clean, +because the resolver (v2) is not MSRV-aware: `clap 4.5.53` (MSRV 1.74; 4.6 +requires 1.85) and `half 2.4.1` (MSRV 1.70; 2.5+ requires 1.81). Re-check those +two if `cargo update` touches the criterion tree. + +**The gate mechanism.** Criterion measures but never asserts, so each bench is +`harness = false` and its `main()` runs criterion's measurements first, then +the budget gate in [`crate::budget`] — a library module (the F0 pattern: shared +logic lives in `src/`, and it deliberately uses no criterion types, so the +library itself takes no new dependency). Every budget row carries an +`Expectation` written in the bench source next to its numeric threshold: + +- `Pass` — must hold today; a miss exits nonzero (the CI tripwire). +- `Xfail(reason)` — documented known-pending miss; the reason names the defect + and owner. A miss prints an expected-failure line; a **pass** prints an XPASS + promotion notice, so a stale marking is loud in both directions. + +Skipped rows (the 50K point in quick mode) print an explicit `skip` line, never +silence. Under `cargo test --benches` (criterion test mode, detected by the +absence of the `--bench` flag `cargo bench` passes) the measurements run once +and the gate is skipped — the gate belongs to `cargo bench`. + +**Sampling calibration (a documented deviation from Chapter 10).** The +conformance methodology — p99 over ≥1000 iterations on the reference hardware +profile — is the reference suite's job, not this gate's: 1000 iterations of the +(pre-fix ~29 s) 50K cold reduction was not a CI shape. The gate takes the +**median of a small per-row iteration count** (reduction: 9/3/1 iterations at +1K/10K/50K; bundle: 40 commit / 25 read; cut to 5/2/skip and 12/8 under +`EPIPHANY_BENCH_QUICK=1`), +cold (no in-gate warm-up — the marquee budget is an explicitly *cold* rate). +Criterion uses flat sampling with `sample_size(10)` for the reduction group; +the 50K point is gate-only (calibrated when a single iteration took ~29 s and +criterion's ≥10-sample loop would have blown the wall-clock budget; ~0.58 s +post-fix). Full `cargo bench -p epiphany-testkit` stays under ~2 minutes of +measurement; quick mode under ~15 seconds. + +**Measured on the dev profile (2026-07, post-K-fix), the table's ground +truth:** + +| row | budget | measured | verdict | +|-----|--------|----------|---------| +| `reduction/1000` | > 10,000 env/s cold | ~674,000 env/s (~1.5 ms) | Pass | +| `reduction/10000` | > 10,000 env/s cold | ~257,000 env/s (~39 ms) | Pass | +| `reduction/50000` | > 10,000 env/s cold | ~87,000 env/s (~0.58 s) | Pass (promoted from Xfail) | +| `bundle/typical_edit_commit` | ≤ 50 ms | ~14.7 ms (real fsync) | Pass | +| `bundle/open_bootstrap_read` | ≤ 200 ms | ~63 µs (moderate corpus) | Pass | + +As F1 first measured it (2026-07, pre-fix), the O(n²) +`canonical_reduction_order` indegree construction was unambiguous in the curve +(~155K / ~12.5K / ~1.7K env/s at 1K/10K/50K — 10x envelopes cost ~120x time), +sinking the 50K row (~29 s per cold reduce, the documented xfail) and leaving +10K only ~25% over budget. Agent K's subquadratic rewrite +(threshold/frontier readiness, byte-identical order — see +`epiphany-ops/DECISIONS.md`) triggered the gate's XPASS promotion notice, and +the 50K row was flipped to Pass in the same change (**F surfaces, K fixes**, +round-tripped). Any future red row is a fresh regression: fix the reducer, do +not re-mark rows xfail without a written decision here. + +**Two honesty notes on the bundle rows.** (1) They run on the build target's +filesystem, *not* `std::env::temp_dir()`: `/tmp` is commonly tmpfs, where fsync +is a near-no-op — measured, the commit row is ~54 µs on tmpfs vs ~14.7 ms on a +real NVMe filesystem, a 270x difference that would have made the 50 ms budget +vacuous. (2) The read budget is spec'd against "a 100-page orchestral score"; +no such corpus generator exists yet, so the row is a moderate-corpus (1K +envelopes + canonical-base snapshot) stand-in with margin to spare, and says so +in its docs. + +**Scope, and the budget rows deliberately not built yet.** F1 covers the three +budgets that are measurable against shipped subsystems: the reduction rate and +the two file-format budgets. The remaining Chapter 10 rows are blocked on +unimplemented or in-flight subsystems and become future benches in this same +`benches/` + gate shape: the interactive keystroke→frame budgets (blocked on +Agent I's engrave/render pipeline maturing past the visible slice), cold +solving of a 100-page score within 2 s p99 and the incremental-propagation +bound (blocked on the real Chapter 9 solver and a 100-page corpus generator — +the latter also unblocks the honest read-row corpus), and the memory ceilings +(blocked on the same corpus). CI runs the gates in the `conformance` job with +`EPIPHANY_BENCH_QUICK=1` (50K skipped) and in the nightly `soak` job in full. + ## F3 — The representative score corpus + eligibility-taxonomy harness (Agent H) *Worklist item F3 — "the most underbuilt dependency; unblocks H entirely."* diff --git a/crates/epiphany-testkit/README.md b/crates/epiphany-testkit/README.md index 02e67a0..f469731 100644 --- a/crates/epiphany-testkit/README.md +++ b/crates/epiphany-testkit/README.md @@ -147,6 +147,40 @@ Per the QUICKSTART, implementation-discovered gaps are batched, not improvised: remaining layout-specific Pass 11 candidates (the `OperationKindTag` variant set and the layout-object id derivation). +## Performance benches (Chapter 10 budgets, worklist F1) + +`benches/` holds the criterion benches for the spec's measurable Chapter 10 +budgets (see `DECISIONS.md` F0 for why they live in this crate, F1 for every +call made). Criterion measures; the **budget gate** (`src/budget.rs`) asserts: +each bench's `main()` ends by re-timing every budget row and exiting nonzero if +a `Pass`-marked row misses its threshold. Known-pending rows are marked +`Xfail(reason)` *in the bench source* next to the numeric budget — a miss is +reported and tolerated, and a pass prints a loud promotion notice so stale +markings cannot linger. This is the "F surfaces, K fixes" handshake, and its +inaugural round has completed: the bench documented the reducer's O(n²) +`canonical_reduction_order` failure at scale, and Agent K's subquadratic +rewrite (see `epiphany-ops/DECISIONS.md`) flipped the xfail row to `Pass`. + +| row | budget (spec Chapter 10) | expectation | +|-----|--------------------------|-------------| +| `reduction/1000` | > 10,000 envelopes/s, cold | Pass (~674K env/s measured) | +| `reduction/10000` | > 10,000 envelopes/s, cold | Pass (~257K env/s measured) | +| `reduction/50000` | > 10,000 envelopes/s, cold | Pass (~87K env/s measured; promoted from Xfail by Agent K's reducer fix — was ~1.7K env/s) | +| `bundle/typical_edit_commit` | ≤ 50 ms (append + manifest + superblock flip, fsync'd) | Pass (~15 ms) | +| `bundle/open_bootstrap_read` | ≤ 200 ms (manifest + bootstrap chunks) | Pass (moderate-corpus stand-in) | + +```sh +# Full run (includes the 50K cold-reduction point, ~0.6 s per iteration): +cargo bench -p epiphany-testkit + +# The reduced CI shape: smaller sampling, 50K point skipped (PR CI runs this): +EPIPHANY_BENCH_QUICK=1 cargo bench -p epiphany-testkit +``` + +The gate is a calibrated median over a few iterations, deliberately not the +spec's p99-over-1000-iterations conformance methodology (that is the reference +suite's job; the deviation is documented in `src/budget.rs`). + ## Running ```sh diff --git a/crates/epiphany-testkit/benches/bundle.rs b/crates/epiphany-testkit/benches/bundle.rs new file mode 100644 index 0000000..80266b3 --- /dev/null +++ b/crates/epiphany-testkit/benches/bundle.rs @@ -0,0 +1,284 @@ +//! The Chapter 10 file-format budgets: typical-edit bundle write and +//! manifest+bootstrap read (Phase 2 worklist F1). +//! +//! The normative budgets (`spec/core_spec.tex`, Chapter 10 §"File Format +//! Performance"): +//! +//! > Bundle write of a typical edit (one or more operation envelopes appended +//! > to the operation-envelope block stream, manifest rewrite, superblock flip) +//! > completes within 50 ms at p99 on the reference hardware profile. +//! > +//! > Bundle read of the manifest and bootstrap chunks (sufficient for first +//! > interactive frame) completes within 200 ms at p99 for a 100-page +//! > orchestral score. +//! +//! Both rows run against a real on-disk bundle (`FileStore`, whose flush is a +//! genuine `fsync`) in a scratch directory under the build's **target dir** — +//! deliberately not `std::env::temp_dir()`, which is commonly tmpfs on Linux, +//! where fsync is a near-no-op and the commit budget would be measured against +//! RAM. The corpus is moderate: 1,000 generated operation envelopes packed +//! into operation blocks plus a canonical `MaterializedState` snapshot wired +//! as the manifest's `canonical_base`. Both +//! are expected to **Pass** today, so a regression fails `cargo bench` loudly. +//! The read row's honesty note: the spec sizes its 200 ms against a 100-page +//! orchestral score; no such corpus generator exists yet, so this row is the +//! moderate-corpus stand-in (recorded in `DECISIONS.md` F1) and the budget is +//! asserted with margin to spare. The OS page cache is warm across iterations +//! (dropping it needs privileges); each iteration's *process-level* state is +//! fresh. +//! +//! The timed sections: +//! +//! * **typical_edit_commit** — on an opened bundle restored to the same base +//! image (restore + open are un-timed setup): one `commit` staging a small +//! envelope block, i.e. block append + manifest rewrite + superblock flip, +//! every write fsync'd. This mirrors `bundle_harness`'s commit driver. +//! * **open_bootstrap_read** — `FileStore::open` + `Bundle::open` (superblock +//! selection + header + manifest decode) + reading the `canonical_base` +//! snapshot and every operation block — the bytes a first interactive frame +//! needs. +//! +//! Criterion measures; the budget gate in `main` asserts (see +//! `epiphany_testkit::budget` for the Pass/Xfail semantics and the documented +//! deviation from Chapter 10's p99-over-1000-iterations conformance +//! methodology). Run: `cargo bench -p epiphany-testkit --bench bundle`; +//! `EPIPHANY_BENCH_QUICK=1` shrinks sampling for PR CI. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use criterion::{BatchSize, Criterion}; +use epiphany_bundle::{ + pack_operation_blocks, Bundle, ChunkKind, CommitContext, DocumentId, FileStore, FileUuid, + FrontierBytes, Manifest, MemStore, ProfileId, ReductionAlgorithmVersion, SchemaVersion, + SnapshotId, SnapshotRef, StagedChunk, +}; +use epiphany_determinism::CanonicalEncode; +use epiphany_ops::{OperationEnvelope, OperationSet}; +use epiphany_testkit::budget::{self, Expectation}; +use epiphany_testkit::{generators, Rng}; + +/// Chapter 10: a typical-edit bundle write completes within 50 ms (p99). +const COMMIT_BUDGET: Duration = Duration::from_millis(50); +/// Chapter 10: manifest + bootstrap read completes within 200 ms (p99). +const READ_BUDGET: Duration = Duration::from_millis(200); + +/// Base-corpus scale: the criterion-5 envelope count, packed into real blocks. +const BASE_ENVELOPES: usize = 1_000; +/// The typical edit: a handful of envelopes appended as one block. +const EDIT_ENVELOPES: usize = 4; + +/// Everything the two rows measure against, built once from a fixed seed. +struct Fixture { + /// The committed base image (blocks + canonical-base snapshot). + base_image: Vec, + /// The typical edit, staged (one small operation block). + edit: Vec, + /// Temp-dir file paths: one per row so commit growth never skews reads. + commit_path: PathBuf, + read_path: PathBuf, +} + +/// The commit-context closure the harness uses: append the new chunks to the +/// previous manifest's `operation_roots`. +fn append_roots(ctx: &CommitContext) -> Manifest { + let mut manifest = ctx.previous_manifest.clone(); + manifest + .operation_roots + .extend(ctx.new_chunks.iter().copied()); + manifest +} + +fn staged_blocks(envelopes: &[OperationEnvelope]) -> Vec { + let payloads: Vec> = envelopes.iter().map(|e| e.to_canonical_bytes()).collect(); + pack_operation_blocks(&payloads) + .into_iter() + .map(StagedChunk::operation_block) + .collect() +} + +/// Builds the moderate base corpus: 1,000 envelopes committed as operation +/// blocks, then their cold reduction committed as a `Snapshot` chunk wired to +/// the manifest's `canonical_base` (the roundtrip harness's snapshot shape). +fn build_fixture(dir: &Path) -> Fixture { + let mut rng = Rng::new(0x00F1_B0DE_0001); + let envelopes = generators::operation_envelopes(&mut rng, BASE_ENVELOPES, 3, 40, 40); + let edit_envelopes = generators::operation_envelopes(&mut rng, EDIT_ENVELOPES, 3, 8, 8); + + let uuid = FileUuid(rng.array16()); + let doc = DocumentId(rng.array16()); + let mut bundle = + Bundle::create(MemStore::new(), uuid, Manifest::empty(doc)).expect("create base bundle"); + bundle + .commit(&staged_blocks(&envelopes), append_roots) + .expect("commit base operation blocks"); + + // The canonical base: the corpus's cold reduction, stored as a snapshot. + let mut set = OperationSet::new(); + set.accept_all(envelopes.iter().cloned()); + let canonical = set.reduce().canonical_bytes(); + let snapshot = StagedChunk { + kind: ChunkKind::Snapshot, + schema_version: SchemaVersion::V0, + payload: canonical, + }; + let frontier = generators::frontier_bytes(&envelopes); + bundle + .commit(&[snapshot], |ctx| { + let mut manifest = ctx.previous_manifest.clone(); + let root = ctx.new_chunks[0]; + let mut sid = [0u8; 16]; + sid.copy_from_slice(&root.hash.as_bytes()[..16]); + manifest.canonical_base = Some(SnapshotRef { + snapshot_id: SnapshotId(sid), + covers_causal_frontier: FrontierBytes::from_bytes(frontier.clone()), + reduction_algorithm_version: ReductionAlgorithmVersion(0), + profile_id: ProfileId::Full, + hash: root.hash, + root, + }); + manifest + }) + .expect("commit canonical-base snapshot"); + + Fixture { + base_image: bundle.into_store().into_bytes(), + edit: staged_blocks(&edit_envelopes), + commit_path: dir.join("commit.epb"), + read_path: dir.join("read.epb"), + } +} + +/// Un-timed setup for the commit row: restore the base image and open it. +fn restore_and_open(path: &Path, image: &[u8]) -> Bundle { + fs::write(path, image).expect("restore base image"); + Bundle::open(FileStore::open(path).expect("open store")).expect("open bundle") +} + +/// The timed commit: block append + manifest rewrite + superblock flip, fsync'd. +fn typical_edit_commit(mut bundle: Bundle, edit: &[StagedChunk]) -> u64 { + bundle + .commit(edit, append_roots) + .expect("typical-edit commit"); + bundle.generation() +} + +/// The timed read: open (superblock selection + manifest decode) + the +/// bootstrap chunks — canonical-base snapshot and every operation block. +fn open_bootstrap_read(path: &Path) -> usize { + let bundle = Bundle::open(FileStore::open(path).expect("open store")).expect("open bundle"); + let manifest = bundle.manifest(); + let mut bytes = 0usize; + let base = manifest + .canonical_base + .as_ref() + .expect("the fixture wires a canonical base"); + bytes += bundle + .read_chunk(&base.root) + .expect("snapshot chunk reads") + .len(); + for root in &manifest.operation_roots { + for envelope in bundle + .read_operation_block(root) + .expect("operation block reads") + { + bytes += envelope.len(); + } + } + bytes +} + +/// The criterion measurement side. +fn criterion_measurements(criterion: &mut Criterion, fixture: &Fixture, quick: bool) { + let mut group = criterion.benchmark_group("bundle"); + group.sample_size(if quick { 10 } else { 30 }); + group.measurement_time(Duration::from_secs(if quick { 1 } else { 4 })); + group.warm_up_time(Duration::from_millis(if quick { 300 } else { 1000 })); + + group.bench_function("typical_edit_commit", |b| { + b.iter_batched( + || restore_and_open(&fixture.commit_path, &fixture.base_image), + |bundle| typical_edit_commit(bundle, &fixture.edit), + BatchSize::PerIteration, + ) + }); + + fs::write(&fixture.read_path, &fixture.base_image).expect("write read-row image"); + group.bench_function("open_bootstrap_read", |b| { + b.iter(|| open_bootstrap_read(&fixture.read_path)) + }); + + group.finish(); +} + +/// The budget-gate side: both rows are expected to **Pass** today. +fn budget_gate(fixture: &Fixture, quick: bool) -> Vec { + let commit_median = budget::median_time( + if quick { 12 } else { 40 }, + || restore_and_open(&fixture.commit_path, &fixture.base_image), + |bundle| typical_edit_commit(bundle, &fixture.edit), + ); + fs::write(&fixture.read_path, &fixture.base_image).expect("write read-row image"); + let read_median = budget::median_time( + if quick { 8 } else { 25 }, + || (), + |()| open_bootstrap_read(&fixture.read_path), + ); + vec![ + budget::latency_gate( + "bundle/typical_edit_commit", + commit_median, + COMMIT_BUDGET, + Expectation::Pass, + ), + budget::latency_gate( + "bundle/open_bootstrap_read", + read_median, + READ_BUDGET, + Expectation::Pass, + ), + ] +} + +/// A scratch directory on the **build target's filesystem** (real disk), not +/// `temp_dir()`: `/tmp` is commonly tmpfs, where the commit row's fsyncs would +/// be free and the 50 ms budget vacuous. Honors `CARGO_TARGET_DIR`. +fn scratch_dir() -> PathBuf { + let target = std::env::var_os("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") + .join("target") + }); + target.join(format!("f1-bundle-bench-{}", std::process::id())) +} + +fn main() { + // `cargo bench` passes `--bench`; its absence means criterion's test mode + // (`cargo test --benches` / `--all-targets`): run each measurement once, + // skip the gate. + let bench_mode = std::env::args().any(|arg| arg == "--bench"); + let quick = budget::quick_mode(); + + let dir = scratch_dir(); + fs::create_dir_all(&dir).expect("create bench scratch dir"); + let fixture = build_fixture(&dir); + + let mut criterion = Criterion::default().configure_from_args(); + criterion_measurements(&mut criterion, &fixture, quick); + criterion.final_summary(); + + let holds = if bench_mode { + budget::verdict(&budget_gate(&fixture, quick)) + } else { + true + }; + + let _ = fs::remove_dir_all(&dir); + if !holds { + std::process::exit(1); + } +} diff --git a/crates/epiphany-testkit/benches/reduction.rs b/crates/epiphany-testkit/benches/reduction.rs new file mode 100644 index 0000000..749a554 --- /dev/null +++ b/crates/epiphany-testkit/benches/reduction.rs @@ -0,0 +1,195 @@ +//! The Chapter 10 operation-envelope reduction-rate bench + budget gate +//! (Phase 2 worklist F1). +//! +//! The normative budget (`spec/core_spec.tex`, Chapter 10 §"File Format +//! Performance"): +//! +//! > Operation-envelope reduction rate MUST exceed 10,000 envelopes per second +//! > on the reference hardware profile, measured during cold reduction from a +//! > fresh canonical base. +//! +//! Three documented scale points: **1K** (the acceptance suite's criterion-5 +//! scale), **10K**, and **50K** envelopes. The timed section is the cold +//! reduction an opener performs — `OperationSet::new()` + `accept_all` + +//! `reduce()` on a fresh set — with the envelope vector generated once per +//! scale point from a fixed seed and *cloned outside* the timed section. +//! +//! Criterion measures (throughput in envelopes/s); the budget gate in `main` +//! asserts, with the budget table below (`epiphany_testkit::budget` explains +//! the Pass/Xfail semantics and the deliberate deviation from Chapter 10's +//! p99-over-1000-iterations conformance methodology). The defect this bench +//! was written to surface — `canonical_reduction_order`'s literal O(n²) +//! double-loop indegree construction plus its per-emission full ready-scan +//! (`crates/epiphany-ops/src/reduce.rs`) — sank the 50K point decisively +//! (~1.7K env/s ≈ 29 s per cold reduce on the dev profile) and left the 10K +//! point clearing the budget with only ~25% margin. Agent K's subquadratic +//! rewrite (threshold/frontier readiness; see the epiphany-ops DECISIONS +//! entry) closed the loop — **F surfaces, K fixes** +//! (`spec/PHASE2_F_WEEK0_WORKLIST.md` F1) — and the gate's XPASS notice +//! promoted the 50K row to `Pass` (~87K env/s measured, ~8.7x budget). +//! +//! Run: `cargo bench -p epiphany-testkit --bench reduction`. Set +//! `EPIPHANY_BENCH_QUICK=1` for the reduced PR-CI shape (smaller sampling, 50K +//! point skipped). Under `cargo test --benches` criterion runs each measurement +//! once in test mode and the gate is skipped — the gate belongs to `cargo +//! bench`. + +use std::time::Duration; + +use criterion::{BatchSize, BenchmarkId, Criterion, SamplingMode, Throughput}; +use epiphany_ops::{MaterializedState, OperationEnvelope, OperationSet}; +use epiphany_testkit::budget::{self, Expectation}; +use epiphany_testkit::{generators, Rng}; + +/// Chapter 10: the reduction rate MUST exceed 10,000 envelopes per second. +const RATE_BUDGET_ENV_PER_SEC: f64 = 10_000.0; + +/// One documented scale point of THE BUDGET TABLE below. +struct ScalePoint { + /// Envelope count (also the criterion throughput element count). + n_ops: usize, + /// Fixed generator seed — bench inputs are reproducible byte-for-byte. + seed: u64, + /// The documented expectation against `RATE_BUDGET_ENV_PER_SEC`. + expectation: Expectation, + /// Budget-gate timed iterations (full mode, quick mode); `0` skips the + /// row in that mode (printed as an explicit skip, never silent). + gate_iters: (usize, usize), + /// Criterion measurement time (full mode), or `None` to leave the point + /// gate-only (the 50K point: it predates the reducer fix, when a single + /// cold reduction was minute-scale; the budget gate's median covers it). + criterion_time: Option, +} + +/// THE BUDGET TABLE (worklist F1). Budget: > 10,000 envelopes/second, cold. +/// +/// | envelopes | expectation | measured (dev profile, 2026-07, post-K-fix) | why | +/// |-----------|-------------|----------------------------------------------|-----| +/// | 1,000 | Pass | ~674,000 env/s (~1.5 ms) | criterion-5 scale; ~67x margin | +/// | 10,000 | Pass | ~257,000 env/s (~39 ms) | ~26x margin | +/// | 50,000 | Pass | ~87,000 env/s (~0.58 s) | promoted from Xfail by Agent K's subquadratic reducer; full/nightly runs only | +/// +/// Pre-fix (the numbers the F1 xfail table documented): ~155K / ~12.5K / +/// ~1.7K env/s — the O(n²) `canonical_reduction_order` indegree construction, +/// which sank 50K (~29 s per cold reduce) and left 10K only ~25% over budget. +/// Agent K's threshold/frontier rewrite (see the epiphany-ops DECISIONS +/// entry) is byte-identical in order and subquadratic; the gate's XPASS +/// notice triggered the 50K promotion recorded here. +/// +/// If any row starts missing the budget again, that is a fresh regression: +/// fix the reducer, do not re-mark rows Xfail without a written decision. +const SCALE_POINTS: &[ScalePoint] = &[ + ScalePoint { + n_ops: 1_000, + seed: 0x00F1_5EED_0001, + expectation: Expectation::Pass, + gate_iters: (9, 5), + criterion_time: Some(Duration::from_secs(6)), + }, + ScalePoint { + n_ops: 10_000, + seed: 0x00F1_5EED_0002, + expectation: Expectation::Pass, + gate_iters: (3, 2), + criterion_time: Some(Duration::from_secs(20)), + }, + ScalePoint { + n_ops: 50_000, + seed: 0x00F1_5EED_0003, + expectation: Expectation::Pass, + gate_iters: (1, 0), + criterion_time: None, + }, +]; + +/// The scale point's reproducible envelope set — the criterion-5 session shape +/// (3 replicas, 40 events, 40 pitches) at `n_ops` envelopes. +fn envelopes_at(point: &ScalePoint) -> Vec { + let mut rng = Rng::new(point.seed); + generators::operation_envelopes(&mut rng, point.n_ops, 3, 40, 40) +} + +/// The timed section: cold reduction from a fresh canonical base — build a +/// fresh set (acceptance) and reduce it. The envelope clone happens in the +/// caller's un-timed setup. +fn cold_reduce(envelopes: Vec) -> MaterializedState { + let mut set = OperationSet::new(); + set.accept_all(envelopes); + set.reduce() +} + +/// The criterion measurement side (envelopes/s via `Throughput::Elements`). +fn criterion_measurements(criterion: &mut Criterion, quick: bool) { + let mut group = criterion.benchmark_group("reduction_cold"); + // Cold multi-second iterations at 10K: flat sampling, the minimum sample + // count, and per-point measurement times keep `cargo bench` wall-clock sane. + group.sampling_mode(SamplingMode::Flat); + group.sample_size(10); + for point in SCALE_POINTS { + let Some(time) = point.criterion_time else { + continue; // 50K is gate-only; see THE BUDGET TABLE. + }; + if quick && point.n_ops > 1_000 { + continue; // quick mode: the gate still measures 10K, cheaply. + } + let envelopes = envelopes_at(point); + group.throughput(Throughput::Elements(point.n_ops as u64)); + group.measurement_time(if quick { Duration::from_secs(2) } else { time }); + group.warm_up_time(Duration::from_millis(if quick { 500 } else { 1500 })); + group.bench_with_input( + BenchmarkId::from_parameter(point.n_ops), + &envelopes, + |b, envs| b.iter_batched(|| envs.clone(), cold_reduce, BatchSize::PerIteration), + ); + } + group.finish(); +} + +/// The budget-gate side: evaluates THE BUDGET TABLE and returns the rows. +fn budget_gate(quick: bool) -> Vec { + let mut reports = Vec::new(); + for point in SCALE_POINTS { + let iters = if quick { + point.gate_iters.1 + } else { + point.gate_iters.0 + }; + if iters == 0 { + println!( + "skip reduction/{}: heaviest scale point; full/nightly runs only \ + (unset EPIPHANY_BENCH_QUICK)", + point.n_ops + ); + continue; + } + let envelopes = envelopes_at(point); + let median = budget::median_time(iters, || envelopes.clone(), cold_reduce); + reports.push(budget::rate_gate( + format!("reduction/{}", point.n_ops), + point.n_ops as u64, + median, + RATE_BUDGET_ENV_PER_SEC, + point.expectation, + )); + } + reports +} + +fn main() { + // `cargo bench` passes `--bench`; its absence means criterion's test mode + // (`cargo test --benches` / `--all-targets`): run each measurement once, + // skip the gate. + let bench_mode = std::env::args().any(|arg| arg == "--bench"); + let quick = budget::quick_mode(); + + let mut criterion = Criterion::default().configure_from_args(); + criterion_measurements(&mut criterion, quick); + criterion.final_summary(); + + if !bench_mode { + return; + } + if !budget::verdict(&budget_gate(quick)) { + std::process::exit(1); + } +} diff --git a/crates/epiphany-testkit/src/budget.rs b/crates/epiphany-testkit/src/budget.rs new file mode 100644 index 0000000..525f69d --- /dev/null +++ b/crates/epiphany-testkit/src/budget.rs @@ -0,0 +1,293 @@ +//! The Chapter 10 performance-budget gate (Phase 2 worklist F1). +//! +//! The benches in `benches/` (home per `DECISIONS.md` F0) *measure* with +//! criterion; criterion never *asserts*, so each bench binary's `main()` ends +//! by running these gates: a calibrated timing check per budget row, with the +//! numeric threshold written at the call site in the bench source. Every row +//! carries an [`Expectation`]: +//! +//! * [`Expectation::Pass`] — the budget must hold **today**; a miss fails the +//! bench run with a nonzero exit (the CI tripwire). +//! * [`Expectation::Xfail`] — a documented, known-pending miss whose reason +//! names the defect and its owner. A miss prints an expected-failure line +//! and does **not** fail the run; a *pass* prints a promotion notice, +//! because the marking is then stale and must be flipped to `Pass`. +//! +//! This is the "F surfaces, K fixes" handshake (`spec/PHASE2_F_WEEK0_WORKLIST.md` +//! F1): a known-pending scale point stays `Xfail` — budget written in the +//! bench — until the named defect is fixed, at which point the gate itself +//! reports that the row should be promoted. The inaugural round completed: the +//! reducer's `O(n²)` `canonical_reduction_order` sank `reduction/50000` until +//! Agent K's subquadratic rewrite, whose XPASS notice promoted the row. +//! +//! ## Methodology note (a deliberate deviation from Chapter 10) +//! +//! Chapter 10's conformance methodology is **p99 over ≥ 1000 iterations per +//! scenario** on the reference hardware profile. That is the reference suite's +//! job, not this gate's: 1000 iterations of a minute-long 50K-envelope cold +//! reduction would be unusable in CI. The gate instead takes the **median of a +//! small, per-row calibrated iteration count** in a release build — enough to +//! reject flukes while keeping `cargo bench` wall-clock sane. Runs are cold +//! (no in-gate warm-up): the marquee budget is an explicitly *cold* reduction +//! rate, and the criterion measurements that precede the gate have already +//! warmed the allocator and caches for the warm-appropriate rows. Conformance +//! *claims* still require the full Chapter 10 methodology. + +use std::time::{Duration, Instant}; + +/// How a budget row is expected to behave on the current implementation. +#[derive(Copy, Clone, Debug)] +pub enum Expectation { + /// The budget must hold; a miss fails the bench run (nonzero exit). + Pass, + /// A documented known-pending miss; the string names the defect and who + /// fixes it. A miss is reported but tolerated; a pass demands promotion. + Xfail(&'static str), +} + +/// One evaluated budget row. +#[derive(Debug)] +pub struct GateReport { + /// The row label, e.g. `reduction/10000`. + pub label: String, + /// Human-readable `measured vs budget` detail. + pub detail: String, + /// Whether the measurement met the budget. + pub met_budget: bool, + /// The row's documented expectation. + pub expectation: Expectation, +} + +impl GateReport { + /// A `Pass`-marked row that missed its budget — the only outcome that + /// fails the bench run. + pub fn unexpected_failure(&self) -> bool { + !self.met_budget && matches!(self.expectation, Expectation::Pass) + } + + /// An `Xfail`-marked row that met its budget: the marking is stale and the + /// row should be promoted to `Pass`. + pub fn unexpected_pass(&self) -> bool { + self.met_budget && matches!(self.expectation, Expectation::Xfail(_)) + } + + /// The verdict line printed for this row. + pub fn line(&self) -> String { + match (self.met_budget, self.expectation) { + (true, Expectation::Pass) => format!("PASS {}: {}", self.label, self.detail), + (false, Expectation::Pass) => format!( + "FAIL {}: {} — budget missed on a Pass-marked row", + self.label, self.detail + ), + (false, Expectation::Xfail(reason)) => format!( + "XFAIL {}: {} — expected failure: {}", + self.label, self.detail, reason + ), + (true, Expectation::Xfail(reason)) => format!( + "XPASS {}: {} — met the budget despite the xfail marking ({}); \ + PROMOTE this row to Pass", + self.label, self.detail, reason + ), + } + } +} + +/// The median over `iters` timed runs of `op`, with per-run input built by +/// `setup` **outside** the timed section (this is how a cold-reduction row +/// clones its envelope vector without the clone being charged to the budget). +/// The output is dropped outside the timed section too. `iters ≥ 1`. +pub fn median_time( + iters: usize, + mut setup: impl FnMut() -> S, + mut op: impl FnMut(S) -> T, +) -> Duration { + assert!(iters >= 1, "a gate row needs at least one timed iteration"); + let mut samples = Vec::with_capacity(iters); + for _ in 0..iters { + let input = setup(); + let start = Instant::now(); + let out = op(input); + let elapsed = start.elapsed(); + std::hint::black_box(&out); + samples.push(elapsed); + drop(out); + } + samples.sort(); + samples[samples.len() / 2] +} + +/// Evaluates a throughput budget: `elements` per timed run must exceed +/// `budget_per_sec` (the Chapter 10 reduction-rate form, "MUST exceed"). +pub fn rate_gate( + label: impl Into, + elements: u64, + median: Duration, + budget_per_sec: f64, + expectation: Expectation, +) -> GateReport { + let rate = elements as f64 / median.as_secs_f64(); + GateReport { + label: label.into(), + detail: format!( + "{rate:.0} elements/s ({elements} elements, median {median:.2?}); \ + budget > {budget_per_sec:.0}/s" + ), + met_budget: rate > budget_per_sec, + expectation, + } +} + +/// Evaluates a latency budget: the median must come in at or under `budget` +/// (the Chapter 10 file-format form, "completes within"). +pub fn latency_gate( + label: impl Into, + median: Duration, + budget: Duration, + expectation: Expectation, +) -> GateReport { + GateReport { + label: label.into(), + detail: format!("median {median:.2?}; budget <= {budget:.0?}"), + met_budget: median <= budget, + expectation, + } +} + +/// Prints every row's verdict and returns whether the gate holds — i.e. no +/// `Pass`-marked row missed its budget. The bench binary exits nonzero when +/// this returns `false`; `Xfail` misses and `XPASS` promotions never fail the +/// run (the latter print a loud promotion notice instead). +pub fn verdict(reports: &[GateReport]) -> bool { + println!("\n== Chapter 10 budget gate (worklist F1) =="); + for report in reports { + println!("{}", report.line()); + } + let unexpected: Vec<&GateReport> = reports.iter().filter(|r| r.unexpected_failure()).collect(); + let promotions = reports.iter().filter(|r| r.unexpected_pass()).count(); + if promotions > 0 { + println!( + "note: {promotions} xfail row(s) met their budget — promote them to Pass \ + (the marking is stale)." + ); + } + if unexpected.is_empty() { + println!("budget gate: OK ({} row(s))", reports.len()); + true + } else { + println!( + "budget gate: FAILED — {} Pass-marked row(s) missed their budget", + unexpected.len() + ); + false + } +} + +/// Whether the CI-friendly quick mode is on (`EPIPHANY_BENCH_QUICK=1`): +/// reduced criterion sampling, reduced gate iteration counts, and the heaviest +/// scale points (the 50K-envelope reduction row) skipped entirely. PR CI sets +/// it; the nightly soak and local full runs leave it unset. +pub fn quick_mode() -> bool { + std::env::var("EPIPHANY_BENCH_QUICK").is_ok_and(|v| !v.is_empty() && v != "0") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn row(met_budget: bool, expectation: Expectation) -> GateReport { + GateReport { + label: "test/row".to_owned(), + detail: "detail".to_owned(), + met_budget, + expectation, + } + } + + #[test] + fn pass_row_meeting_budget_holds() { + let r = row(true, Expectation::Pass); + assert!(!r.unexpected_failure()); + assert!(!r.unexpected_pass()); + assert!(verdict(&[r])); + } + + #[test] + fn pass_row_missing_budget_fails_the_gate() { + let r = row(false, Expectation::Pass); + assert!(r.unexpected_failure()); + assert!(r.line().starts_with("FAIL")); + assert!(!verdict(&[r])); + } + + #[test] + fn xfail_row_missing_budget_is_tolerated() { + let r = row(false, Expectation::Xfail("documented defect")); + assert!(!r.unexpected_failure()); + assert!(r.line().starts_with("XFAIL")); + assert!(r.line().contains("documented defect")); + assert!(verdict(&[r])); + } + + #[test] + fn xfail_row_meeting_budget_demands_promotion_but_holds() { + let r = row(true, Expectation::Xfail("documented defect")); + assert!(r.unexpected_pass()); + assert!(r.line().starts_with("XPASS")); + assert!(r.line().contains("PROMOTE")); + assert!(verdict(&[r])); + } + + #[test] + fn gates_evaluate_their_thresholds() { + // 100 elements in 1 ms = 100,000/s. + let fast = rate_gate( + "rate/fast", + 100, + Duration::from_millis(1), + 10_000.0, + Expectation::Pass, + ); + assert!(fast.met_budget); + let slow = rate_gate( + "rate/slow", + 100, + Duration::from_millis(100), + 10_000.0, + Expectation::Pass, + ); + assert!(!slow.met_budget); + + let ok = latency_gate( + "lat/ok", + Duration::from_millis(10), + Duration::from_millis(50), + Expectation::Pass, + ); + assert!(ok.met_budget); + let over = latency_gate( + "lat/over", + Duration::from_millis(60), + Duration::from_millis(50), + Expectation::Pass, + ); + assert!(!over.met_budget); + } + + #[test] + fn median_time_takes_the_middle_sample() { + // Deterministic ordering check via a controlled op: the median of an + // odd sample count must be a real observed sample, not an average. + let mut calls = 0u32; + let d = median_time( + 5, + || (), + |()| { + calls += 1; + }, + ); + assert_eq!(calls, 5); + // No timing assertion (flaky); the structural property is that a + // duration was produced at all and the closure ran `iters` times. + let _ = d; + } +} diff --git a/crates/epiphany-testkit/src/corpus.rs b/crates/epiphany-testkit/src/corpus.rs index 2a21bb6..d3a01aa 100644 --- a/crates/epiphany-testkit/src/corpus.rs +++ b/crates/epiphany-testkit/src/corpus.rs @@ -177,7 +177,7 @@ pub struct Fixture { pub tier: Tier, pub build: fn() -> Score, /// Buckets this fixture must drive non-zero (validated per-fixture, and - /// aggregated for corpus coverage). For the [`UNUSUAL_BUCKETS`] this list is + /// aggregated for corpus coverage). For the crate-private `UNUSUAL_BUCKETS` this list is /// also an *exact whitelist*: a fixture that lands an event in a /// loss/deferral bucket it did not declare fails the harness. pub expect: &'static [Bucket], diff --git a/crates/epiphany-testkit/src/lib.rs b/crates/epiphany-testkit/src/lib.rs index d9b5966..f670e20 100644 --- a/crates/epiphany-testkit/src/lib.rs +++ b/crates/epiphany-testkit/src/lib.rs @@ -92,6 +92,11 @@ pub mod rng; +// Phase 2, Agent F (worklist F1): the Chapter 10 performance-budget gate the +// `benches/` targets assert through (Pass / Xfail rows with thresholds written +// in the bench source; see `DECISIONS.md` F0/F1). +pub mod budget; + pub mod fixtures; pub mod generators; pub mod roundtrip; diff --git a/spec/PASS12_BATCH.md b/spec/PASS12_BATCH.md index 2f2e2e8..ea7e65d 100644 --- a/spec/PASS12_BATCH.md +++ b/spec/PASS12_BATCH.md @@ -52,10 +52,10 @@ code instead is the failure mode this batch exists to prevent. | P12-I4 | `epiphany-layout-ir` I | Constraint-strength attachment: Ch9 defines `ConstraintStrength` and says the solver consumes constraints "in normalized form", but neither the normalized form nor Ch7's `LayoutConstraint` provides a channel for an instance to carry strength. Implemented rule: break strength = `BreakKind` (Hard→Required, Soft→Preferred{1.0}); other core families Required; `Registered` conservative Required. Bless the rule or add a strength field. | G / Pass 12 (solver) | | P12-I5 | `epiphany-layout-ir` I | No renderable status exists for "constraints present but not evaluated": every renderable `SolveStatus` is documented as "all hard constraints satisfied", leaving a below-conformance passthrough solver no honest report. Implemented encoding: `SolvedWithWarnings` + `satisfied_hard_constraints == false` + a warning. Sanction it or define a non-evaluating-tier report shape. | G / Pass 12 (solver) | | P12-I6 | `epiphany-layout-ir` I | The spacing pass MUST "build collision constraints" but no per-tier minimum emission set is named. Implemented floor: successive-notehead-column no-collision chains + per-glyph region containment + user-break constraints. A normative Minimal-tier floor would make the acceptance surface testable. | G / Pass 12 (solver) | -| P12-D1 | `epiphany-bundle` D | Operation-index provisional encoding (block-refs + id-sorted entries with u32 block ordinal and u32 in-block offset; golden-locked) awaiting Binary Format companion ratification, together with: the offset's meaning (first content byte within the uncompressed block payload), a normative definition of "stale" (implemented: index block-set ≠ manifest `operation_roots` under full-`ChunkRef` equality), the one-slot-per-id invariant, the load-bearing property that the envelope encoding *leads* with the 16-byte OperationId, and whether the commit-time "grown significantly" SHOULD gets a threshold or stays implementation-defined. | J (Binary Format companion) | -| P12-E1 | `epiphany-layout-ir` E | Provisional canonical byte form for the `EditBarrier`/`BarrierScope`/`BarrierCondition` tree and the two `ExtensionDeclaration` blobs (`push_set` framing, u64 LE lengths; golden-locked). Ratify into the Binary Format companion. | J (Binary Format companion) | -| P12-E2 | `epiphany-layout-ir` E | The spec places no bound on `BarrierCondition` recursion; the decoder needs one against adversarial bytes. `MAX_CONDITION_DEPTH = 64` implemented — ratify a normative bound or bless the constant. | J (Binary Format companion) | -| P12-E3 | `epiphany-layout-ir` E | Barrier `ObjectKind` byte form = the `TypedObjectId` 16-bit discriminant (2 LE bytes) with open-value decode (unknown kinds never match, preserving append-only forward compat). Ratify representation + stance. | J (Binary Format companion) | +| ~~P12-D1~~ **RESOLVED (Binary Format 0.1.0 §7.6, `req:binfmt:opindex`)** | `epiphany-bundle` D | Operation-index provisional encoding (block-refs + id-sorted entries with u32 block ordinal and u32 in-block offset; golden-locked) awaiting Binary Format companion ratification, together with: the offset's meaning (first content byte within the uncompressed block payload), a normative definition of "stale" (implemented: index block-set ≠ manifest `operation_roots` under full-`ChunkRef` equality), the one-slot-per-id invariant, the load-bearing property that the envelope encoding *leads* with the 16-byte OperationId, and whether the commit-time "grown significantly" SHOULD gets a threshold or stays implementation-defined. Ratified as-implemented; refresh threshold pinned implementation-defined (open question retained in the companion). | ✅ done | +| ~~P12-E1~~ **RESOLVED (Binary Format 0.1.0 §8.1, `req:binfmt:ext-blobs`)** | `epiphany-layout-ir` E | Provisional canonical byte form for the `EditBarrier`/`BarrierScope`/`BarrierCondition` tree and the two `ExtensionDeclaration` blobs (`push_set` framing, u64 LE lengths; golden-locked). Ratified as-implemented. | ✅ done | +| ~~P12-E2~~ **RESOLVED (Binary Format 0.1.0 §8.2, `req:binfmt:condition-depth`)** | `epiphany-layout-ir` E | The spec places no bound on `BarrierCondition` recursion; the decoder needs one against adversarial bytes. `MAX_CONDITION_DEPTH = 64` implemented — the companion pins 64 as the normative bound (decoders MUST reject deeper; writers MUST NOT emit deeper). | ✅ done | +| ~~P12-E3~~ **RESOLVED (Binary Format 0.1.0 §8.1, `req:binfmt:object-kind-open`)** | `epiphany-layout-ir` E | Barrier `ObjectKind` byte form = the `TypedObjectId` 16-bit discriminant (2 LE bytes) with open-value decode (unknown kinds never match, preserving append-only forward compat). Representation and open-value stance ratified. | ✅ done | | P12-E4 | `epiphany-editor-core` E | Barrier matching for operations with no graph target (`SetMetadata`, `DeclareTransaction` — implemented: score-wide barriers only) and for opaque `Registered` operations (implemented: fully conservative match) is unspecified. | G (Ch. 8) | | P12-E5 | `epiphany-editor-core` E | The unsafe-edit tombstone MUST has no defined mechanism: the manifest-side form (drop declaration + preserved roots? an explicit tombstone record?), interaction with `required = true`, and whether crossing immediately deactivates the extension's remaining barriers (implemented: yes, recorded via `extensions_requiring_tombstone()` for the next bundle write). | G (Ch. 8) | @@ -65,7 +65,10 @@ Agent I (Track A) has contributed P12-I1..I6. Track B's Agent K has contributed P12-K1..K7; H has contributed P12-H1..H7 (H6/H7 from the 2026-07 spec-compliance audit follow-up, alongside K3/K4). The 2026-07 Push-3 wiring work added C1..C4 (re-anchoring), D1 (bundle operation index), and E1..E5 -(edit barriers). Agent J (Binary Format companion) has not yet contributed; -when it does, append rows — the batch is already open, so it joins directly -(no new threshold). Note the P12-D1/E1/E2/E3 rows are *inputs* to J's -companion rather than G-dispositions. +(edit barriers). Agent J's Binary Format companion now exists +(`spec/binary_format.tex`, v0.1.0): it ratified the P12-D1/E1/E2/E3 inputs +(struck through above) and discharged the crates' provisional-codec notes +(core P11-4, ops "provisional canonical encoding", bundle P11-D2/D4/D5). Its +three open questions (SnapshotId derivation, index-refresh threshold, u64/u32 +prefix unification at the next schema major) live in the companion itself, not +as batch rows. diff --git a/spec/binary_format.pdf b/spec/binary_format.pdf new file mode 100644 index 0000000..cb415b6 Binary files /dev/null and b/spec/binary_format.pdf differ diff --git a/spec/binary_format.tex b/spec/binary_format.tex new file mode 100644 index 0000000..0aec1d4 --- /dev/null +++ b/spec/binary_format.tex @@ -0,0 +1,2525 @@ +% !TEX program = xelatex +% +% Epiphany --- Binary Format (companion specification) +% Companion to the Core Specification. Compile with XeLaTeX. +% +% This document is versioned independently of the Core Specification +% (independent semver; see the Versioning note in the front matter). Its preamble +% is intentionally a self-contained copy of the core specification's preamble so +% the two documents build independently; factoring a shared preamble file is a +% later cleanup, not a v0.1 deliverable. + +\documentclass[11pt,letterpaper]{report} + +% --------------------------------------------------------------------------- +% Packages +% --------------------------------------------------------------------------- +\usepackage{fontspec} +\usepackage{geometry} +\geometry{ + letterpaper, + top=1.05in, + bottom=1.05in, + left=1.15in, + right=1.15in, + headheight=15pt +} + +\usepackage[english]{babel} +\usepackage{microtype} +\usepackage{parskip} +\usepackage{xcolor} +\usepackage{hyperref} +\usepackage{enumitem} +\usepackage{titlesec} +\usepackage{fancyhdr} +\usepackage{booktabs} +\usepackage{array} +\usepackage{longtable} +\usepackage{listings} +\usepackage{amsmath} +\usepackage{amssymb} +\usepackage{tcolorbox} +\tcbuselibrary{breakable, skins} + +% --------------------------------------------------------------------------- +% Color palette (shared with the core specification) +% --------------------------------------------------------------------------- +\definecolor{epiphanyteal}{HTML}{1A4044} +\definecolor{epiphanygold}{HTML}{8E6E2E} +\definecolor{epiphanyink}{HTML}{1F1B16} +\definecolor{epiphanyslate}{HTML}{6B6660} +\definecolor{epiphanycream}{HTML}{F8F4ED} +\definecolor{epiphanymist}{HTML}{ECE8E0} +\definecolor{epiphanycode}{HTML}{2A2520} +\definecolor{epiphanycrimson}{HTML}{7A2424} + +\hypersetup{ + colorlinks=true, + linkcolor=epiphanyteal, + citecolor=epiphanyteal, + urlcolor=epiphanygold, + pdftitle={Epiphany --- Binary Format}, + pdfauthor={The Epiphany Project}, + pdfsubject={Binary Format companion for the Epiphany music notation platform}, + pdfkeywords={music notation, binary format, serialization, wire format, canonical encoding}, + bookmarksnumbered=true, + bookmarksopen=true +} + +% --------------------------------------------------------------------------- +% Typography (shared with the core specification) +% --------------------------------------------------------------------------- +\setmainfont{TeX Gyre Pagella}[Numbers={OldStyle, Proportional}, Ligatures={TeX, Common}] +\setsansfont{TeX Gyre Heros}[Scale=0.94, Ligatures={TeX, Common}] +\setmonofont{TeX Gyre Cursor}[Scale=0.88, Ligatures={TeX}] +\newfontfamily\titlefont{TeX Gyre Pagella}[Numbers={OldStyle}, Ligatures={TeX, Common}] +\newcommand{\tablenums}[1]{{\addfontfeatures{Numbers={Lining,Tabular}}#1}} +\newcommand{\sectionsc}[1]{{\addfontfeatures{Letters=SmallCaps}#1}} + +% --------------------------------------------------------------------------- +% Section styling (shared with the core specification) +% --------------------------------------------------------------------------- +\titleformat{\chapter}[display] + {\normalfont\filright} + {\raggedright\color{epiphanygold}\fontsize{14pt}{16pt}\selectfont + \scshape Chapter\ \thechapter} + {16pt} + {\raggedright\color{epiphanyteal}\fontsize{32pt}{36pt}\selectfont\bfseries} + [\vspace{4pt}{\color{epiphanygold}\rule{2in}{0.6pt}}] +\titlespacing*{\chapter}{0pt}{-20pt}{30pt} +\titleformat{\section} + {\normalfont\Large\bfseries\color{epiphanyteal}} + {\color{epiphanygold}\thesection}{1em}{} +\titleformat{\subsection} + {\normalfont\large\bfseries\color{epiphanyteal}} + {\color{epiphanygold}\thesubsection}{1em}{} +\titleformat{\subsubsection} + {\normalfont\normalsize\bfseries\color{epiphanyink}} + {\thesubsubsection}{1em}{} + +% --------------------------------------------------------------------------- +% Headers and footers (shared with the core specification) +% --------------------------------------------------------------------------- +\pagestyle{fancy} +\fancyhf{} +\renewcommand{\headrulewidth}{0pt} +\renewcommand{\footrulewidth}{0pt} +\fancyhead[L]{\small\scshape\color{epiphanyslate}Epiphany --- Binary Format} +\fancyhead[R]{\small\itshape\color{epiphanyslate}\leftmark} +\fancyfoot[C]{\small\color{epiphanyslate}\thepage} +\renewcommand{\headrule}{ + \color{epiphanygold!50}\hrule width\headwidth height 0.4pt + \vspace{1pt} + \color{epiphanygold!30}\hrule width\headwidth height 0.2pt +} + +% --------------------------------------------------------------------------- +% Code listing style (shared with the core specification) +% --------------------------------------------------------------------------- +\lstdefinelanguage{Rust}{ + keywords={fn,let,mut,pub,struct,enum,impl,trait,for,in,if,else,match,return, + use,mod,crate,self,Self,as,where,move,async,await,const,static, + ref,type,unsafe,extern,dyn,box,break,continue,loop,while}, + keywordstyle=\color{epiphanyteal}\bfseries, + ndkeywords={i8,i16,i32,i64,i128,u8,u16,u32,u64,u128,f32,f64,bool,char,str, + String,Vec,Option,Result,Box,Rc,Arc,HashMap,BTreeMap, + NonZeroU16,NonZeroU32,NonZeroU64,Duration,Timestamp}, + ndkeywordstyle=\color{epiphanygold}\bfseries, + sensitive=true, + comment=[l]{//}, + morecomment=[s]{/*}{*/}, + commentstyle=\color{epiphanyslate}\itshape, + stringstyle=\color{epiphanycrimson}, + morestring=[b]", + morestring=[b]' +} +\lstset{ + basicstyle=\ttfamily\small\color{epiphanycode}, + backgroundcolor=\color{epiphanycream}, + frame=leftline, + rulecolor=\color{epiphanygold!60}, + framesep=8pt, + framerule=1.5pt, + xleftmargin=10pt, + xrightmargin=4pt, + breaklines=true, + showstringspaces=false, + numberstyle=\tiny\color{epiphanyslate}, + numbersep=10pt, + captionpos=b, + aboveskip=10pt, + belowskip=10pt, + language=Rust +} + +% --------------------------------------------------------------------------- +% Custom environments (shared with the core specification) +% --------------------------------------------------------------------------- +\newtcolorbox{openquestion}[1][]{ + enhanced, breakable, + colback=epiphanymist, colframe=epiphanycrimson, + fonttitle=\bfseries\color{white}, title={\scshape\hspace{2pt}Open Question}, + coltitle=white, colbacktitle=epiphanycrimson, + arc=1pt, boxrule=0pt, leftrule=2pt, + left=10pt, right=10pt, top=8pt, bottom=8pt, + attach boxed title to top left={xshift=0pt, yshift=0pt}, + boxed title style={arc=0pt, sharp corners, boxrule=0pt, left=6pt, right=8pt, top=2pt, bottom=2pt}, + #1 +} +\newtcolorbox{rationale}[1][]{ + enhanced, breakable, + colback=epiphanymist, colframe=epiphanyteal, + fonttitle=\bfseries\color{white}, title={\scshape\hspace{2pt}Rationale}, + coltitle=white, colbacktitle=epiphanyteal, + arc=1pt, boxrule=0pt, leftrule=2pt, + left=10pt, right=10pt, top=8pt, bottom=8pt, + attach boxed title to top left={xshift=0pt, yshift=0pt}, + boxed title style={arc=0pt, sharp corners, boxrule=0pt, left=6pt, right=8pt, top=2pt, bottom=2pt}, + #1 +} +\newtcolorbox{requirement}[1][]{ + enhanced, breakable, + colback=white, colframe=epiphanygold, + fonttitle=\bfseries\color{white}, title={\scshape\hspace{2pt}Requirement}, + coltitle=white, colbacktitle=epiphanygold, + arc=1pt, boxrule=0pt, leftrule=2pt, + left=10pt, right=10pt, top=8pt, bottom=8pt, + attach boxed title to top left={xshift=0pt, yshift=0pt}, + boxed title style={arc=0pt, sharp corners, boxrule=0pt, left=6pt, right=8pt, top=2pt, bottom=2pt}, + #1 +} +\newtcolorbox{nongoal}[1][]{ + enhanced, breakable, + colback=epiphanymist, colframe=epiphanyslate, + fonttitle=\bfseries\color{white}, title={\scshape\hspace{2pt}Non-Goal}, + coltitle=white, colbacktitle=epiphanyslate, + arc=1pt, boxrule=0pt, leftrule=2pt, + left=10pt, right=10pt, top=8pt, bottom=8pt, + attach boxed title to top left={xshift=0pt, yshift=0pt}, + boxed title style={arc=0pt, sharp corners, boxrule=0pt, left=6pt, right=8pt, top=2pt, bottom=2pt}, + #1 +} + +\newcommand{\MUST}{\textbf{MUST}} +\newcommand{\MUSTNOT}{\textbf{MUST}\nobreak\ \textbf{NOT}} +\newcommand{\SHOULD}{\textbf{SHOULD}} +\newcommand{\SHOULDNOT}{\textbf{SHOULD}\nobreak\ \textbf{NOT}} +\newcommand{\MAY}{\textbf{MAY}} + +% Byte concatenation within a wire layout. +\newcommand{\cat}{\ensuremath{\,\Vert\,}} +% A breakable underscore for long monospace anchor names. +\newcommand{\ub}{\_\allowbreak} +% Long monospace tokens (requirement names, test paths) cannot hyphenate; +% allow loose lines instead of overfull ones. +\emergencystretch=3em +\tolerance=2000 + +\setlist[itemize]{topsep=2pt, itemsep=3pt, parsep=0pt} +\setlist[enumerate]{topsep=2pt, itemsep=3pt, parsep=0pt} +\setlist[description]{topsep=2pt, itemsep=5pt, parsep=0pt} +\AtBeginDocument{\color{epiphanyink}} + +% --------------------------------------------------------------------------- +% Document +% --------------------------------------------------------------------------- +\begin{document} + +\begin{titlepage} + \thispagestyle{empty} + \centering + \vspace*{2.2in} + {\color{epiphanygold}\rule{3in}{0.8pt}}\\[18pt] + {\titlefont\fontsize{34pt}{38pt}\selectfont\color{epiphanyteal}\bfseries Epiphany}\\[10pt] + {\Large\scshape\color{epiphanyslate}Binary Format}\\[6pt] + {\large\itshape\color{epiphanyslate}A companion to the Core Specification}\\[14pt] + {\color{epiphanygold}\rule{3in}{0.8pt}}\\[24pt] + {\normalsize\color{epiphanyink}Version 0.1.0 --- Phase 2 (canonical wire format: primitives through bundle physical layout + K0 payload framing)}\\[4pt] + {\small\color{epiphanyslate}Normative for the byte layouts it defines} + \vfill +\end{titlepage} + +\tableofcontents + +% =========================================================================== +\chapter{About This Companion} +\label{ch:about} + +The \emph{Binary Format} document is a companion to the Epiphany Core +Specification. It fulfils the core specification's delegation in Chapter~8, +\sectionsc{Binary Format Companion} (the \texttt{sec:format:binary} section): +the byte-level encoding --- integer conventions, exact field widths, record +alignment, endianness, string encoding details, struct layouts --- is delivered +here. In the core specification's words: \emph{``The Binary Format document is +normative; an implementation cannot conform to the file format specification +without conforming to the Binary Format specification.''} + +This release (v0.1.0) delivers the full \textbf{K-independent} encoding +surface --- identifier and derivation layouts, primitive value encodings, the +whole-\texttt{Score} composite value codec, the operation-layer wire forms, +the bundle physical layout, and the extension-declaration blob forms --- +\emph{plus} the \textbf{K0/M2 operation-payload wire framing}. The Operation +Catalog's K0 set (the representative primitives and the M2 broad-K0 groups, +catalog version 0.4.0) now exists in ratified form, so the historical ``waits +for Agent~K'' clause in the Phase-2 charter is discharged: every operation +kind the catalog defines has its literal wire form pinned in +Chapter~\ref{ch:ops}. + +This document does \emph{not} cover: + +\begin{itemize} + \item the canonical s-expression form --- that is the \emph{Text Projection} + companion's; + \item per-profile feature lists --- the \emph{Profile Conformance} + companion's; + \item the Chapter-4 tuning-catalog values (pitch-space and tuning-system + registries), which are undelivered Track-C work and have no wire form yet; + \item the required conformance harnesses (the cross-implementation decoder + test and the wire-format fuzzer). Those are \emph{implementation} + deliverables tracked in the Phase-2 process documents, not part of this + document's normative text; Chapter~\ref{ch:goldens} records the golden + anchors that any such harness must reproduce. +\end{itemize} + +\section{Relationship to the Core Specification and the Operation Catalog} +\label{sec:about:relationship} + +This companion \emph{inherits} --- it never re-derives --- the core +specification's ratified convention baseline: core specification Chapter~8, +\sectionsc{Binary Format Companion}, requirement +\texttt{req:format:codec-conventions}, together with the +\sectionsc{Canonical Byte-Layout Reference} appendix (\texttt{app:bytes}). +That appendix consolidates every discriminant table, derivation preimage, and +primitive encoding Pass~11 and Pass~12 ratified, precisely so that this +document could import it as a starting point rather than recovering the +layouts from the crates. Chapter~\ref{ch:ids} therefore reproduces only +\emph{compact} normative tables, each with a citation back to the appendix +row and requirement that governs it; where this document and a ratified core +requirement disagree, \textbf{the core requirement governs} and the +discrepancy is a defect in this document. + +What this document adds is the completion of the appendix's deferred list +(core specification, \sectionsc{Layouts deferred to the companions}, +\texttt{sec:bytes:deferred}): the \texttt{OperationKind} and +\texttt{OperationKindTag} discriminants and full per-operation payload +encodings; the full composite struct layouts (whole-\texttt{Score}, operation +envelopes, the manifest body's field order); schema-version wire evolution; +and the varint question, which Chapter~\ref{ch:conventions} settles. + +The division of labour with the \emph{Operation Catalog} is exact: the +catalog fixes each operation's \textbf{field set and order} (Operation +Catalog, \sectionsc{Per-Primitive Schema Template}) and its semantics; this +document pins the \textbf{bytes} of those fields. Chapter~\ref{ch:ops} cites +the catalog section for every payload rather than restating what the fields +mean. Likewise, graph \emph{types} are the core specification's Chapters 2--5, +and the structural layer of the bundle --- what a header, superblock, chunk +graph, or manifest \emph{is} and how it behaves --- is core Chapter~8; this +document specifies only how those structures' bits are laid out. + +\begin{rationale} +\textbf{Versioning.} This companion is versioned independently of the core +specification (independent semver), like the Operation Catalog --- but tied +tighter to it: the binary-format \textsc{major} version tracks the core +specification's major version (both are pre-1.0 working drafts today, hence +v0.1.0), while \textsc{minor} versions diverge freely. Byte layouts change on +a slower, more deliberate cadence than prose; a core-spec minor revision that +touches no wire fact requires no revision here, and an additive discriminant +append here (Chapter~\ref{ch:evolution}) requires no core-spec revision. +\end{rationale} + +\section{Conformance} +\label{sec:about:conformance} + +The byte layouts in this document are \textbf{normative}. An implementation +conforms to the Epiphany binary format if and only if, for every layout this +document defines, it produces exactly the specified bytes when encoding and +accepts exactly the specified language of byte strings when decoding. + +Decode discipline is \textbf{reject, never normalize} --- the workspace-wide +convention the core specification establishes in Appendix~D (canonical +serialization determinism and ordered iteration). Concretely, each of the +following is a \emph{decode error}, never an input to be repaired: + +\begin{itemize} + \item an unknown discriminant in any tagged union; + \item a set- or map-valued field whose elements are not in strictly + ascending canonical order, or which contains duplicates; + \item text that is not in Unicode NFC where NFC is required (catalog + identifiers, barrier pitch-space names, operation labels); + \item trailing bytes after a complete top-level value + (Requirement~\ref{req:binfmt:trailing-bytes}); + \item recursion beyond a normative depth bound + (Requirement~\ref{req:binfmt:condition-depth}). +\end{itemize} + +A decoder \MUSTNOT{} silently sort, deduplicate, NFC-fold, truncate, or +otherwise canonicalize non-canonical input: two implementations that disagree +on repair would disagree on canonical state. + +The \textbf{golden anchor registry} (Chapter~\ref{ch:goldens}) is the +conformance contract binding this text to the reference implementation: every +layout in this document is anchored by at least one named test in the +reference crates, and a change that breaks an anchor is either a defect or a +deliberate, versioned format revision (Chapter~\ref{ch:evolution}). + +% =========================================================================== +\chapter{Encoding Conventions} +\label{ch:conventions} + +This chapter restates the ratified convention baseline (core specification, +requirement \texttt{req:format:codec-conventions}) as normative fact and +completes it: the three prefix/endianness regimes, the primitive composition +rules, the varint disposition, and the trailing-bytes rule. + +\section{The Three Encoding Regimes} +\label{sec:conventions:regimes} + +Three internally consistent regimes coexist in the format. Every layout in +this document belongs to exactly one of them, and each layout's chapter says +which. + +\begin{longtable}{p{0.55in} p{2.25in} p{2.85in}} + \toprule + \textbf{Regime} & \textbf{Surfaces} & \textbf{Rules} \\ + \midrule + \endhead + (a) & Canonical document encodings: the core value codec + (Chapter~\ref{ch:values}), the operation layer + (Chapter~\ref{ch:ops}), the bundle codec + (Chapter~\ref{ch:bundle}). & + Little-endian fixed-width integers; \texttt{u32} little-endian counts + and length prefixes on every variable-width field. \\ + (b) & Layout-side pinned encodings: the extension-declaration blobs and + edit-barrier trees (Chapter~\ref{ch:barriers}) and the + \texttt{ResolvedLayoutIR} canonical output + (Chapter~\ref{ch:noncanon}). & + Little-endian integers; \texttt{u64} little-endian counts and length + prefixes. Deliberate, golden-locked divergence from regime (a). \\ + (c) & Identifiers and hash-derived forms: the 128-bit identifier family, + \texttt{ReplicaId}, \texttt{OperationId}, \texttt{ConflictId}, the + \texttt{TypedObjectId} discriminant, digest truncations + (Chapter~\ref{ch:ids}). & + Big-endian fixed width. Numeric order equals canonical byte order, + which is what Appendix~D's ordered iteration requires. \\ + \bottomrule +\end{longtable} + +\begin{rationale} +Regimes (a) and (b) differ only in prefix width, and the divergence is +\emph{tolerated rather than repaired}: the two surfaces were golden-locked +independently (the barrier blobs and the resolved-layout output carry their +own literal-byte anchors), they never embed one another's framing, and +unifying the prefix width now would break locked bytes for zero benefit. The +divergence is recorded as an open question for the next schema-major revision +(Chapter~\ref{ch:evolution}). Regime (c) is not a divergence at all: an +identifier's byte form \emph{is} its sort key, so big-endian is load-bearing. +\end{rationale} + +\section{Primitive Composition Rules} +\label{sec:conventions:primitives} + +The following rules apply throughout regime (a); regimes (b) and (c) state +their deltas in their own chapters. + +\begin{description} + \item[Booleans.] A single byte: \texttt{0} = false, \texttt{1} = true. Any + other value is a decode error. + \item[Integers.] Fixed-width little-endian two's complement + (\texttt{u8}/\texttt{i8} through \texttt{u128}), except within regime (c). + \item[\texttt{Option}.] A presence byte: \texttt{0} = absent (no further + bytes), \texttt{1} = present, followed by \texttt{T}'s encoding. A + presence byte greater than~1 is a decode error. + \item[Sequences, sets, maps.] A \texttt{u32} little-endian element count, + then each element (for maps: key then value). Set- and map-valued fields + are emitted in canonical iteration order --- ascending by the key's + canonical byte form --- and decoders reject out-of-order or duplicated + elements wherever the field is a set or map (as opposed to an + order-significant list). + \item[Free text.] A \texttt{u32} little-endian byte length, then the raw + UTF-8 bytes. Free-text fields (e.g.\ a score title) are \textbf{never} + NFC-folded by the codec: $\mathrm{decode}(\mathrm{encode}(x)) = x$ is the + round-trip identity, and folding would break it. By contrast, + \emph{catalog identifiers} (pitch-space ids, tuning-system ids, and the + other \texttt{catalog\_id} newtypes) are NFC-normalized \emph{at + construction}, so their stored form is already NFC; the codec encodes the + stored string verbatim. Operation-layer text fields (the transaction + label, conflict field paths) are NFC-normalized at encode time + (Chapter~\ref{ch:ops}). + \item[Structs.] Positional, unframed concatenation of the fields' encodings + in declaration order. No field tags, no per-field lengths, no padding, no + alignment. (This is the layout the frozen-layout rule, + Requirement~\ref{req:binfmt:frozen-layout}, freezes.) + \item[Tagged unions.] A single discriminant byte, then the selected + variant's payload encoded positionally. Two documented exceptions: + \texttt{TypedObjectId} uses a \emph{16-bit big-endian} discriminant + (core requirement \texttt{req:graph:typed-object-id-discriminants}; + Chapter~\ref{ch:ids}), and the bundle's \texttt{ProfileId} uses a + \texttt{u32} little-endian discriminant (core requirement + \texttt{req:format:profileid-discriminants}; Chapter~\ref{ch:bundle}). + \item[Determinism-canonical leaves.] Where a composite codec embeds a value + whose canonical byte form is owned by the determinism layer or the + identifier layer (a graph identifier, \texttt{ContentHash}, + \texttt{CanonicalF64}, \texttt{RationalTime}, a wall-clock integer), the + core value codec frames it as a \texttt{u32} little-endian length prefix + followed by the leaf's own canonical bytes --- even when the leaf is + fixed-width. The leaf's bytes are then self-describing under its own + decoder. (The operation layer instead embeds fixed-width leaves + \emph{raw}, unprefixed; Chapter~\ref{ch:ops} states this explicitly.) +\end{description} + +\begin{requirement} +\label{req:binfmt:no-varint} +\textbf{No varint.} Schema major~0 defines \emph{no} variable-length integer +encoding. Every integer field in every layout in this document is +fixed-width. The core specification's ``varint conventions'' language +(Chapter~8, \sectionsc{Binary Format Companion}) is hereby discharged as +\emph{none}: an encoder \MUSTNOT{} emit, and a decoder \MUSTNOT{} accept, any +LEB128-style or otherwise variable-width integer under schema major~0. +Variable width in this format comes only from explicit \texttt{u32}/% +\texttt{u64} length prefixes and from \texttt{RationalTime}'s length-prefixed +big-integer magnitudes (Chapter~\ref{ch:primitives}). +\end{requirement} + +\begin{requirement} +\label{req:binfmt:trailing-bytes} +\textbf{Trailing bytes.} Every top-level decode entry point --- a +whole-\texttt{Score} value, a per-value \texttt{CanonicalValue} decode, an +\texttt{OperationKindTag}, a \texttt{MaterializedState}, a manifest payload, +an operation-index payload, a block payload, an extension-declaration blob, a +fixed-width primitive --- \MUST{} reject input with bytes remaining after the +complete value has been decoded. Trailing bytes are a decode error, never +ignored padding. +\end{requirement} + +% =========================================================================== +\chapter{Identifiers and Derivations} +\label{ch:ids} + +This chapter is an \emph{import-by-citation} of the core specification's +\sectionsc{Canonical Byte-Layout Reference} appendix (\texttt{app:bytes}): +that appendix and its golden anchors govern every table in this chapter, and +the tables here are compact restatements for self-containedness, not a second +source of truth. All layouts in this chapter are regime (c): big-endian, +fixed width. + +\section{The 128-Bit Identifier Family} +\label{sec:ids:family} + +\begin{description} + \item[\texttt{ReplicaId}.] 8 bytes, big-endian \texttt{u64}. The reserved + system namespace is \texttt{ReplicaId::SYSTEM\ub DERIVED} = + \texttt{0xFFFF\ub FFFF\ub FFFF\ub FFFF}; user-authored replicas + \MUSTNOT{} use it. + \item[Typed graph identifiers.] Every typed 128-bit graph identifier + (\texttt{EventId}, \texttt{PitchId}, \texttt{VoiceId}, + \texttt{RegionId}, \texttt{StaffInstanceId}, \texttt{TransactionId}, + \ldots{} --- the \texttt{graph\_id} family) packs the 64-bit replica + into the high half and the counter into the low half, and encodes as + exactly 16~bytes: replica big-endian (8) \cat{} counter big-endian (8). + Numeric order, canonical byte order, and Appendix-D iteration order + coincide. + \item[\texttt{OperationId}.] The same 16-byte form: replica big-endian (8) + \cat{} counter big-endian (8). Ordering is $(\mathit{replica}, + \mathit{counter})$ lexicographic, which equals byte order. + \item[Registry and author identifiers (operation layer).] The opaque + 128-bit registry newtypes (\texttt{OperationKind\allowbreak RegistryId}, + \texttt{ConflictKind\allowbreak RegistryId}, + \texttt{Resolution\allowbreak RegistryId}, + \texttt{RepairKind\allowbreak RegistryId}, + \texttt{ReanchorReason\allowbreak RegistryId}, + \texttt{ReplicaAnomaly\allowbreak RegistryId}, + \texttt{IntegrityAnomaly\allowbreak RegistryId}, + \texttt{Extension\allowbreak PreconditionId}, + \texttt{PreconditionFailure\allowbreak RegistryId}) and \texttt{AuthorId} + all encode as 16 big-endian bytes. +\end{description} + +\section{\texttt{TypedObjectId}} +\label{sec:ids:typed-object-id} + +Canonical form: a \textbf{16-bit big-endian discriminant} \cat{} the variant +payload (core requirement \texttt{req:graph:typed-object-id-discriminants}). +Non-\texttt{Registered} variants carry one 16-byte identifier, for a total of +\textbf{18~bytes}; \texttt{Registered} carries the +\texttt{ObjectKindRegistryId} (16 big-endian bytes) \cat{} the extension's +own raw \texttt{u128} (16 big-endian bytes), for a total of \textbf{34~bytes}. +An unknown discriminant is a decode error. + +\begingroup\small +\begin{longtable}{p{0.5in} p{1.6in} p{0.5in} p{1.6in}} + \toprule + \textbf{Disc} & \textbf{Variant} & \textbf{Disc} & \textbf{Variant} \\ + \midrule + \endhead + \tablenums{0} & \texttt{Event} & \tablenums{14} & \texttt{Spanner} \\ + \tablenums{1} & \texttt{Pitch} & \tablenums{15} & \texttt{Marker} \\ + \tablenums{2} & \texttt{Voice} & \tablenums{16} & \texttt{AnalyticalAnnotation} \\ + \tablenums{3} & \texttt{Staff} & \tablenums{17} & \texttt{Comment} \\ + \tablenums{4} & \texttt{StaffInstance} & \tablenums{18} & \texttt{GraphicObject} \\ + \tablenums{5} & \texttt{StaffGroup} & \tablenums{19} & \texttt{GraphicGesture} \\ + \tablenums{6} & \texttt{Region} & \tablenums{20} & \texttt{TimeSignature} \\ + \tablenums{7} & \texttt{Instrument} & \tablenums{21} & \texttt{AnalysisLayer} \\ + \tablenums{8} & \texttt{PartDefinition} & \tablenums{22} & \texttt{Tuplet} \\ + \tablenums{9} & \texttt{Measure} & \tablenums{23} & \texttt{RepeatStructure} \\ + \tablenums{10} & \texttt{BarlineAlignmentGroup} & \tablenums{24} & \texttt{LyricLine} \\ + \tablenums{11} & \texttt{Slur} & \tablenums{25} & \texttt{ChordSymbol} \\ + \tablenums{12} & \texttt{Tie} & \tablenums{26} & \texttt{View} \\ + \tablenums{13} & \texttt{Beam} & \tablenums{27} & \texttt{Registered} \\ + \bottomrule +\end{longtable} +\endgroup + +\section{System-Derived Counters and Digest Truncations} +\label{sec:ids:derivations} + +All content derivation uses BLAKE3-256 over a domain-separated preimage: the +8-byte domain tag is always the first bytes hashed. Two truncations recur, +both taking the digest's \emph{leading} bytes big-endian: + +\begin{itemize} + \item $\mathrm{trunc64}(d) = \mathtt{u64::from\_be\_bytes}(d[0..8])$ --- + the counter of a system-derived identifier; + \item $\mathrm{trunc128}(d) = \mathtt{u128::from\_be\_bytes}(d[0..16])$ --- + content-derived 128-bit identifiers. +\end{itemize} + +The system-derived counter function is +$\mathrm{trunc64}(\mathrm{BLAKE3}(\mathit{tag} \cat \mathit{inputs}))$; the +resulting identifier is +$\texttt{from\_parts}(\texttt{ReplicaId::SYSTEM\_DERIVED}, \mathit{counter})$. +Only a system domain tag (prefix \texttt{MUSCS}) may seed it. The ratified +instances (governed by \texttt{app:bytes} and the cited requirements): + +\begingroup\small +\begin{longtable}{p{1.35in} p{0.7in} p{3.5in}} + \toprule + \textbf{Identifier} & \textbf{Tag} & \textbf{Canonical inputs} \\ + \midrule + \endhead + Promoted \texttt{VoiceId} & \texttt{MUSCSVCE} & + 64-byte preimage: \texttt{staff\_instance} (16 BE) \cat{} + \texttt{original\_voice} (16 BE) \cat{} \texttt{winning\_op} (16 BE) + \cat{} \texttt{losing\_op} (16 BE). Core \sectionsc{System-Promoted + Voices}. \\ + System \texttt{PitchId} & \texttt{MUSCSPCH} & + The pitch's intrinsic canonical bytes (core requirement + \texttt{req:graph:system-derived-pitch-id}). \\ + \texttt{IntegrityAnomalyId} & \texttt{MUSCSANM} & + The anomaly kind's canonical bytes (Chapter~\ref{ch:ops}; core + requirement \texttt{req:graph:integrity-anomaly-id}). \\ + \bottomrule +\end{longtable} +\endgroup + +The 128-bit content-derived identifiers: + +\begingroup\small +\begin{longtable}{p{1.1in} p{0.75in} p{3.7in}} + \toprule + \textbf{Identifier} & \textbf{Tag} & \textbf{Preimage after the tag} \\ + \midrule + \endhead + \texttt{ConflictId} & \texttt{MUSCCONF} & + \texttt{kind.canonical\_bytes()} \cat{} causing operations (each 16 BE, + sorted by canonical bytes) \cat{} affected objects (each + \texttt{TypedObjectId} canonical bytes, sorted); $\mathrm{trunc128}$. + Wire form of the id itself: 16 big-endian bytes. \\ + \texttt{ManifestId} & \texttt{MUSCMNIF} & + \texttt{document\_id} (16) \cat{} \texttt{generation} (\texttt{u64} LE) + \cat{} manifest body bytes (body excludes the \texttt{manifest\_id} + field); $\mathrm{trunc128}$. Restated normatively in + Chapter~\ref{ch:bundle} (core requirement + \texttt{req:format:manifest-id}). Note the \emph{on-disk} field encodes + little-endian (Chapter~\ref{ch:bundle}). \\ + \texttt{EnvelopeHash} & \texttt{MUSCENVH} & + The envelope's complete canonical bytes; the full 32-byte digest (no + truncation). Chapter~\ref{ch:ops}. \\ + \texttt{LayoutObjectId} & \texttt{MUSCLOID} & + Non-canonical; three preimage shapes, Chapter~\ref{ch:noncanon} + (core requirement \texttt{req:layoutir:object-id-derivation}). \\ + \texttt{BlobId} & \texttt{MUSCBLOB} & + The blob payload, bare (\texttt{req:format:blob-hash-shape}); full + 32-byte digest. Chapter~\ref{ch:primitives}. \\ + \bottomrule +\end{longtable} +\endgroup + +\section{The Domain-Tag Registry} +\label{sec:ids:tags} + +Every domain tag is exactly 8 printable-ASCII bytes beginning \texttt{MUSC}; +the vocabulary is closed (governed by \texttt{app:bytes}, +\sectionsc{Domain-tag registry}). Extension system tags \MUST{} begin +\texttt{MUSCS}, be exactly 8 bytes, and collide with neither the built-ins +nor the file magics. + +\begingroup\small +\begin{longtable}{p{1.05in} p{1.55in} p{3.0in}} + \toprule + \textbf{Tag} & \textbf{Name} & \textbf{Use} \\ + \midrule + \endhead + \texttt{MUSCCHNK} & chunk & non-manifest chunk hash preimages \\ + \texttt{MUSCMANI} & manifest & manifest chunk hash preimages \\ + \texttt{MUSCBLOB} & blob & \texttt{BlobId} (bare $\mathit{tag} \cat \mathit{payload}$) \\ + \texttt{MUSCCONF} & conflict & \texttt{ConflictId} derivation \\ + \texttt{MUSCENVH} & envelope & \texttt{EnvelopeHash} \\ + \texttt{MUSCFNTM} & font metrics & glyph-catalog metrics hash (non-canonical) \\ + \texttt{MUSCMNIF} & manifest id & \texttt{ManifestId} derivation \\ + \texttt{MUSCSVCE} & system voice & promoted-voice counters \\ + \texttt{MUSCSPCH} & system pitch & system-derived pitch counters \\ + \texttt{MUSCSANM} & system anomaly & \texttt{IntegrityAnomalyId} \\ + \texttt{MUSCLOID} & layout object id & \texttt{LayoutObjectId} (non-canonical) \\ + \midrule + \texttt{MUSCBND\textbackslash0} & bundle magic & file magic, \emph{not} a hash tag (note the trailing NUL) \\ + \texttt{MUSCSUPR} & superblock magic & slot magic, \emph{not} a hash tag \\ + \bottomrule +\end{longtable} +\endgroup + +\section{\texttt{SnapshotId}} +\label{sec:ids:snapshot-id} + +The core specification's support-type identity table says \emph{``the Binary +Format companion defines the exact derivation''} of \texttt{SnapshotId}. v0 +has no snapshot producer --- pruning and canonical-base creation are deferred +--- so there is nothing to derive \emph{from} yet. + +\begin{requirement} +\label{req:binfmt:snapshot-id-opaque} +In schema major~0, \texttt{SnapshotId} is an \textbf{opaque 16-byte +identifier}: 16 raw bytes on the wire (Chapter~\ref{ch:primitives}), with no +normative derivation function. Readers \MUST{} treat two +\texttt{SnapshotId}s as equal iff their 16 bytes are equal and \MUSTNOT{} +attempt to derive or verify a snapshot id from snapshot content. +\end{requirement} + +\begin{openquestion} +The \texttt{SnapshotId} derivation (presumably a content derivation in the +\texttt{trunc128} family, over the snapshot's canonical payload and its +covering frontier) is deferred together with the pruning/garbage-collection +track that will produce the first snapshots. It must be pinned before any +writer emits a \texttt{canonical\_base}; until then the opaque reading above +is complete. +\end{openquestion} + +% =========================================================================== +\chapter{Primitive Value Encodings} +\label{ch:primitives} + +The leaf encodings owned by the determinism layer and the small fixed +identifier types. These byte forms are what the framings of +Chapters~\ref{ch:values}--\ref{ch:bundle} embed. + +\begingroup\small +\begin{longtable}{>{\raggedright\arraybackslash}p{1.7in} p{0.5in} p{3.45in}} + \toprule + \textbf{Type} & \textbf{Width} & \textbf{Encoding} \\ + \midrule + \endhead + \texttt{CanonicalF64} & 8 & + IEEE-754 binary64, little-endian. $-0.0$ is canonicalized to $+0.0$ at + construction, so the negative-zero bit pattern never appears in canonical + bytes. Decode rejects NaN and $\pm\infty$ (non-finite floats are + corruption). \\ + \texttt{QuantizedCoord} & 8 & + \texttt{i64} little-endian, in units of $1/1024$ staff space + (\texttt{STAFF\_SPACE\_GRID} = 1024). Construction from a float rounds + ties-to-even and rejects non-finite or out-of-range input. \\ + \texttt{ContentHash} & 32 & Raw BLAKE3-256 digest bytes. \\ + \texttt{ChunkId} & 32 & Raw digest bytes (a \texttt{ContentHash} newtype). \\ + \texttt{DomainTag} & 8 & + The raw ASCII tag bytes. Decode validates against the closed vocabulary + of Section~\ref{sec:ids:tags}; foreign bytes are a decode error. \\ + \texttt{WallClockTime} & 8 & + \texttt{i64} little-endian, nanoseconds. Floating-point wall-clock time + is forbidden in stored data. \\ + \texttt{WallClockDuration} & 8 & \texttt{i64} little-endian, nanoseconds. \\ + \texttt{SchemaVersion} & 4 & + \texttt{major} \texttt{u16} LE \cat{} \texttt{minor} \texttt{u16} LE. \\ + \texttt{SemVer} & 12 & + \texttt{major} \texttt{u32} LE \cat{} \texttt{minor} \texttt{u32} LE + \cat{} \texttt{patch} \texttt{u32} LE. Canonical \emph{ordering} of + \texttt{SemVer}-keyed collections is \textbf{numeric} on + $(\mathit{major},\mathit{minor},\mathit{patch})$, never byte-lexicographic + on the little-endian encoding (which would order 256.0.0 before 1.0.0). \\ + \texttt{Reduction\allowbreak AlgorithmVersion} & 4 & \texttt{u32} + little-endian. \\ + \texttt{FileUuid}, \texttt{DocumentId}, \texttt{LineageId}, + \texttt{SnapshotId}, \texttt{ExtensionId}, \texttt{ProfileRegistryId} & 16 & + 16 raw opaque bytes each. No internal structure is interpreted; + equality is byte equality. \\ + \texttt{BlobId} & 32 & + Raw digest bytes: $\mathrm{BLAKE3}(\texttt{MUSCBLOB} \cat + \mathit{payload})$, the only bare $\mathit{tag} \cat \mathit{payload}$ + content hash in the format (core requirement + \texttt{req:format:blob-hash-shape}). \\ + \texttt{FrontierBytes} & var & + \texttt{u32} LE length \cat{} opaque bytes (an ops-computed causal + frontier the bundle carries verbatim). \\ + \bottomrule +\end{longtable} +\endgroup + +\section{\texttt{RationalTime}} +\label{sec:primitives:rationaltime} + +Restating core requirement \texttt{req:format:rationaltime-encoding}, which +governs. \texttt{RationalTime} is an arbitrary-precision rational, always +stored \emph{reduced} (equal rationals encode identically; the sign lives on +the numerator; the denominator is strictly positive). Its canonical byte +form: + +\begin{center} +\begin{tabular}{l l} + \toprule + \textbf{Field} & \textbf{Encoding} \\ + \midrule + sign & 1 byte: \tablenums{0} = zero, \tablenums{1} = positive, + \tablenums{2} = negative \\ + numerator & \texttt{u32} LE magnitude byte length \cat{} big-endian + magnitude bytes \\ + denominator & \texttt{u32} LE magnitude byte length \cat{} big-endian + magnitude bytes \\ + \bottomrule +\end{tabular} +\end{center} + +Decode rejects a sign byte greater than~2, a zero denominator, and trailing +bytes. \texttt{MusicalPosition} and \texttt{MusicalDuration} are newtypes +whose canonical bytes are exactly their inner \texttt{RationalTime}'s. +Wall-clock integers deliberately do \emph{not} use this form: they are the +fixed-width little-endian integers of the table above, matching +\texttt{QuantizedCoord}'s convention. + +% =========================================================================== +\chapter{Graph Value Layouts} +\label{ch:values} + +This chapter defines the schema-major-0 wire form of every value reachable +from a \texttt{Score} --- the whole-document composite codec. The core +specification's \texttt{DECISIONS} record (P11-4) held this codec +\emph{provisional} pending this companion; +Requirement~\ref{req:binfmt:score-envelope} and +Requirement~\ref{req:binfmt:frozen-layout} \textbf{ratify} it: the reference +implementation's layout, as specified here, \emph{is} the schema-major-0 wire +form, and P11-4's provisional status is discharged. + +All layouts in this chapter are regime (a): little-endian, \texttt{u32} +prefixes. + +\section{The \texttt{Score} Envelope} +\label{sec:values:score} + +\begin{requirement} +\label{req:binfmt:score-envelope} +The canonical bytes of a \texttt{Score} are the bare positional +concatenation of its nineteen fields' encodings, in exactly this order: + +\begin{enumerate} + \item \texttt{metadata} + \item \texttt{canvas} + \item \texttt{instruments} + \item \texttt{staves} + \item \texttt{staff\_groups} + \item \texttt{parts} + \item \texttt{cross\_cutting} + \item \texttt{time\_signatures} + \item \texttt{tuning\_context} + \item \texttt{tempo\_map} + \item \texttt{events} + \item \texttt{spelling\_attachments} + \item \texttt{decomposition\_attachments} + \item \texttt{spelling\_precedence} + \item \texttt{analysis\_layers} + \item \texttt{views} + \item \texttt{identity} + \item \texttt{tombstoned\_pitches} + \item \texttt{tombstoned\_events} +\end{enumerate} + +There is \textbf{no magic} and \textbf{no version prefix} at the value +level: format identification lives in the bundle's fixed header +(Chapter~\ref{ch:bundle}) and versioning lives at the chunk layer +(\texttt{SchemaVersion} in every \texttt{ChunkRef} and superblock; +Chapter~\ref{ch:evolution}). Decoders \MUST{} reject trailing bytes after +the nineteenth field. +\end{requirement} + +\section{Composition Rules as Normative Encoding Facts} +\label{sec:values:composition} + +The reference codec is built from a small set of conventions; each is a +normative statement about the bytes, not an implementation detail: + +\begin{description} + \item[Positional structs.] Every struct in this chapter encodes as its + fields in declaration order, unframed + (Section~\ref{sec:conventions:primitives}). Zero-field structs encode as + zero bytes. + \item[C-style enums.] A fieldless enum is a single discriminant byte + (tables in Section~\ref{sec:values:discriminants}). + \item[Tagged unions.] A discriminant byte, then the variant's fields + positionally. + \item[Leaf framing.] The determinism-canonical leaf types --- + \texttt{ReplicaId}, \texttt{OperationId}, every embedded typed graph + identifier (\texttt{EventId}, \texttt{PitchId}, \texttt{VoiceId}, + \texttt{StaffId}, \texttt{StaffInstanceId}, \texttt{StaffGroupId}, + \texttt{RegionId}, \texttt{InstrumentId}, \texttt{PartDefinitionId}, + \texttt{MeasureId}, \texttt{BarlineAlignmentGroupId}, \texttt{SlurId}, + \texttt{TieId}, \texttt{BeamId}, \texttt{SpannerId}, \texttt{TupletId}, + \texttt{MarkerId}, \texttt{AnalyticalAnnotationId}, \texttt{CommentId}, + \texttt{RepeatStructureId}, \texttt{LyricLineId}, \texttt{ChordSymbolId}, + \texttt{GraphicObjectId}, \texttt{GraphicGestureId}, + \texttt{TimeSignatureId}, \texttt{AnalysisLayerId}, \texttt{ViewId}) --- + plus \texttt{ContentHash}, \texttt{CanonicalF64}, \texttt{RationalTime}, + \texttt{MusicalPosition}, \texttt{MusicalDuration}, + \texttt{WallClockTime}, and \texttt{WallClockDuration}, are each emitted + as \texttt{u32} LE length \cat{} the leaf's canonical bytes + (Chapters~\ref{ch:ids} and \ref{ch:primitives}). An embedded 16-byte + identifier therefore occupies 20 bytes inside a \texttt{Score}. Decode + runs the leaf's own validating decoder over the framed region. + \item[Catalog identifiers.] The \texttt{catalog\_id} newtypes + (\texttt{PitchSpaceId}, \texttt{TuningSystemId}, \texttt{AccidentalId}, + registry-name ids, \ldots) encode as their stored NFC string under the + String rule (\texttt{u32} LE length \cat{} UTF-8). + \item[\texttt{EventArena}.] \texttt{u32} LE event count, then each + \texttt{Event} in ascending-\texttt{EventId} order (identity travels + inside each event; the arena adds no per-event framing). + \item[Validated reconstruction.] Types with checked constructors decode + through them: \texttt{TupletRatio} rejects degenerate ratios, + \texttt{KeySignature} encodes its \texttt{fifths()} as \texttt{i8} and + rejects out-of-range values, \texttt{TimeSignature} re-validates that + beat groups sum to the measure duration, \texttt{NonZeroU16} rejects + zero, \texttt{EventOrderingDAG} rejects cyclic edge maps, + \texttt{ReferencePitch}, \texttt{Tempo}, and + \texttt{SpellingPrecedence} re-validate their invariants. Structurally + well-formed bytes that fail a type invariant are a decode error + (reject-never-normalize). +\end{description} + +\section{Discriminant Tables} +\label{sec:values:discriminants} + +The complete discriminant assignment for schema major~0. Every table is +governed by Requirement~\ref{req:binfmt:frozen-layout}: an unknown +discriminant is a decode error; changes are gated by +Chapter~\ref{ch:evolution}. + +\subsection{C-style enums (one byte, no payload)} + +\begingroup\small +\begin{longtable}{p{1.9in} p{3.7in}} + \toprule + \textbf{Type} & \textbf{Discriminants} \\ + \midrule + \endhead + \texttt{MeasurePosition} & \tablenums{0} Start, \tablenums{1} End \\ + \texttt{RegionEdge} & \tablenums{0} Start, \tablenums{1} End \\ + \texttt{CmnNominal} & \tablenums{0} C, \tablenums{1} D, \tablenums{2} E, + \tablenums{3} F, \tablenums{4} G, \tablenums{5} A, \tablenums{6} B \\ + \texttt{SpellingSourceKind} & \tablenums{0} UserChosen, + \tablenums{1} Imported, \tablenums{2} Propagated, \tablenums{3} Inferred, + \tablenums{4} Analytical \\ + \texttt{TempoShape} & \tablenums{0} Constant, \tablenums{1} Linear, + \tablenums{2} Exponential, \tablenums{3} Curve \\ + \texttt{StemDirection} & \tablenums{0} Up, \tablenums{1} Down \\ + \texttt{MeasureNumberVisibility} & \tablenums{0} Auto, + \tablenums{1} Always, \tablenums{2} Never \\ + \texttt{Aleatoric\allowbreak AnchoringDiscipline} & \tablenums{0} Musical, + \tablenums{1} WallClock, \tablenums{2} EitherPerEvent, + \tablenums{3} FreelyMixed \\ + \texttt{NoteValue} & \tablenums{0} Whole, \tablenums{1} Half, + \tablenums{2} Quarter, \tablenums{3} Eighth, \tablenums{4} Sixteenth, + \tablenums{5} ThirtySecond, \tablenums{6} SixtyFourth \\ + \texttt{ClefShape} & \tablenums{0} G, \tablenums{1} F, \tablenums{2} C, + \tablenums{3} Percussion \\ + \bottomrule +\end{longtable} +\endgroup + +\subsection{Tagged unions (one byte, then the variant payload)} + +\begingroup\small +\begin{longtable}{p{1.9in} p{3.7in}} + \toprule + \textbf{Type} & \textbf{Discriminants} \\ + \midrule + \endhead + \texttt{AnchorOffset} & \tablenums{0} Musical, \tablenums{1} WallClock, + \tablenums{2} Zero \\ + \texttt{TimeAnchor} & \tablenums{0} Event, \tablenums{1} Measure, + \tablenums{2} Region, \tablenums{3} WallClock + (Section~\ref{sec:values:representative}) \\ + \texttt{EventPosition} & \tablenums{0} Musical, \tablenums{1} WallClock \\ + \texttt{ConcreteDuration} & \tablenums{0} Musical, \tablenums{1} WallClock \\ + \texttt{EventDuration} & \tablenums{0} Musical, \tablenums{1} WallClock, + \tablenums{2} Indeterminate \\ + \texttt{TimeBounds} & \tablenums{0} MusicalRange, + \tablenums{1} WallClockRange, \tablenums{2} Unbounded \\ + \texttt{PitchSpacePosition} & \tablenums{0} Cmn, \tablenums{1} Integer, + \tablenums{2} JiVector, \tablenums{3} Registered \\ + \texttt{TuningReference} & \tablenums{0} Inherit, \tablenums{1} Explicit \\ + \texttt{AcousticRealization} & \tablenums{0} Implicit, + \tablenums{1} CentsOffset, \tablenums{2} AbsoluteHz \\ + \texttt{SpellingNominal} & \tablenums{0} Cmn, \tablenums{1} Integer, + \tablenums{2} Registered \\ + \texttt{SpellingSource} & \tablenums{0} UserChosen, \tablenums{1} Inferred, + \tablenums{2} Imported, \tablenums{3} Propagated, + \tablenums{4} Analytical \\ + \texttt{SpellingScope} & \tablenums{0} Pitch, \tablenums{1} Range \\ + \texttt{SpellingDirective} & \tablenums{0} Explicit, \tablenums{1} Rule \\ + \texttt{VoiceSelector} & \tablenums{0} All, \tablenums{1} Voices \\ + \texttt{GraceKind} & \tablenums{0} Acciaccatura, \tablenums{1} Appoggiatura, + \tablenums{2} Unmeasured, \tablenums{3} MeasuredFraction \\ + \texttt{IndeterminacyKind} & \tablenums{0} Pitch, \tablenums{1} Duration, + \tablenums{2} Choice, \tablenums{3} Compound \\ + \texttt{TrajectoryEndpoint} & \tablenums{0} EventPitch, + \tablenums{1} ExplicitPitch \\ + \texttt{TrajectoryShape} & \tablenums{0} Linear, \tablenums{1} Exponential, + \tablenums{2} Curve, \tablenums{3} Stepwise \\ + \texttt{Event} & \tablenums{0} Pitched, \tablenums{1} Unpitched, + \tablenums{2} Rest, \tablenums{3} Indeterminate, \tablenums{4} Trajectory, + \tablenums{5} Graphic, \tablenums{6} Cue + (Section~\ref{sec:values:representative}) \\ + \texttt{RegionTimeModel} & \tablenums{0} Metric, \tablenums{1} Proportional, + \tablenums{2} Aleatoric \\ + \texttt{RegionContent} & \tablenums{0} StaffBased, \tablenums{1} FreeGraphic, + \tablenums{2} Hybrid \\ + \texttt{VoiceOrigin} & \tablenums{0} UserDeclared, \tablenums{1} Imported, + \tablenums{2} SystemPromoted \\ + \texttt{StaffGroupKind} & \tablenums{0} GrandStaff, \tablenums{1} Bracket, + \tablenums{2} SubBracket, \tablenums{3} Choral, \tablenums{4} Registered \\ + \texttt{TieClass} & \tablenums{0} Standard, \tablenums{1} Editorial, + \tablenums{2} CrossVoice, \tablenums{3} LaissezVibrer, + \tablenums{4} Registered \\ + \texttt{AnnotationAnchor} & \tablenums{0} Event, \tablenums{1} Range, + \tablenums{2} Region \\ + \texttt{GestureAnchoring} & \tablenums{0} Events, \tablenums{1} Range, + \tablenums{2} Free \\ + \texttt{DecompositionSource} & \tablenums{0} UserChosen, + \tablenums{1} Inferred, \tablenums{2} Imported, \tablenums{3} Propagated \\ + \texttt{TimeSignatureDisplay} & \tablenums{0} Standard, + \tablenums{1} Compound, \tablenums{2} Irrational, + \tablenums{3} MixedDenominators, \tablenums{4} None, + \tablenums{5} Symbolic \\ + \bottomrule +\end{longtable} +\endgroup + +\begin{requirement} +\label{req:binfmt:spelling-source-order} +\textbf{The \texttt{SpellingSource} / \texttt{SpellingSourceKind} trap.} +These two vocabularies name the same five provenance concepts but assign +them \emph{different} discriminants: \texttt{SpellingSourceKind} (the +C-style precedence-list element) is \tablenums{0}~UserChosen, +\tablenums{1}~Imported, \tablenums{2}~Propagated, \tablenums{3}~Inferred, +\tablenums{4}~Analytical, while \texttt{SpellingSource} (the payload-carrying +attachment provenance) is \tablenums{0}~UserChosen, \tablenums{1}~Inferred, +\tablenums{2}~Imported, \tablenums{3}~Propagated, \tablenums{4}~Analytical. +The orders differ (\texttt{Inferred} is 3 in one and 1 in the other), both +are golden-locked, and an implementation \MUSTNOT{} share one discriminant +function between them. +\end{requirement} + +\section{Representative Complete Layouts} +\label{sec:values:representative} + +The following layouts are spelled byte-by-byte both as worked examples of the +composition rules and as normative fact. Recall that inside this chapter's +codec every leaf is framed (\texttt{u32} LE length \cat{} bytes), so an +embedded 16-byte identifier occupies $4+16=20$ bytes. + +\subsection{\texttt{RationalTime}} +As a leaf inside a composite: \texttt{u32} LE length \cat{} the canonical +form of Section~\ref{sec:primitives:rationaltime} (sign \cat{} +length-prefixed big-endian numerator magnitude \cat{} length-prefixed +big-endian denominator magnitude). + +\subsection{\texttt{TimeAnchor}} +One discriminant byte, then: +\begin{description} + \item[\tablenums{0} Event] \texttt{id} (\texttt{EventId} leaf, 20) \cat{} + \texttt{offset} (\texttt{AnchorOffset}: tag byte, then for Musical a + \texttt{MusicalDuration} leaf, for WallClock a + \texttt{WallClockDuration} leaf ($4+8=12$), for Zero nothing). + \item[\tablenums{1} Measure] \texttt{id} (\texttt{MeasureId} leaf, 20) + \cat{} \texttt{position} (\texttt{MeasurePosition}, 1) \cat{} + \texttt{offset} (\texttt{AnchorOffset}). + \item[\tablenums{2} Region] \texttt{id} (\texttt{RegionId} leaf, 20) \cat{} + \texttt{edge} (\texttt{RegionEdge}, 1) \cat{} \texttt{offset} + (\texttt{AnchorOffset}). + \item[\tablenums{3} WallClock] \texttt{time} (\texttt{WallClockTime} leaf, + 12). +\end{description} + +\subsection{\texttt{Pitch}} +Positional struct: \texttt{scale\_position} \cat{} \texttt{acoustic}. +\begin{itemize} + \item \texttt{ScalePosition} = \texttt{space} (\texttt{PitchSpaceId}: NFC + string, \texttt{u32} LE length \cat{} UTF-8) \cat{} \texttt{position} + (\texttt{PitchSpacePosition}: tag byte, then Cmn = \texttt{nominal} + (\texttt{CmnNominal}, 1) \cat{} \texttt{alteration} (\texttt{i8}, 1) + \cat{} \texttt{octave} (\texttt{i8}, 1); Integer = + \texttt{space\_size} (\texttt{u16} LE, 2) \cat{} \texttt{index} + (\texttt{i32} LE, 4); JiVector = \texttt{u32} LE count \cat{} that many + \texttt{i32} LE components; Registered = a catalog-id string). + \item \texttt{AcousticPitch} = \texttt{tuning} (\texttt{TuningReference}: + tag byte; Explicit carries a \texttt{TuningSystemId} catalog-id string) + \cat{} \texttt{realization} (\texttt{AcousticRealization}: tag byte; + CentsOffset and AbsoluteHz each carry a \texttt{CanonicalF64} leaf, + $4+8=12$). +\end{itemize} + +\subsection{\texttt{Event}} +One discriminant byte (\tablenums{0}--\tablenums{6}), then the variant struct +positionally. All seven variants share the prefix \texttt{id} +(\texttt{EventId} leaf) \cat{} \texttt{voice} (\texttt{VoiceId} leaf) \cat{} +\texttt{position} (\texttt{EventPosition}) \cat{} \texttt{duration} +(\texttt{EventDuration}); the remaining fields, in order: + +\begingroup\small +\begin{longtable}{p{0.35in} p{1.35in} p{3.8in}} + \toprule + \textbf{Disc} & \textbf{Variant} & \textbf{Fields after the common prefix} \\ + \midrule + \endhead + \tablenums{0} & \texttt{Pitched} & + \texttt{pitches} \cat{} \texttt{articulations} \cat{} \texttt{dynamic} + \cat{} \texttt{ornaments} \cat{} \texttt{stem} \cat{} \texttt{grace} \\ + \tablenums{1} & \texttt{Unpitched} & + \texttt{staff\_position} \cat{} \texttt{instrument\_member} \cat{} + \texttt{articulations} \cat{} \texttt{dynamic} \cat{} \texttt{stem} + \cat{} \texttt{grace} \\ + \tablenums{2} & \texttt{Rest} & + \texttt{vertical\_position} \cat{} \texttt{visible} \\ + \tablenums{3} & \texttt{Indeterminate} & + \texttt{indeterminacy} \cat{} \texttt{hints} \\ + \tablenums{4} & \texttt{Trajectory} & + \texttt{start} \cat{} \texttt{end} \cat{} \texttt{shape} \cat{} + \texttt{display} \\ + \tablenums{5} & \texttt{Graphic} & + \texttt{graphics} \cat{} \texttt{playback\_bindings} \\ + \tablenums{6} & \texttt{Cue} & + \texttt{source} \cat{} \texttt{rendering} \\ + \bottomrule +\end{longtable} +\endgroup + +Each field encodes under this chapter's rules for its type (vectors are +\texttt{u32}-counted, options carry a presence byte, embedded identifiers are +leaves, unions carry their tag byte). + +\subsection{\texttt{Slur}} +Positional struct of three leaves: \texttt{id} (\texttt{SlurId} leaf, 20) +\cat{} \texttt{start\_event} (\texttt{EventId} leaf, 20) \cat{} +\texttt{end\_event} (\texttt{EventId} leaf, 20) --- 60 bytes total. + +\section{The \texttt{CanonicalValue} Seam} +\label{sec:values:canonical-value} + +The public per-value codec --- the seam between this document and the +Operation Catalog's value-typed payloads (Operation Catalog, +\sectionsc{Value-Typed Payloads}, requirement +\texttt{req:catalog:value-encoding}) --- is defined for exactly these +eighteen types: + +\begin{quote} +\texttt{Event}, \texttt{Rest}, \texttt{Pitch}, \texttt{IdentifiedPitch}, +\texttt{PitchSpelling}, \texttt{Tie}, \texttt{Slur}, \texttt{Beam}, +\texttt{Spanner}, \texttt{RegionTimeModel}, \texttt{TimeAnchor}, +\texttt{Region}, \texttt{StaffInstance}, \texttt{Voice}, +\texttt{DecompositionAttachment}, \texttt{SpellingSourceKind}, +\texttt{ScoreMetadata}, \texttt{MetricGrid}. +\end{quote} + +The seam introduces \textbf{no new bytes}: a value's stand-alone canonical +bytes are byte-for-byte the bytes the whole-\texttt{Score} codec embeds for +that value. Stand-alone decode applies the same validation and rejects +trailing bytes. + +\section{The Frozen-Layout Rule} +\label{sec:values:frozen} + +\begin{requirement} +\label{req:binfmt:frozen-layout} +\textbf{Frozen positional layouts (the schema-evolution keystone).} Within +schema major~0, every positional struct layout in this chapter is +\textbf{frozen}: there is no field-addition mechanism at the value level --- +no per-field framing, no optional-field encoding, no trailing-field +tolerance. Any change to any struct's field \emph{set} or field \emph{order} +is a schema-\textsc{major} change and requires a documented migration in this +document's revision history (Chapter~\ref{ch:evolution}). + +Appending a \emph{new variant} to an \textbf{open} discriminant vocabulary is +a schema-\textsc{minor} change under the rules of +Chapter~\ref{ch:evolution}. At the value layer, the open vocabularies are +those with a \texttt{Registered} escape variant --- +\texttt{PitchSpacePosition}, \texttt{SpellingNominal}, +\texttt{StaffGroupKind}, \texttt{TieClass} --- through which extensions +attach without any wire change at all; every \emph{other} union in +Section~\ref{sec:values:discriminants} (including \texttt{Event}, +\texttt{TimeAnchor}, \texttt{TimeSignatureDisplay}) is closed in v0 and may +grow only by an append ratified in a revision of this document. + +This rule ratifies the project's staging decision that data-model payload +expansion --- \texttt{SlurKind}, beam geometry, voltas, instrument bodies, +score-metadata growth, and their kin --- lands as a coordinated +schema-\textsc{major} revision of this document, not as ad-hoc field +insertion. +\end{requirement} + +% =========================================================================== +\chapter{Operation Wire Forms} +\label{ch:ops} + +This chapter pins the operation layer: envelopes, payloads, stamps, causal +contexts, effects, and materialized state. It \textbf{ratifies} the +encodings the ops crate's \texttt{DECISIONS} record held as ``provisional +canonical encoding'' (mirroring core P11-4); that provisional status is +discharged. + +Regime (a) applies, with one composition difference from +Chapter~\ref{ch:values}: the operation layer embeds \emph{fixed-width} +canonical leaves (identifiers, hashes, \texttt{ConflictId}) \textbf{raw}, +without a length prefix (\texttt{push\_canon}); only variable-width parts +carry \texttt{u32} LE length prefixes (\texttt{push\_lp\_bytes}). Sequences +(\texttt{push\_seq}) are a \texttt{u32} LE count, then each element +individually length-prefixed (\texttt{u32} LE), regardless of element width. +Text (\texttt{push\_str}) is NFC-normalized at encode time, then +length-prefixed UTF-8. Embedded graph values are the +\texttt{CanonicalValue} bytes of Section~\ref{sec:values:canonical-value}, +framed by a \texttt{u32} LE length prefix (Operation Catalog requirement +\texttt{req:catalog:value-encoding}). + +\section{The Operation Envelope} +\label{sec:ops:envelope} + +\begin{requirement} +\label{req:binfmt:envelope} +An \texttt{OperationEnvelope}'s canonical bytes are, in order: + +\begin{center} +\begin{tabular}{p{1.35in} p{3.9in}} + \toprule + \textbf{Field} & \textbf{Encoding} \\ + \midrule + \texttt{id} & \texttt{OperationId}, 16 big-endian bytes, raw \\ + \texttt{author} & \texttt{AuthorId}, 16 big-endian bytes, raw \\ + \texttt{stamp} & \texttt{OperationStamp}, 28 bytes + (Section~\ref{sec:ops:stamp}) \\ + \texttt{causal\_context} & \texttt{CausalContext} + (Section~\ref{sec:ops:causal}) \\ + \texttt{transaction} & option tag byte (\tablenums{0} absent / + \tablenums{1} present), then \texttt{TransactionId} 16 BE if present \\ + \texttt{payload} & \texttt{OperationPayload} + (Section~\ref{sec:ops:payload}) \\ + \bottomrule +\end{tabular} +\end{center} + +\textbf{Id-leads property.} The envelope's first 16 bytes \emph{are} the +\texttt{OperationId}'s canonical bytes. This is load-bearing: the operation +index (Chapter~\ref{ch:bundle}, Section~\ref{sec:bundle:opindex}) keys its +entries on those leading 16 bytes without decoding envelopes, and the +sanctioned partial read (\emph{peek}) of an envelope is exactly its leading +16 bytes. Any future envelope revision \MUST{} preserve id-leads or revise +the operation index in the same schema-major step. +\end{requirement} + +$\texttt{EnvelopeHash} = \mathrm{BLAKE3}(\texttt{MUSCENVH} \cat +\mathit{envelope\ canonical\ bytes})$ --- the full 32-byte digest. + +Envelopes are \textbf{encode-only} at this layer, deliberately: once +committed, an envelope is stored and transported as opaque bytes and is +never reconstructed into typed form by the storage layer; the peek above is +the only sanctioned partial read. (Reducers decode payloads they authored or +received through the ops layer's own typed surface; the wire contract is +that stored envelope bytes are preserved verbatim, since +\texttt{EnvelopeHash} commits to them.) + +\section{\texttt{OperationPayload} and the Meta-Operations} +\label{sec:ops:payload} + +\texttt{OperationPayload}: one discriminant byte, then the variant. + +\begin{center} +\small +\begin{tabular}{p{0.35in} p{1.55in} p{3.3in}} + \toprule + \textbf{Disc} & \textbf{Variant} & \textbf{Payload} \\ + \midrule + \tablenums{0} & \texttt{Primitive} & an \texttt{OperationKind} + (Section~\ref{sec:ops:kinds}) \\ + \tablenums{1} & \texttt{ResolveConflict} & \texttt{target} + (\texttt{ConflictId}, 16 BE) \cat{} \texttt{action} + (\texttt{ResolutionAction}) \\ + \tablenums{2} & \texttt{UndoTransaction} & \texttt{target} + (\texttt{TransactionId}, 16 BE) \cat{} \texttt{policy} + (\texttt{UndoPolicy}, 1 byte) \\ + \tablenums{3} & \texttt{ResolveEquivocation} & \texttt{target} + (\texttt{OperationId}, 16 BE) \cat{} \texttt{chosen} + (\texttt{EnvelopeHash}, 32) \\ + \bottomrule +\end{tabular} +\end{center} + +Discriminants \tablenums{0}--\tablenums{2} are the ratified v1 values; +\tablenums{3} was appended by the catalog's \sectionsc{ResolveEquivocation +(meta-operation)} entry. The vocabulary is append-only: new meta-operations +take $\geq 4$. + +\texttt{UndoPolicy} is one byte: \tablenums{0} StrictInverse, +\tablenums{1} BestEffort, \tablenums{2} Cascade. +\texttt{ResolutionAction} is one byte then an optional payload +(\texttt{app:bytes}, core requirement +\texttt{req:semops:\allowbreak resolution-action-discriminants}): \tablenums{0} +AcceptLoser, \tablenums{1} KeepWinner, \tablenums{2} Override +(\cat{} \texttt{OperationId} 16 BE), \tablenums{3} Reanchor (\cat{} +\texttt{TypedObjectId}), \tablenums{4} Dismiss, \tablenums{5} Registered +(\cat{} \texttt{ResolutionRegistryId} 16 BE). + +\section{\texttt{OperationKind}: the Primitive Wire Discriminants} +\label{sec:ops:kinds} + +\begin{requirement} +\label{req:binfmt:kind-discriminants} +The \texttt{OperationKind} wire discriminant is one byte, golden-locked to +the table below. The vocabulary is \textbf{append-only}: new primitive kinds +take discriminants past \tablenums{23}; the assignments below never change. +\end{requirement} + +Each row also pins the payload's byte layout; the field \emph{set and order} +is the Operation Catalog's (the cited section governs semantics). +``$\mathrm{lp}(T)$'' means the \texttt{u32}-LE-length-prefixed +\texttt{CanonicalValue} bytes of a \texttt{T} +(Section~\ref{sec:values:canonical-value}); bare identifiers are raw 16 +big-endian bytes; ``$\mathrm{seq}^{\uparrow}$'' is a \texttt{push\_seq} +sequence sorted ascending by canonical bytes. + +\begingroup\footnotesize +\begin{longtable}{p{0.32in} >{\raggedright\arraybackslash}p{1.5in} >{\raggedright\arraybackslash}p{2.35in} >{\raggedright\arraybackslash}p{1.35in}} + \toprule + \textbf{Disc} & \textbf{Kind} & \textbf{Payload layout} & + \textbf{Catalog section} \\ + \midrule + \endhead + \tablenums{0} & \texttt{InsertEvent} & + \texttt{staff\_instance} (16) \cat{} $\mathrm{lp}$(\texttt{Event}) & + \sectionsc{InsertEvent} \\ + \tablenums{1} & \texttt{DeleteEvent} & + \texttt{event} (16) \cat{} \texttt{TupletCompensation} & + \sectionsc{DeleteEvent} \\ + \tablenums{2} & \texttt{RespellPitch} & + \texttt{pitch} (16) \cat{} $\mathrm{lp}$(\texttt{PitchSpelling}) & + \sectionsc{RespellPitch} \\ + \tablenums{3} & \texttt{CreateCrossCutting} & + \texttt{CrossCuttingValue} & + \sectionsc{CreateCrossCutting} \\ + \tablenums{4} & \texttt{ChangeRegionTimeModel} & + \texttt{region} (16) \cat{} $\mathrm{lp}$(\texttt{RegionTimeModel}) + \cat{} $\mathrm{seq}^{\uparrow}$(\texttt{EventId}) \cat{} + \texttt{PositionRemapping} & + \sectionsc{ChangeRegion\allowbreak TimeModel} \\ + \tablenums{5} & \texttt{SetUserSystemBreak} & + \texttt{region} (16) \cat{} $\mathrm{lp}$(\texttt{TimeAnchor}) \cat{} + \texttt{present} (bool, 1) & + \sectionsc{SetUserSystemBreak} \\ + \tablenums{6} & \texttt{DeclareTransaction} & + \texttt{id} (16) \cat{} \texttt{label} (NFC string) \cat{} option tag + \cat{} \texttt{TransactionCategory} if present & + \sectionsc{DeclareTransaction} \\ + \tablenums{7} & \texttt{Registered} & + \texttt{OperationKindRegistryId} (16) \cat{} + $\mathrm{lp}$(opaque bytes) & + \sectionsc{K1 --- Framework Slots} \\ + \tablenums{8} & \texttt{ModifyEvent} & + $\mathrm{lp}$(\texttt{Event}) & + \sectionsc{ModifyEvent} \\ + \tablenums{9} & \texttt{Transpose} & + $\mathrm{seq}^{\uparrow}$(\texttt{PitchId}) \cat{} + \texttt{chromatic\_steps} (\texttt{i32} LE, 4) & + \sectionsc{Transpose} \\ + \tablenums{10} & \texttt{InsertIdentifiedPitch} & + \texttt{event} (16) \cat{} $\mathrm{lp}$(\texttt{IdentifiedPitch}) & + \sectionsc{Identified-Pitch Operations} \\ + \tablenums{11} & \texttt{DeleteIdentifiedPitch} & + \texttt{pitch} (16) & + \sectionsc{Identified-Pitch Operations} \\ + \tablenums{12} & \texttt{ModifyIdentifiedPitch} & + \texttt{pitch} (16) \cat{} $\mathrm{lp}$(\texttt{Pitch}) & + \sectionsc{Identified-Pitch Operations} \\ + \tablenums{13} & \texttt{DeleteCrossCutting} & + \texttt{structure} (\texttt{TypedObjectId}, 18/34) & + \sectionsc{DeleteCrossCutting} \\ + \tablenums{14} & \texttt{ModifyCrossCutting} & + \texttt{CrossCuttingValue} & + \sectionsc{ModifyCrossCutting} \\ + \tablenums{15} & \texttt{CreateRegion} & + $\mathrm{lp}$(\texttt{Region}) & + \sectionsc{Structural Containers} \\ + \tablenums{16} & \texttt{DeleteRegion} & + \texttt{region} (16) & + \sectionsc{Structural Containers} \\ + \tablenums{17} & \texttt{CreateStaffInstance} & + \texttt{region} (16) \cat{} $\mathrm{lp}$(\texttt{StaffInstance}) & + \sectionsc{Structural Containers} \\ + \tablenums{18} & \texttt{DeleteStaffInstance} & + \texttt{staff\_instance} (16) & + \sectionsc{Structural Containers} \\ + \tablenums{19} & \texttt{CreateVoice} & + \texttt{staff\_instance} (16) \cat{} $\mathrm{lp}$(\texttt{Voice}) & + \sectionsc{Structural Containers} \\ + \tablenums{20} & \texttt{DeleteVoice} & + \texttt{voice} (16) & + \sectionsc{Structural Containers} \\ + \tablenums{21} & \texttt{SetMetadata} & + $\mathrm{lp}$(\texttt{ScoreMetadata}) & + \sectionsc{Score Settings} \\ + \tablenums{22} & \texttt{SetMetricGrid} & + \texttt{region} (16) \cat{} option tag (\tablenums{0}/\tablenums{1}) + \cat{} $\mathrm{lp}$(\texttt{MetricGrid}) if present & + \sectionsc{Score Settings} \\ + \tablenums{23} & \texttt{SetUserPageBreak} & + \texttt{region} (16) \cat{} $\mathrm{lp}$(\texttt{TimeAnchor}) \cat{} + \texttt{present} (bool, 1) & + \sectionsc{Score Settings} \\ + \bottomrule +\end{longtable} +\endgroup + +Sub-vocabularies used above (one discriminant byte, then the listed +payload): + +\begin{description} + \item[\texttt{TupletCompensation}.] \tablenums{0} NotInTuplet (empty); + \tablenums{1} ReplaceWithRest \cat{} $\mathrm{lp}$(\texttt{Rest}); + \tablenums{2} RewriteTuplets \cat{} + $\mathrm{seq}^{\uparrow}$(\texttt{TupletId}); + \tablenums{3} CascadeDeleteTuplets \cat{} + $\mathrm{seq}^{\uparrow}$(\texttt{TupletId}). + \item[\texttt{CrossCuttingValue}.] \tablenums{0} Tie, \tablenums{1} Slur, + \tablenums{2} Beam, \tablenums{3} Spanner; in every case the tag is + followed by $\mathrm{lp}$(the typed structure's \texttt{CanonicalValue} + bytes). + \item[\texttt{PositionRemapping}.] \tablenums{0} PreserveTime (empty); + \tablenums{1} Reassign \cat{} \texttt{u32} LE entry count \cat{} + entries, each \texttt{EventId} (16, raw) \cat{} + $\mathrm{lp}$(\texttt{MusicalPosition}), ascending by \texttt{EventId}. + \item[\texttt{TransactionCategory}.] \tablenums{0} NoteEntry, + \tablenums{1} Structural, \tablenums{2} Layout, \tablenums{3} Import, + \tablenums{4} Registered \cat{} \texttt{OperationKindRegistryId} (16 BE) + --- governed by \texttt{app:bytes} (core requirement + \texttt{req:semops:transaction-category}). +\end{description} + +\texttt{OperationKind} is encode-only at the storage boundary, like the +envelope that carries it. + +\section{\texttt{OperationKindTag}: a Separate Discriminant Space} +\label{sec:ops:kind-tag} + +\begin{requirement} +\label{req:binfmt:kind-tag} +\texttt{OperationKindTag} --- the payload-free projection of an operation +kind, stored by edit barriers +(Chapter~\ref{ch:barriers}) --- has its \textbf{own} discriminant space, +assigned independently of Section~\ref{sec:ops:kinds}, with both encode and +validating decode. One byte; only \texttt{Registered} carries a payload (its +\texttt{OperationKindRegistryId}, 16 big-endian bytes, total 17). Unknown +discriminants and wrong lengths are decode errors. Append-only past +\tablenums{23}. +\end{requirement} + +\begingroup\small +\begin{longtable}{p{0.5in} p{2.0in} p{0.5in} p{2.0in}} + \toprule + \textbf{Disc} & \textbf{Tag} & \textbf{Disc} & \textbf{Tag} \\ + \midrule + \endhead + \tablenums{0} & \texttt{InsertEvent} & \tablenums{12} & \texttt{DeleteStaffInstance} \\ + \tablenums{1} & \texttt{DeleteEvent} & \tablenums{13} & \texttt{SetUserSystemBreak} \\ + \tablenums{2} & \texttt{ModifyEvent} & \tablenums{14} & \texttt{SetUserPageBreak} \\ + \tablenums{3} & \texttt{RespellPitch} & \tablenums{15} & \texttt{DeclareTransaction} \\ + \tablenums{4} & \texttt{Transpose} & \tablenums{16} & \texttt{Registered} \\ + \tablenums{5} & \texttt{CreateCrossCutting} & \tablenums{17} & \texttt{InsertIdentifiedPitch} \\ + \tablenums{6} & \texttt{DeleteCrossCutting} & \tablenums{18} & \texttt{DeleteIdentifiedPitch} \\ + \tablenums{7} & \texttt{ModifyCrossCutting} & \tablenums{19} & \texttt{ModifyIdentifiedPitch} \\ + \tablenums{8} & \texttt{ChangeRegionTimeModel} & \tablenums{20} & \texttt{CreateVoice} \\ + \tablenums{9} & \texttt{InsertRegion} & \tablenums{21} & \texttt{DeleteVoice} \\ + \tablenums{10} & \texttt{DeleteRegion} & \tablenums{22} & \texttt{SetMetadata} \\ + \tablenums{11} & \texttt{InsertStaffInstance} & \tablenums{23} & \texttt{SetMetricGrid} \\ + \bottomrule +\end{longtable} +\endgroup + +\textbf{Naming mapping.} The kind-to-tag projection renames two pairs: +\texttt{OperationKind::\allowbreak CreateRegion} projects to +\texttt{OperationKindTag::\allowbreak InsertRegion}, and +\texttt{OperationKind::\allowbreak CreateStaffInstance} projects to +\texttt{OperationKindTag::\allowbreak InsertStaffInstance}. The mismatch is +intentional +(the tag space predates the M2c naming) and is pinned here so barrier +authors target the right tag. + +\section{Stamps and Causal Contexts} +\label{sec:ops:stamp} + +\texttt{HybridLogicalClock}: \texttt{physical\_time} (\texttt{WallClockTime}, +\texttt{i64} LE, 8) \cat{} \texttt{logical\_counter} (\texttt{u32} LE, 4). +\texttt{OperationStamp}: \texttt{hlc} (12) \cat{} \texttt{id} +(\texttt{OperationId}, 16 BE) --- \textbf{28 bytes} total, fixed width. The +canonical reduction tuple orders stamps as $(\mathit{physical\_time}, +\mathit{logical\_counter}, \mathit{replica}, \mathit{counter})$, ascending; +the replica comparison equals big-endian byte order. + +\label{sec:ops:causal} +\texttt{CausalContext} (dotted version vector): + +\begin{center} +\begin{tabular}{p{0.7in} p{4.55in}} + \toprule + \textbf{Field} & \textbf{Encoding} \\ + \midrule + vector & \texttt{u32} LE entry count \cat{} entries, each + \texttt{replica} (8 BE) \cat{} \texttt{counter} (\texttt{u64} LE, 8), + ascending by replica \\ + dots & \texttt{u32} LE count \cat{} \texttt{OperationId}s (16 BE each), + ascending \\ + \bottomrule +\end{tabular} +\end{center} + +The vector is \emph{zero-based-floor} (P11-C7): an entry $(r, c)$ covers +counters $0..=c$ of replica $r$; absence of a replica covers nothing. + +\section{Effects, Conflicts, Anomalies, and Materialized State} +\label{sec:ops:effects} + +These types are canonical \emph{materialized} facts: they appear in +\texttt{MaterializedState}'s canonical bytes, so their discriminants and +layouts are normative wire facts even though they are never stored inside +envelopes. All decode validation is performed by the ops layer's validating +decoder; unknown discriminants, bad lengths, non-canonical orders, and +trailing bytes are decode errors. + +\begingroup\footnotesize +\begin{longtable}{>{\raggedright\arraybackslash}p{1.6in} p{4.1in}} + \toprule + \textbf{Type} & \textbf{Layout (tag byte, then payload)} \\ + \midrule + \endhead + \texttt{OperationEffect} & + \tablenums{0} Applied (empty); + \tablenums{1} AppliedWithRepair \cat{} seq(\texttt{RepairRecord}); + \tablenums{2} Conflicted \cat{} \texttt{ConflictId} (16 BE); + \tablenums{3} TombstonedTarget \cat{} \texttt{TypedObjectId}; + \tablenums{4} NoOp \cat{} \texttt{NoOpReason}. \\ + \texttt{NoOpReason} & + \tablenums{0} TargetTombstoned; \tablenums{1} AlreadyApplied; + \tablenums{2} SupersededByLaterOperation \cat{} \texttt{OperationId} + (16 BE); \tablenums{3} PreconditionFailedUnderReduction \cat{} + \texttt{PreconditionFailureReason}; \tablenums{4} TransactionConflict. \\ + \texttt{Precondition\allowbreak FailureReason} & + \tablenums{0} TargetMissing; \tablenums{1} TargetTombstoned; + \tablenums{2} WrongRegionTimeModel; + \tablenums{3} TupletCompensationInvalid; + \tablenums{4} EventDurationInvalid; \tablenums{5} PositionOutsideRegion; + \tablenums{6} PitchSpaceMismatch; \tablenums{7} VoiceMissing; + \tablenums{8} ExtensionPrecondition \cat{} id (16 BE); + \tablenums{9} Registered \cat{} id (16 BE); + \tablenums{10} ContainerNotEmpty. \\ + \texttt{RepairRecord} & + (struct) \texttt{kind} (\texttt{RepairKind}) \cat{} \texttt{target} + (\texttt{TypedObjectId}). \\ + \texttt{RepairKind} & + \tablenums{0} Reanchored \cat{} \texttt{from} (\texttt{TypedObjectId}) + \cat{} \texttt{to} (\texttt{TypedObjectId}) \cat{} + \texttt{ReanchorReason}; + \tablenums{1} SpannerTruncated \cat{} seq(\texttt{TypedObjectId}); + \tablenums{2} Orphaned; \tablenums{3} CascadeDeleted; + \tablenums{4} AttachmentTombstoned; + \tablenums{5} VoicePromoted \cat{} \texttt{from} (\texttt{VoiceId}, + 16 BE) \cat{} \texttt{to} (\texttt{VoiceId}, 16 BE); + \tablenums{6} TupletCompensated \cat{} \texttt{TupletCompensationKind}; + \tablenums{7} Registered \cat{} id (16 BE). \\ + \texttt{ReanchorReason} & + \tablenums{0} SameVoiceNearer; \tablenums{1} SameStaffInstanceNearer; + \tablenums{2} SameStaffNearer; \tablenums{3} SameRegionNearer; + \tablenums{4} ExplicitFallback; + \tablenums{5} DeclaredByExtension \cat{} id (16 BE). \\ + \texttt{TupletCompensationKind} & + \tablenums{0} ReplaceWithRest; \tablenums{1} RewriteTuplets; + \tablenums{2} CascadeDeleteTuplets. (No payload; distinct from the + payload-carrying \texttt{TupletCompensation} of + Section~\ref{sec:ops:kinds}.) \\ + \texttt{PendingReason} & + tag \cat{} blocker (\texttt{OperationId}, 16 BE) in every variant: + \tablenums{0} MissingCausalPredecessor; \tablenums{1} + DependsOnEquivocated; \tablenums{2} DependsOnExcluded; + \tablenums{3} DependsOnPending; \tablenums{4} HaltedBySystemCollision. \\ + \texttt{ConflictKind} & + \tablenums{0} StructuralFieldCollision \cat{} \texttt{winner} (16 BE) + \cat{} \texttt{loser} (16 BE) \cat{} \texttt{FieldPath} (NFC string); + \tablenums{1} TransactionConflict \cat{} \texttt{transaction} (16 BE) + \cat{} $\mathrm{seq}^{\uparrow}$(\texttt{OperationId}); + \tablenums{2} TombstonedTarget \cat{} \texttt{TypedObjectId} \cat{} + \texttt{OperationId} (16 BE); + \tablenums{3} ReanchorFailure \cat{} \texttt{original\_referent} + (\texttt{TypedObjectId}) \cat{} \texttt{referencing\_object} + (\texttt{TypedObjectId}); + \tablenums{4} TimeModelMigrationFailure \cat{} \texttt{region} (16 BE) + \cat{} $\mathrm{seq}^{\uparrow}$(\texttt{TypedObjectId}); + \tablenums{5} ExtensionConflict \cat{} \texttt{kind\_id} (16 BE) \cat{} + $\mathrm{lp}$(opaque details). \\ + \texttt{ConflictResolutionState} & + \tablenums{0} Unresolved; \tablenums{1} Resolved \cat{} \texttt{by} + (16 BE) \cat{} \texttt{ResolutionAction}; \tablenums{2} Dismissed \cat{} + \texttt{by} (16 BE). \\ + \texttt{ConflictRecord} & + (struct) \texttt{id} (\texttt{ConflictId}, 16 BE) \cat{} + seq(\texttt{caused\_by}: \texttt{OperationId}, stored sorted) \cat{} + \texttt{kind} \cat{} seq(\texttt{affected\_objects}: + \texttt{TypedObjectId}, stored sorted) \cat{} + \texttt{resolution\_state}. \\ + \texttt{IntegrityAnomaly} & + (struct) \texttt{id} (\texttt{IntegrityAnomalyId}, 16 BE) \cat{} + \texttt{kind}. \\ + \texttt{IntegrityAnomalyKind} & + \tablenums{0} SystemIdentifierCollision \cat{} \texttt{ObjectKind} + \cat{} \texttt{colliding\_counter} (\texttt{u64} LE) \cat{} the two + input-set blobs, each $\mathrm{lp}$(bytes), emitted lo${}\leq{}$hi by + byte comparison; + \tablenums{1} OperationSlotEquivocated \cat{} \texttt{OperationId} + (16 BE); + \tablenums{2} ReplicaStreamQuarantined \cat{} \texttt{replica} (8 BE) + \cat{} \texttt{first\_bad\_counter} (\texttt{u64} LE); + \tablenums{3} Registered \cat{} id (16 BE). \\ + \texttt{AnomalousReplicaSegment} & + (struct) \texttt{replica} (8 BE) \cat{} \texttt{first\_bad\_counter} + (\texttt{u64} LE) \cat{} \texttt{ReplicaAnomalyReason} + (\tablenums{0} HlcMonotonicityViolation \cat{} two + \texttt{OperationId}s; \tablenums{1} Registered \cat{} id 16 BE) + \cat{} seq(\texttt{excluded}: \texttt{OperationId}, ascending counter). \\ + \texttt{FieldPath} & + an NFC string (\texttt{u32} LE length \cat{} UTF-8). \\ + \texttt{ObjectState} & + \tablenums{0} Live (empty); \tablenums{1} Tombstoned \cat{} + \texttt{deleted\_by} (\texttt{OperationId}, 16 BE) \cat{} + \texttt{minted\_by} (\texttt{OperationId}, 16 BE). \\ + \texttt{ObjectKind} (ops) & + \tablenums{0} Voice; \tablenums{1} Pitch; \tablenums{2} Registered + \cat{} \texttt{OperationKindRegistryId} (16 BE) --- governed by + \texttt{app:bytes} (core requirement + \texttt{req:graph:object-kind-vocab}). Distinct from the barrier layer's + 2-byte \texttt{ObjectKind} (Chapter~\ref{ch:barriers}). \\ + \bottomrule +\end{longtable} +\endgroup + +\subsection{\texttt{MaterializedState}} +\label{sec:ops:materialized} + +The canonical bytes of a reduction's materialized state are eight sections, +concatenated; every section is \texttt{u32}-LE-counted (the conflict +registry carries its own count) and in the stated normative order, so the +bytes are identical across any permutation of the input operation set: + +\begin{enumerate} + \item \textbf{effects} --- count, then per entry \texttt{OperationId} + (16 BE, raw) \cat{} $\mathrm{lp}$(\texttt{OperationEffect}); in + canonical reduction order; + \item \textbf{conflict registry} --- a seq of + $\mathrm{lp}$(\texttt{ConflictRecord}), ascending \texttt{ConflictId}; + \item \textbf{anomalies} --- count, then each + $\mathrm{lp}$(\texttt{IntegrityAnomaly}), ascending + \texttt{IntegrityAnomalyId}; + \item \textbf{objects} --- count, then per entry \texttt{TypedObjectId} + (raw) \cat{} \texttt{ObjectState}, ascending \texttt{TypedObjectId}; + \item \textbf{spellings} --- count, then per entry \texttt{PitchId} + (16 BE, raw) \cat{} $\mathrm{lp}$(\texttt{PitchSpelling}), ascending + \texttt{PitchId}; + \item \textbf{breaks} --- count, then per entry \texttt{RegionId} + (16 BE, raw) \cat{} \texttt{MusicalPosition} (raw \texttt{RationalTime} + form, self-describing, \emph{not} leaf-framed) \cat{} \texttt{present} + (bool, 1); ascending $(\mathit{region}, \mathit{position})$; + \item \textbf{page breaks} --- same shape and order as breaks; + \item \textbf{pending} --- count, then per entry \texttt{OperationId} + (16 BE, raw) \cat{} \texttt{PendingReason}, ascending + \texttt{OperationId}. +\end{enumerate} + +Decode is fully validating (orders, tags, lengths, trailing bytes) and +rejects non-canonical input. + +\section{v0 Payloads Have No Wire Form} +\label{sec:ops:v0} + +The prototype's v0 identifier-projection payload types are an +\textbf{in-memory migration guard only}: they carry no canonical encoding, +and no v0 wire form was ever shipped. The v0${\to}$v1 migration is +\emph{semantic} --- typed value reconstruction, specified in the Operation +Catalog's \sectionsc{v0 $\rightarrow$ v1 Payload Migration} chapter --- not a +byte-level translation, and nothing in this document defines bytes for a v0 +payload. + +% =========================================================================== +\chapter{Bundle Physical Layout} +\label{ch:bundle} + +The physical file: the fixed prelude, chunks, blocks, the manifest, and the +operation index. This chapter \textbf{ratifies} the bundle crate's +provisional encodings (its \texttt{DECISIONS} entries P11-D2, P11-D4, and +P11-D5 pending-companion status is discharged) and the operation-index +byte form (P12-D1). Structural \emph{semantics} --- superblock selection, +commit protocol, recovery, retention --- are core specification Chapter~8; +this chapter pins bytes only. Regime (a) applies except where a table says +otherwise. + +\section{The Fixed Header (64 bytes)} +\label{sec:bundle:header} + +Offset 0 of every bundle. Written at creation, never rewritten within a +format major version. All integers little-endian. + +\begingroup\small +\begin{longtable}{p{0.8in} p{1.7in} p{3.1in}} + \toprule + \textbf{Range} & \textbf{Field} & \textbf{Value / rule} \\ + \midrule + \endhead + \tablenums{0..8} & magic & \texttt{MUSCBND\textbackslash0} (8 bytes, ASCII, + trailing NUL) \\ + \tablenums{8..10} & \texttt{format\_major} (\texttt{u16}) & \tablenums{0} + in this version; readers reject other values \\ + \tablenums{10..12} & \texttt{format\_minor} (\texttt{u16}) & \tablenums{1} + written in this version \\ + \tablenums{12..16} & \texttt{header\_length} (\texttt{u32}) & + \tablenums{64}; readers reject other values \\ + \tablenums{16..24} & \texttt{superblock\_a\_offset} (\texttt{u64}) & + \tablenums{64}; readers reject other values \\ + \tablenums{24..32} & \texttt{superblock\_b\_offset} (\texttt{u64}) & + \tablenums{320}; readers reject other values \\ + \tablenums{32..48} & \texttt{file\_uuid} & 16 opaque bytes \\ + \tablenums{48..60} & reserved & written zero; ignored on read \\ + \tablenums{60..64} & \texttt{header\_crc} & CRC-32C of bytes + \tablenums{0..60}, \texttt{u32} LE \\ + \bottomrule +\end{longtable} +\endgroup + +Readers \MUST{} verify the magic and the CRC before consulting any other part +of the file. + +\section{The Superblock Slots (256 bytes each)} +\label{sec:bundle:superblock} + +Two slots, at offsets 64 (A) and 320 (B); the only mutable on-disk objects. +Selection \emph{semantics} (highest valid committed generation, the +equal-generation rule, torn-write fallback) are core Chapter~8, +\sectionsc{Superblock Selection}; the bytes: + +\begingroup\small +\begin{longtable}{p{0.9in} p{2.15in} p{2.55in}} + \toprule + \textbf{Range} & \textbf{Field} & \textbf{Encoding} \\ + \midrule + \endhead + \tablenums{0..8} & magic & \texttt{MUSCSUPR} (8 ASCII bytes) \\ + \tablenums{8..16} & \texttt{generation} & \texttt{u64} LE \\ + \tablenums{16..24} & \texttt{manifest\_offset} & \texttt{u64} LE \\ + \tablenums{24..32} & \texttt{manifest\_length} & \texttt{u64} LE (also the + uncompressed length; the manifest is mandatorily uncompressed) \\ + \tablenums{32..64} & \texttt{manifest\_hash} & 32 raw digest bytes \\ + \tablenums{64..68} & \texttt{manifest\_schema\_version} & + \texttt{SchemaVersion} (\texttt{u16} LE \cat{} \texttt{u16} LE) \\ + \tablenums{68..72} & \texttt{reduction\_algorithm\_version} & + \texttt{u32} LE \\ + \tablenums{72..92} & \texttt{profile\_id} & 20 bytes: \texttt{u32} LE + discriminant (\tablenums{0} Full, \tablenums{1} ReadOnly, + \tablenums{2} Lite, \tablenums{3} Custom) \cat{} + \texttt{ProfileRegistryId} (16 raw bytes; writers \MUST{} emit zero + unless Custom, readers ignore it unless Custom). Core requirement + \texttt{req:format:profileid-discriminants}. \\ + \tablenums{92..100} & \texttt{commit\_state} & two \texttt{u32} LE words: + Committed $= (\tablenums{0}, \tablenums{0})$; any tag + ${}\neq \tablenums{0}$ decodes as Reserved(value) and the slot is + invalid for ordinary selection. Writers in this version \MUST{} produce + only Committed. \\ + \tablenums{100..108} & \texttt{commit\_timestamp} & \texttt{i64} LE + nanoseconds; advisory --- selection never consults it \\ + \tablenums{108..252} & reserved & written zero; ignored on read \\ + \tablenums{252..256} & \texttt{superblock\_crc} & CRC-32C of bytes + \tablenums{0..252}, \texttt{u32} LE \\ + \bottomrule +\end{longtable} +\endgroup + +\section{Chunks} +\label{sec:bundle:chunks} + +The bundle body begins at +$\texttt{BODY\_START} = 320 + 256 = \tablenums{576}$. A chunk's on-disk form +is its raw (possibly compressed) payload bytes at a byte offset --- +\textbf{no per-chunk length prefix, magic, or trailer on disk}; the offset, +lengths, compression, and hash live in the referencing \texttt{ChunkRef}. A +chunk reference whose offset is below \texttt{BODY\_START} is malformed. + +\textbf{Content hash preimage} (core Chapter~8, \sectionsc{Domain-Separated +Preimages}; governs identity): +\[ + \mathrm{BLAKE3}\bigl(\mathit{domain} \cat \mathit{kind}\,(1) \cat + \mathit{schema}\,(4) \cat \mathit{uncompressed\_length}\, + (\texttt{u64}\ \mathrm{LE}) \cat \mathit{payload}\bigr) +\] +where $\mathit{domain}$ is \texttt{MUSCMANI} for \texttt{Manifest} chunks and +\texttt{MUSCCHNK} otherwise. Compression is \emph{not} in the preimage. +Blob chunks are addressed by the bare \texttt{BlobId} form instead +(Chapter~\ref{ch:primitives}). + +\textbf{\texttt{ChunkKind}} is a single byte, a \textbf{closed} vocabulary +(no \texttt{Registered} variant; core requirement +\texttt{req:format:chunkkind-discriminants}): \tablenums{0} +OperationEnvelopeBlock, \tablenums{1} OperationIndex, \tablenums{2} +Snapshot, \tablenums{3} Blob, \tablenums{4} ExtensionData, \tablenums{5} +TextProjection, \tablenums{6} LayoutCache, \tablenums{7} IntegrityIndex, +\tablenums{8} Manifest. + +\textbf{\texttt{CompressionAlgorithm}} is a fixed two bytes, discriminant +\cat{} parameter: None $= (\tablenums{0},\tablenums{0})$; +Zstd$\{$level$\}$ $= (\tablenums{1}, \mathit{level})$; Reserved$(v)$ $= +(\tablenums{2}, v)$. \texttt{None} is \emph{not} a bare tag --- its zero +parameter byte is always present. + +\textbf{Read rules.} Writers in this format version emit \texttt{None} only; +\emph{reading} zstd at any level is a conformance \MUST{}. A zstd payload is +decompressed into a buffer sized \emph{exactly} by the declared +\texttt{uncompressed\_length} (checked against the reader's resource policy +before allocation): a stream that would exceed the declaration, ends short of +it, or carries trailing garbage is corruption. After decompression the +content hash is verified over the uncompressed bytes, and +$\mathit{id} = \mathit{hash}$ is checked. A chunk whose declared schema +major differs from the reader's supported major is rejected +(Chapter~\ref{ch:evolution}). \texttt{Reserved} compression is rejected. +The \textbf{manifest chunk is mandatorily uncompressed}: a compressed +manifest reference is rejected before any bytes are read. The reference +reader bounds any single chunk at 256\,MiB +(\texttt{MAX\_CHUNK\_BYTES}); the bound is reader policy, not a wire fact. + +\subsection{\texttt{ChunkRef} (95 bytes)} +\label{sec:bundle:chunkref} + +\begingroup\small +\begin{longtable}{p{0.9in} p{1.9in} p{2.8in}} + \toprule + \textbf{Range} & \textbf{Field} & \textbf{Encoding} \\ + \midrule + \endhead + \tablenums{0..32} & \texttt{id} & 32 raw digest bytes \\ + \tablenums{32..33} & \texttt{kind} & \texttt{ChunkKind} byte \\ + \tablenums{33..37} & \texttt{schema\_version} & \texttt{u16} LE \cat{} + \texttt{u16} LE \\ + \tablenums{37..45} & \texttt{offset} & \texttt{u64} LE \\ + \tablenums{45..53} & \texttt{compressed\_length} & \texttt{u64} LE \\ + \tablenums{53..61} & \texttt{uncompressed\_length} & \texttt{u64} LE \\ + \tablenums{61..63} & \texttt{compression} & 2 bytes \\ + \tablenums{63..95} & \texttt{hash} & 32 raw digest bytes (restates + \texttt{id}) \\ + \bottomrule +\end{longtable} +\endgroup + +The canonical sort key for chunk-reference collections is +$(\mathit{kind\ discriminant}, \mathit{hash}, \mathit{offset})$, ascending +--- the Appendix-D chunk-reference order. Every set-valued +\texttt{ChunkRef} field in this chapter is sorted by it and deduplicated +before encoding. + +\section{Operation-Envelope Block Payloads} +\label{sec:bundle:blocks} + +The uncompressed payload of an \texttt{OperationEnvelopeBlock} chunk: + +\begin{center} +\texttt{u32} LE envelope count \cat{} per envelope +\{\texttt{u32} LE byte length \cat{} envelope bytes\} +\end{center} + +Envelope bytes are opaque to the bundle (their internal form is +Chapter~\ref{ch:ops}). Each envelope's offset --- the byte offset of its +\emph{first content byte} within the uncompressed block payload, with its +length prefix at $\mathit{offset}-4$ --- is deterministically recoverable +from this framing alone; it is the coordinate the operation index records. +Writers \SHOULD{} begin a new block rather than let an uncompressed payload +exceed 1\,MiB (an individual envelope larger than that occupies its own +block); readers \MUST{} accept blocks up to the active profile's bound +(default 64\,MiB). Block boundaries are storage artifacts: the envelope set +is the union across blocks. + +\section{The Manifest} +\label{sec:bundle:manifest} + +On-disk manifest chunk payload: + +\begin{center} +\texttt{manifest\_id} (16 bytes, \texttt{u128} \emph{little-endian}) \cat{} +\emph{body} +\end{center} + +(The little-endian on-disk field is a deliberate, golden-locked quirk: the +\emph{derivation} truncates big-endian like every content-derived id, but +the bundle codec serializes the resulting \texttt{u128} under its +little-endian integer rule.) + +\begin{requirement} +\label{req:binfmt:manifest-id} +Restating core requirement \texttt{req:format:manifest-id}, which governs: +\[ + \texttt{ManifestId} = \mathrm{trunc128}\bigl(\mathrm{BLAKE3}( + \texttt{MUSCMNIF} \cat \mathit{document\_id} \cat + \mathit{generation} \cat \mathit{body}\bigr)\bigr) +\] +with $\mathit{document\_id}$ as its 16 raw bytes, $\mathit{generation}$ as +\texttt{u64} little-endian, and $\mathit{body}$ the complete manifest body +below (which itself opens with \texttt{document\_id} and +\texttt{generation} --- the double commitment is intentional and locked). +Writers re-derive the id from the body at encode time; decoders verify it. +\end{requirement} + +\textbf{The body} is a positional struct of thirteen fields: + +\begingroup\small +\begin{longtable}{p{0.3in} p{1.9in} p{3.45in}} + \toprule + \textbf{\#} & \textbf{Field} & \textbf{Encoding and canonical order} \\ + \midrule + \endhead + \tablenums{1} & \texttt{document\_id} & 16 raw bytes \\ + \tablenums{2} & \texttt{lineage\_id} & option tag \cat{} 16 raw bytes if + present \\ + \tablenums{3} & \texttt{generation} & \texttt{u64} LE \\ + \tablenums{4} & \texttt{operation\_roots} & \texttt{u32} LE count \cat{} + \texttt{ChunkRef}s, sorted by the canonical chunk-reference key, + deduplicated \\ + \tablenums{5} & \texttt{operation\_block\_summaries} & \texttt{u32} LE + count \cat{} entries \{\texttt{chunk\_id} (32) \cat{} + \texttt{OperationBlockSummary}\}, ascending \texttt{chunk\_id} \\ + \tablenums{6} & \texttt{operation\_index\_root} & option tag \cat{} + \texttt{ChunkRef} \\ + \tablenums{7} & \texttt{canonical\_base} & option tag \cat{} + \texttt{SnapshotRef} \\ + \tablenums{8} & \texttt{acceleration\_snapshots} & \texttt{u32} LE count + \cat{} \texttt{SnapshotRef}s, sorted ascending by their full encoded + bytes, deduplicated \\ + \tablenums{9} & \texttt{blob\_roots} & \texttt{u32} LE count \cat{} + \texttt{BlobRef}s, sorted ascending by encoded bytes, deduplicated \\ + \tablenums{10} & \texttt{profile\_declarations} & \texttt{u32} LE count + \cat{} \texttt{ProfileDeclaration}s, sorted by the key + $(\texttt{profile\_id}, \texttt{version})$ with \texttt{SemVer} compared + \emph{numerically}, deduplicated \\ + \tablenums{11} & \texttt{extension\_declarations} & \texttt{u32} LE count + \cat{} \texttt{ExtensionDeclaration}s, sorted by + $(\texttt{extension\_id}, \texttt{version})$, \texttt{SemVer} numeric, + deduplicated \\ + \tablenums{12} & \texttt{text\_projection\_root} & option tag \cat{} + \texttt{ChunkRef} \\ + \tablenums{13} & \texttt{integrity\_root} & option tag \cat{} + \texttt{ChunkRef} \\ + \bottomrule +\end{longtable} +\endgroup + +Sub-records, positionally: + +\begin{description} + \item[\texttt{OperationBlockSummary}] = \texttt{dvv\_summary} + (\texttt{FrontierBytes}: \texttt{u32}-LE-prefixed opaque bytes) \cat{} + \texttt{min\_stamp} (\texttt{u32}-LE-prefixed opaque stamp bytes) \cat{} + \texttt{max\_stamp} (same). The contents are ops-computed + (Chapter~\ref{ch:ops}); the bundle carries them verbatim. + \item[\texttt{SnapshotRef}] = \texttt{snapshot\_id} (16 raw) \cat{} + \texttt{covers\_causal\_frontier} (\texttt{FrontierBytes}) \cat{} + \texttt{reduction\_algorithm\_version} (\texttt{u32} LE) \cat{} + \texttt{profile\_id} (20 bytes, as in the superblock) \cat{} + \texttt{root} (\texttt{ChunkRef}, 95) \cat{} \texttt{hash} (32 raw). + \item[\texttt{BlobRef}] = \texttt{blob\_id} (32 raw) \cat{} + \texttt{media\_type} (\texttt{u32}-LE-prefixed ASCII; decode validates a + well-formed RFC~6838 \texttt{type/subtype} restricted name) \cat{} + \texttt{offset} (\texttt{u64} LE) \cat{} \texttt{compressed\_length} + (\texttt{u64} LE) \cat{} \texttt{uncompressed\_length} (\texttt{u64} LE) + \cat{} \texttt{compression} (2) \cat{} \texttt{hash} (32 raw) \cat{} + option tag \cat{} \texttt{declared\_max\_uncompressed\_length} + (\texttt{u64} LE) if present. + \item[\texttt{ProfileDeclaration}] = \texttt{profile\_id} (20) \cat{} + \texttt{version} (\texttt{SemVer}, 12) \cat{} \texttt{constraints} + (\texttt{ProfileConstraints} = + \texttt{max\_uncompressed\_block\_size} (\texttt{u64} LE) \cat{} + \texttt{RetentionPolicy}). \texttt{RetentionPolicy} = + \texttt{retain\_previous\_manifests} (\texttt{u32} LE) \cat{} option tag + \cat{} \texttt{retain\_duration} (\texttt{i64} LE) if present \cat{} + \texttt{retain\_named\_checkpoints} (bool, 1). + \item[\texttt{ExtensionDeclaration}] = \texttt{extension\_id} (16 raw) + \cat{} \texttt{version} (\texttt{SemVer}, 12) \cat{} \texttt{required} + (bool, 1) \cat{} \texttt{u32} LE count \cat{} + \texttt{preserved\_chunk\_roots} (\texttt{ChunkRef}s, canonical order, + deduplicated) \cat{} \texttt{affected\_object\_kinds} + (\texttt{u32}-LE-prefixed opaque blob) \cat{} \texttt{edit\_barriers} + (\texttt{u32}-LE-prefixed opaque blob). The two blobs' internal form is + Chapter~\ref{ch:barriers}; the bundle preserves them verbatim. +\end{description} + +\begin{requirement} +\label{req:binfmt:manifest-canonical} +\textbf{Reject non-canonical manifests.} A decoder \MUST{} accept a manifest +payload only if re-encoding the decoded manifest reproduces the input +byte-for-byte. This single check subsumes: the stored \texttt{manifest\_id} +matching the body-derived id, every set-valued vector being in its canonical +order with no duplicates, and every optional field using presence byte +\tablenums{0}/\tablenums{1}. Unsorted, duplicated, or wrongly-identified +manifest bytes are malformed --- never re-sorted or repaired. +\end{requirement} + +\section{The Operation Index} +\label{sec:bundle:opindex} + +\begin{requirement} +\label{req:binfmt:opindex} +\textbf{Operation-index payload (ratifies P12-D1).} The uncompressed payload +of an \texttt{OperationIndex} chunk is: + +\begin{center} +\texttt{u32} LE \texttt{block\_count} \cat{} that many \texttt{ChunkRef}s +(95 bytes each, strictly ascending canonical order) \cat{} +\texttt{u32} LE \texttt{entry\_count} \cat{} that many entries, each +\{\texttt{id} (16 raw operation-id bytes) \cat{} \texttt{block} +(\texttt{u32} LE ordinal into the block vector) \cat{} \texttt{offset} +(\texttt{u32} LE)\}, strictly ascending by \texttt{id} bytes. +\end{center} + +The empty index is exactly 8 zero bytes. \texttt{offset} is the byte offset +of the envelope's \emph{first content byte} within the uncompressed payload +of the named block (the envelope's \texttt{u32} length prefix sits at +$\mathit{offset}-4$) --- well-defined by the id-leads property +(Requirement~\ref{req:binfmt:envelope}) and the block framing +(Section~\ref{sec:bundle:blocks}). + +\textbf{One slot per id.} Strict ascent makes duplicate ids unrepresentable: +an index payload containing two entries with equal id bytes is malformed. +Equivocation candidates (multiple envelopes claiming one +\texttt{OperationId}) live \emph{inside} the operation set's slot model +(core Chapter~6), never as duplicate index rows. + +\textbf{Staleness.} An index \emph{covers} a manifest iff its block vector +equals the manifest's deduplicated, canonically sorted +\texttt{operation\_roots} under \textbf{full \texttt{ChunkRef} equality} --- +every field, not just the content hash, since the index hands out its stored +references for reading. A reader that finds the index stale \MUST{} reject +it and rebuild from the blocks. + +A rejected or stale index is \textbf{never bundle corruption}: the index is +a non-canonical accelerator, and rebuild-from-blocks is always sound. + +Decoders \MUST{} reject: unsorted or duplicated blocks or entry ids, a block +reference whose kind is not \texttt{OperationEnvelopeBlock}, an entry whose +block ordinal is out of range, and trailing bytes. +\end{requirement} + +The core specification's commit-time \SHOULD{} --- refresh the index when +the operation log has ``grown significantly'' since it was built --- has an +\textbf{implementation-defined} threshold in v0: this document deliberately +does not pin a number, and conforming writers may use any policy (including +always or never refreshing at commit), because the index is non-canonical. + +\begin{openquestion} +Whether to pin a normative refresh threshold (e.g.\ a block-count or +byte-size ratio) or leave it permanently implementation-defined. Leaving it +open costs only lookup performance on stale-index bundles; pinning it too +early would constrain writers for no interoperability gain. Revisit when a +second writer implementation exists. +\end{openquestion} + +% =========================================================================== +\chapter{Extension Declaration Blobs and Edit Barriers} +\label{ch:barriers} + +The two opaque blobs a manifest \texttt{ExtensionDeclaration} carries --- +\texttt{affected\_object\_kinds} and \texttt{edit\_barriers} --- have their +internal byte form pinned here. This chapter \textbf{ratifies} P12-E1 +(the blob and barrier-tree byte form), P12-E2 (the recursion bound), and +P12-E3 (the barrier \texttt{ObjectKind} representation). Barrier +\emph{semantics} --- scope matching, prohibited-edit evaluation, +conservative treatment of unknown extensions --- are core Chapter~8 +(\sectionsc{Forward Compatibility and Edit Barriers}); this chapter pins +bytes only. + +These layouts are \textbf{regime (b)}: all counts and length prefixes are +\texttt{u64} little-endian --- a deliberate, golden-locked divergence from +the \texttt{u32} regime (Section~\ref{sec:conventions:regimes}). + +\section{Framing} +\label{sec:barriers:framing} + +\begin{description} + \item[element] $\mathrm{elem}(x)$ = \texttt{u64} LE byte length \cat{} + $x$'s canonical bytes. + \item[set] $\mathrm{set}(xs)$ = elements' canonical byte strings, sorted + ascending byte-lexicographic and \emph{deduplicated}; then \texttt{u64} + LE element count \cat{} each element as $\mathrm{elem}$. Order and + repetition in the source collection cannot affect the bytes; decoders + reject unsorted or duplicated elements. + \item[list] $\mathrm{list}(xs)$ = \texttt{u64} LE count \cat{} each + element as $\mathrm{elem}$, order-significant (used only for condition + subtrees). +\end{description} + +\begin{requirement} +\label{req:binfmt:ext-blobs} +\textbf{The two blobs (ratifies P12-E1).} The +\texttt{affected\_object\_kinds} blob is $\mathrm{set}$ of barrier +\texttt{ObjectKind} elements; the \texttt{edit\_barriers} blob is +$\mathrm{set}$ of \texttt{EditBarrier} elements. Both decode under +reject-never-normalize with trailing bytes forbidden. +\end{requirement} + +\begin{requirement} +\label{req:binfmt:object-kind-open} +\textbf{Barrier \texttt{ObjectKind} (ratifies P12-E3).} A barrier +\texttt{ObjectKind} is the \texttt{TypedObjectId} discriminant of +Section~\ref{sec:ids:typed-object-id}, encoded as \textbf{2 little-endian +bytes}. Decode is \textbf{open-value}: every \texttt{u16} decodes +successfully --- an unknown kind (a future core kind or an +extension-registered kind) is a \emph{value}, not a decode branch, and +simply never matches any object the reader knows. This is the format's +forward-compatibility stance for barrier kinds: unknown kinds are preserved +and re-emitted verbatim, never dropped or rejected. +\end{requirement} + +\section{\texttt{EditBarrier}} +\label{sec:barriers:barrier} + +An \texttt{EditBarrier}'s canonical bytes, positionally: + +\begin{center} +\texttt{scope} \cat{} $\mathrm{set}$(\texttt{affected\_object\_kinds}: +\texttt{ObjectKind}) \cat{} +$\mathrm{set}$(\texttt{prohibited\_operation\_kinds}: +\texttt{OperationKindTag}, Section~\ref{sec:ops:kind-tag}) \cat{} +\texttt{condition} +\end{center} + +\texttt{BarrierScope}: one discriminant byte, then the payload: + +\begingroup\small +\begin{longtable}{p{0.35in} p{1.35in} p{3.8in}} + \toprule + \textbf{Disc} & \textbf{Variant} & \textbf{Payload} \\ + \midrule + \endhead + \tablenums{0} & \texttt{WholeScore} & (none) \\ + \tablenums{1} & \texttt{Region} & \texttt{RegionId}, 16 big-endian bytes \\ + \tablenums{2} & \texttt{StaffInstance} & \texttt{StaffInstanceId}, 16 BE \\ + \tablenums{3} & \texttt{AnalysisLayer} & \texttt{AnalysisLayerId}, 16 BE \\ + \tablenums{4} & \texttt{ObjectSet} & $\mathrm{set}$ of + \texttt{TypedObjectId} canonical bytes (18/34 each) \\ + \tablenums{5} & \texttt{PitchSpace} & \texttt{u64} LE byte length \cat{} + the pitch-space id's NFC UTF-8 bytes; decoders reject non-NFC input \\ + \tablenums{6} & \texttt{TuningContext} & (none) \\ + \tablenums{7} & \texttt{Registered} & \texttt{BarrierScopeRegistryId}, + \texttt{u128} \emph{little-endian} (16 bytes) \\ + \bottomrule +\end{longtable} +\endgroup + +\texttt{BarrierCondition}: one discriminant byte, then the payload; the tree +is recursive: + +\begingroup\small +\begin{longtable}{p{0.35in} p{1.85in} p{3.3in}} + \toprule + \textbf{Disc} & \textbf{Variant} & \textbf{Payload} \\ + \midrule + \endhead + \tablenums{0} & \texttt{Always} & (none) \\ + \tablenums{1} & \texttt{ObjectExists} & \texttt{TypedObjectId} \\ + \tablenums{2} & \texttt{ObjectHasExtensionData} & \texttt{object} + (\texttt{TypedObjectId}) \cat{} \texttt{extension} + (\texttt{ExtensionRef}, \texttt{u128} LE, 16) \\ + \tablenums{3} & \texttt{All} & $\mathrm{list}$ of conditions \\ + \tablenums{4} & \texttt{Any} & $\mathrm{list}$ of conditions \\ + \tablenums{5} & \texttt{Not} & one $\mathrm{elem}$-framed condition \\ + \tablenums{6} & \texttt{Registered} & + \texttt{BarrierConditionRegistryId}, \texttt{u128} LE (16) \\ + \bottomrule +\end{longtable} +\endgroup + +Note the endianness: the barrier layer's registry identifiers +(\texttt{BarrierScopeRegistryId}, \texttt{BarrierConditionRegistryId}, +\texttt{ExtensionRef}) encode \texttt{u128} \emph{little-endian}, unlike the +big-endian identifier family of Chapter~\ref{ch:ids}. This is part of the +golden-locked regime-(b) surface. + +\begin{requirement} +\label{req:binfmt:condition-depth} +\textbf{Recursion bound (ratifies P12-E2).} +$\texttt{MAX\_CONDITION\_DEPTH} = \tablenums{64}$ is the normative bound on +\texttt{BarrierCondition} nesting. Decoders \MUST{} reject a condition tree +nested deeper than 64 levels of \texttt{All}/\texttt{Any}/\texttt{Not}; +writers \MUSTNOT{} emit one. The bound exists so adversarial bytes cannot +drive unbounded recursion; real barriers are depth 1--2. +\end{requirement} + +% =========================================================================== +\chapter{Schema Evolution} +\label{ch:evolution} + +The core specification delegates schema evolution to this document (core +Chapter~8, \sectionsc{Schema Versioning}: \emph{``Schema evolution is +governed by the Binary Format companion specification, which defines the +wire encoding for each schema version''}). This chapter defines the wire +rules for schema major~0. + +\section{The Chunk-Level Gate} +\label{sec:evolution:gate} + +Every chunk declares a \texttt{SchemaVersion} (major, minor) in its +\texttt{ChunkRef} and the superblock declares the manifest's. The gate: + +\begin{itemize} + \item \textbf{Major} = incompatible. A reader \MUST{} reject a chunk whose + schema major it does not support. The current major is~\tablenums{0}. + \item \textbf{Minor} = additive. v0 readers verify the major only; the + minor is a \emph{record}, not a gate --- but it is a mandatory record: + a writer \MUST{} raise the chunk schema minor when it emits any + discriminant appended after the minor it otherwise declares, so that a + decode failure on an unknown appended discriminant is attributable to a + version skew rather than corruption. +\end{itemize} + +\section{What ``Additive'' Means Here} +\label{sec:evolution:additive} + +Under the frozen-layout rule (Requirement~\ref{req:binfmt:frozen-layout}), +positional struct layouts admit \emph{no} additive change: there is no +optional-field or trailing-field mechanism at the value level. In +particular, and stated honestly: \textbf{the manifest body cannot grow a +fourteenth field within major~0}. The manifest is a positional struct whose +decoder rejects trailing bytes, so manifest-body extension is +major-gated in v0 exactly like every other struct. + +The \emph{only} minor-additive mechanism in schema major~0 is +\textbf{appending discriminants to open vocabularies}: + +\begin{itemize} + \item \texttt{OperationKind}: append at ${\geq}\,\tablenums{24}$ + (Requirement~\ref{req:binfmt:kind-discriminants}); + \item \texttt{OperationKindTag}: append at ${\geq}\,\tablenums{24}$ + (Requirement~\ref{req:binfmt:kind-tag}); + \item \texttt{OperationPayload}: append at ${\geq}\,\tablenums{4}$ + (Section~\ref{sec:ops:payload}); + \item the value-layer unions enumerated as closed in + Requirement~\ref{req:binfmt:frozen-layout} may gain appended variants + only through a ratified revision of this document, also minor-additive. +\end{itemize} + +\texttt{ChunkKind} is \textbf{closed} --- it has no \texttt{Registered} +variant and no append story inside major~0, because its discriminant enters +every chunk's hash preimage; a new chunk kind is a format-major event. + +Separately, many vocabularies carry a \texttt{Registered} \emph{escape +variant} (\texttt{RepairKind}, \texttt{ReanchorReason}, +\texttt{PreconditionFailureReason}, \texttt{IntegrityAnomalyKind}, +\texttt{ReplicaAnomalyReason}, \texttt{TransactionCategory}, +\texttt{ResolutionAction}, \texttt{ConflictKind} via +\texttt{ExtensionConflict}, \texttt{BarrierScope}/\texttt{BarrierCondition}, +\texttt{TieClass}, \texttt{StaffGroupKind}, \texttt{PitchSpacePosition}, +\texttt{SpellingNominal}, \texttt{TypedObjectId}, and the barrier +\texttt{ObjectKind}'s open value space). Extension through an escape variant +is \textbf{not} a schema change at all: the wire form is already defined. + +\section{Migration Discipline} +\label{sec:evolution:migration} + +A schema-\textsc{major} bump \MUST{} ship with a migration documented in +this document's revision history (Chapter~\ref{ch:history}): what changed, +byte-for-byte, and the deterministic translation from the old form. The +precedent is the v0${\to}$v1 operation-payload migration, which lives in the +Operation Catalog (\sectionsc{v0 $\rightarrow$ v1 Payload Migration}) +because it was semantic rather than byte-level +(Section~\ref{sec:ops:v0}); byte-level migrations belong here. + +\begin{openquestion} +Whether the \texttt{u64}/\texttt{u32} length-prefix divergence between +regimes (a) and (b) (Section~\ref{sec:conventions:regimes}) gets unified at +the next schema major. Unification would simplify decoders and cost one +coordinated re-lock of the barrier-blob and resolved-layout goldens; +leaving it is free but permanent. Decide when the first major bump is +scheduled for other reasons --- the divergence alone does not justify one. +\end{openquestion} + +% =========================================================================== +\chapter{Non-Canonical Pinned Encodings} +\label{ch:noncanon} + +\textbf{Non-canonical} means: never part of document identity. Nothing in +this chapter enters a content hash that canonical state depends on, and all +of it can be discarded and rebuilt without changing the document. The +encodings are pinned for \emph{reproducibility} --- byte-equal conformance +claims and cache correctness --- not for durability. + +\section{\texttt{LayoutObjectId} (\texttt{MUSCLOID})} +\label{sec:noncanon:muscloid} + +Governed by core requirement \texttt{req:layoutir:object-id-derivation}. +$\texttt{LayoutObjectId} = \mathrm{trunc128}(\mathrm{BLAKE3}( +\texttt{MUSCLOID} \cat \mathit{inputs}))$, with three preimage shapes: + +\begin{description} + \item[stable] $\mathit{inputs}$ = the source + \texttt{TypedObjectId}'s canonical bytes. + \item[manifestation] $\mathit{inputs}$ = source canonical bytes \cat{} + \texttt{RegionId} canonical bytes (16 BE) --- distinct regions give one + source distinct manifestation ids. + \item[synthesized] $\mathit{inputs}$ = source canonical bytes \cat{} + five \texttt{u64} LE words: the \texttt{SynthesisKind} discriminant, + the registry id's high and low 64 bits (zero unless + \texttt{Registered}), and the instance key's high and low 64 bits. +\end{description} + +\texttt{SynthesisKind} discriminants: \tablenums{0} CancellationAccidental, +\tablenums{1} KeySignatureNatural, \tablenums{2} GeneratedRest, +\tablenums{3} EngravedBreak, \tablenums{4} MultimeasureRest, +\tablenums{5} Cautionary, \tablenums{6} Registered. + +\section{\texttt{ResolvedLayoutIR} Canonical Output} +\label{sec:noncanon:resolved} + +The resolved layout's canonical bytes are the \textbf{byte-equal conformance +surface} of core Chapter~7: two conforming engravers given the same score, +profile, and glyph catalog must produce identical bytes. The encoding is +regime (b) --- \texttt{u64} LE counts and length prefixes --- with every +geometric coordinate quantized to the $1/1024$ staff-space grid as a +\texttt{QuantizedCoord} (8 LE bytes; Chapter~\ref{ch:primitives}); +a non-finite or out-of-range coordinate is a determinism violation and +\MUST{} be rejected, never normalized. The top-level section order is: +source score version \cat{} pages \cat{} glyphs \cat{} strokes \cat{} +engraving decisions \cat{} glyph-catalog identity; within each section, +elements carry their provenance (built on \texttt{MUSCLOID} ids) and their +quantized geometry. The full leaf grammar is pinned by the reference +implementation's golden and round-trip anchors (Chapter~\ref{ch:goldens}) +rather than restated field-by-field here, because the surface is +non-canonical and evolves with the engraving feature set. + +\section{Layout Caches and Font Metrics} +\label{sec:noncanon:caches} + +\texttt{LayoutCache} chunks (\texttt{ChunkKind} \tablenums{6}) hold cached +layout artifacts; they are \textbf{always discardable}, and a reader +\MUSTNOT{} treat a missing, stale, or undecodable layout cache as bundle +corruption. The glyph-metrics identity hash is domain-separated under +\texttt{MUSCFNTM} (a reserved built-in, non-canonical tag): it names the +metrics a layout was computed against, so caches and conformance runs can +detect font drift; it never enters canonical state. + +% =========================================================================== +\chapter{Golden Anchor Registry} +\label{ch:goldens} + +The conformance contract binding this document to the reference +implementation. Three lock classes: + +\begin{description} + \item[literal-byte] the test asserts exact expected bytes or exact + discriminant literals --- the strongest lock; + \item[round-trip] the test asserts $\mathrm{decode} \circ \mathrm{encode} = + \mathrm{id}$ and re-encode byte-stability over a corpus; + \item[canonical-equality] the test asserts that semantically equal values + encode identically (order-independence, reduction identities). +\end{description} + +The first block restates the anchors already recorded in the core +specification's \texttt{app:bytes} \sectionsc{Reference-implementation +locks} table (which governs them); the second block adds the anchors this +document introduces, which \texttt{app:bytes} predates. + +Each anchor names its crate file, then the test on a second line. + +\begingroup\footnotesize +\begin{longtable}{>{\raggedright\arraybackslash}p{1.5in} >{\raggedright\arraybackslash}p{3.0in} p{1.0in}} + \toprule + \textbf{Layout} & \textbf{Anchor} & \textbf{Class} \\ + \midrule + \endhead + \multicolumn{3}{l}{\emph{Imported from \texttt{app:bytes}:}} \\ + \addlinespace[2pt] + \texttt{TypedObjectId} & + \texttt{epiphany-core/src/ids.rs}\newline + \texttt{::typed\ub object\ub id\ub byte\ub form\ub is\ub locked} & + literal-byte \\ + Promoted \texttt{VoiceId} & + \texttt{epiphany-core/src/graph.rs}\newline + \texttt{::promoted\ub voice\ub id\ub byte\ub form\ub is\ub locked} & + literal-byte \\ + System \texttt{PitchId} & + \texttt{epiphany-core/src/pitch.rs}\newline + \texttt{::system\ub pitch\ub id\ub byte\ub form\ub is\ub locked} & + literal-byte \\ + \texttt{IntegrityAnomalyId} & + \texttt{epiphany-ops/src/anomaly.rs}\newline + \texttt{::integrity\ub anomaly\ub id\ub byte\ub form\ub is\ub locked} & + literal-byte \\ + \texttt{ObjectKind} (ops) & + \texttt{epiphany-ops/src/support.rs}\newline + \texttt{::object\ub kind\ub discriminants\ub are\ub golden} & + literal-byte \\ + \texttt{ChunkKind} & + \texttt{epiphany-bundle/src/chunk.rs}\newline + \texttt{::chunk\ub kind\ub discriminants\ub are\ub golden} & + literal-byte \\ + \texttt{Compression\allowbreak Algorithm} & + \texttt{epiphany-bundle/src/chunk.rs}\newline + \texttt{::compression\ub algorithm\ub encoding\ub is\ub golden} & + literal-byte \\ + \texttt{ProfileId} & + \texttt{epiphany-bundle/src/superblock.rs}\newline + \texttt{::profile\ub id\ub discriminants\ub are\ub golden} & + literal-byte \\ + \texttt{ManifestId} & + \texttt{epiphany-bundle/src/ids.rs}\newline + \texttt{::manifest\ub id\ub is\ub content\ub derived\ub and\ub deterministic} & + canonical-equality \\ + \texttt{TransactionCategory} & + \texttt{epiphany-ops/src/payload.rs}\newline + \texttt{::transaction\ub category\ub discriminants\ub are\ub golden} & + literal-byte \\ + \texttt{ResolutionAction} & + \texttt{epiphany-ops/src/conflict.rs}\newline + \texttt{::resolution\ub action\ub discriminants\ub are\ub golden} & + literal-byte \\ + \texttt{BlobId} & + \texttt{epiphany-determinism/src/hash.rs}\newline + \texttt{ContentHash::of\ub blob} tests & + round-trip \\ + \texttt{RationalTime} & + \texttt{epiphany-core/src/time.rs}\newline + \texttt{::equal\ub rationals\ub encode\ub identically} & + canonical-equality \\ + \texttt{MUSCLOID} derivation & + \texttt{epiphany-layout-ir/src/provenance.rs}\newline + \texttt{::stable\ub id\ub uses\ub the\ub ratified\ub muscloid\ub derivation} & + literal-byte \\ + Domain-tag spellings & + \texttt{epiphany-determinism/src/domain.rs}\newline + \texttt{::exact\ub tag\ub spellings\ub match\ub spec} & + literal-byte \\ + \addlinespace[4pt] + \multicolumn{3}{l}{\emph{Added by this document:}} \\ + \addlinespace[2pt] + \texttt{OperationKind} discriminants (\S\ref{sec:ops:kinds}) & + \texttt{epiphany-ops/src/payload.rs}\newline + \texttt{::operation\ub kind\ub wire\ub discriminants\ub are\ub golden} & + literal-byte \\ + \texttt{OperationPayload} discriminants (\S\ref{sec:ops:payload}) & + \texttt{epiphany-ops/src/payload.rs}\newline + \texttt{::operation\ub payload\ub discriminants\ub are\ub golden} & + literal-byte \\ + \texttt{OperationKindTag} (\S\ref{sec:ops:kind-tag}) & + \texttt{epiphany-ops/src/payload.rs}\newline + \texttt{::operation\ub kind\ub tag\ub decode\ub mirrors\ub encode\ub exactly} & + round-trip \\ + \texttt{ResolveEquivocation} payload (\S\ref{sec:ops:payload}) & + \texttt{epiphany-ops/src/payload.rs}\newline + \texttt{::resolve\ub equivocation\ub payload\ub encodes\ub target\ub then\ub hash} & + literal-byte \\ + \texttt{OperationStamp} (\S\ref{sec:ops:stamp}) & + \texttt{epiphany-ops/src/stamp.rs}\newline + \texttt{::stamp\ub encode\ub is\ub stable} & + literal-byte \\ + \texttt{CausalContext} (\S\ref{sec:ops:causal}) & + \texttt{epiphany-ops/src/causal.rs}\newline + \texttt{::canonical\ub encoding\ub is\ub build\ub order\ub independent} & + canonical-equality \\ + \texttt{MaterializedState} (\S\ref{sec:ops:materialized}) & + \texttt{epiphany-ops/src/decode.rs}\newline + \texttt{::reduced\ub states\ub decode\ub and\ub reencode} & + round-trip \\ + Operation-index payload (\S\ref{sec:bundle:opindex}) & + \texttt{epiphany-bundle/src/opindex.rs}\newline + \texttt{::payload\ub encoding\ub is\ub golden} & + literal-byte \\ + Fixed header (\S\ref{sec:bundle:header}) & + \texttt{epiphany-bundle/src/header.rs}\newline + \texttt{::header\ub round\ub trips} & + round-trip \\ + Superblock (\S\ref{sec:bundle:superblock}) & + \texttt{epiphany-bundle/src/superblock.rs}\newline + \texttt{::superblock\ub round\ub trips\ub through\ub 256\ub bytes} & + round-trip \\ + Chunk hash preimage (\S\ref{sec:bundle:chunks}) & + \texttt{epiphany-bundle/src/chunk.rs}\newline + \texttt{::hash\ub preimage\ub matches\ub the\ub spec\ub layout} & + literal-byte \\ + \texttt{ChunkRef} (\S\ref{sec:bundle:chunkref}) & + \texttt{epiphany-bundle/src/chunk.rs}\newline + \texttt{::chunk\ub ref\ub round\ub trips} & + round-trip \\ + Manifest body (\S\ref{sec:bundle:manifest}) & + \texttt{epiphany-bundle/src/manifest.rs}\newline + \texttt{::re\ub encode\ub is\ub byte\ub identical}, + \texttt{::manifest\ub round\ub trips}, + \texttt{::duplicate\ub roots\ub collapse\ub on\ub encode}, + \texttt{::semver\ub orders\ub numerically\ub not\ub byte\ub wise} & + round-trip + canonical-equality \\ + Barrier kinds blob (\S\ref{sec:barriers:framing}) & + \texttt{epiphany-layout-ir/src/barrier.rs}\newline + \texttt{::affected\ub object\ub kinds\ub blob\ub bytes\ub are\ub golden} & + literal-byte \\ + Barriers blob (\S\ref{sec:barriers:barrier}) & + \texttt{epiphany-layout-ir/src/barrier.rs}\newline + \texttt{::edit\ub barriers\ub blob\ub bytes\ub are\ub golden} & + literal-byte \\ + Barrier variants (\S\ref{sec:barriers:barrier}) & + \texttt{epiphany-layout-ir/src/barrier.rs}\newline + \texttt{::every\ub scope\ub and\ub condition\ub variant\ub round\ub trips\ub byte\ub identically} & + round-trip \\ + Whole-\texttt{Score} codec (\S\ref{sec:values:score}) & + \texttt{epiphany-core/src/codec.rs}\newline + \texttt{::generator\ub scores\ub round\ub trip}, + \texttt{::trailing\ub and\ub truncated\ub bytes\ub are\ub rejected} & + round-trip \\ + \texttt{CanonicalValue} seam (\S\ref{sec:values:canonical-value}) & + \texttt{epiphany-core/src/codec.rs}\newline + \texttt{::value\ub types\ub round\ub trip\ub over\ub generator\ub corpus} & + round-trip \\ + \bottomrule +\end{longtable} +\endgroup + +Struct \emph{bodies} in Chapter~\ref{ch:values} are round-trip-locked rather +than literal-byte-locked: their byte identity follows from the frozen +positional rule plus the literal-byte locks on every discriminant and leaf +they embed. A future cross-implementation decoder test (the deferred +conformance harness) should add literal-byte vectors for the representative +layouts of Section~\ref{sec:values:representative}. + +% =========================================================================== +\chapter{Revision History} +\label{ch:history} + +\begin{longtable}{p{2cm} p{2.5cm} p{9cm}} + \toprule + \textbf{Date} & \textbf{Section} & \textbf{Change} \\ + \midrule + \endhead + \today & All & 0.1.0 --- Initial companion: ratifies the previously + provisional whole-\texttt{Score} value codec (core P11-4), operation-layer + wire forms (ops), bundle physical layout (bundle P11-D2/D4/D5), + operation-index payload (P12-D1), and extension-blob/edit-barrier byte + forms (P12-E1/E2/E3); pins the no-varint rule, the frozen-positional + schema-evolution keystone, and the id-leads envelope property; + \texttt{SnapshotId} derivation deferred (open question). \\ + \bottomrule +\end{longtable} + +\end{document}