diff --git a/crates/epiphany-bundle/src/vectors.rs b/crates/epiphany-bundle/src/vectors.rs index c99c5cf..e004801 100644 --- a/crates/epiphany-bundle/src/vectors.rs +++ b/crates/epiphany-bundle/src/vectors.rs @@ -14,13 +14,17 @@ use crate::{block, DecodeError}; use epiphany_determinism::{ChunkId, ContentHash}; /// `(surface, verdict, class, name, bytes)` — see `epiphany_ops::vectors`. -pub type DecodeVector = ( - &'static str, - &'static str, - &'static str, - &'static str, - Vec, -); +pub type DecodeVector = (&'static str, &'static str, &'static str, String, Vec); + +fn row( + surface: &'static str, + verdict: &'static str, + class: &'static str, + name: impl Into, + bytes: Vec, +) -> DecodeVector { + (surface, verdict, class, name.into(), bytes) +} /// A `ChunkRef` encodes as 95 bytes: id 32, kind 1, schema 4, offset 8, /// compressed_length 8, uncompressed_length 8, compression 2, hash 32. @@ -57,18 +61,24 @@ pub fn decode_vectors() -> Vec { const MAN: &str = "bundle.manifest"; let doc = DocumentId([5; 16]); let empty = Manifest::empty(doc); - v.push((MAN, "accept", "-", "empty_manifest", empty.encode())); + v.push(row(MAN, "accept", "-", "empty_manifest", empty.encode())); let mut one = Manifest::empty(doc); one.operation_roots.push(block_ref(0x11, 576)); let one_bytes = one.encode(); - v.push((MAN, "accept", "-", "one_operation_root", one_bytes.clone())); + v.push(row( + MAN, + "accept", + "-", + "one_operation_root", + one_bytes.clone(), + )); // Body edit: `manifest_id` is derived from the body, so it no longer matches. let mut id_mismatch = one_bytes.clone(); let last = id_mismatch.len() - 1; id_mismatch[last] ^= 0xFF; - v.push(( + v.push(row( MAN, "reject", "manifest-id-mismatch", @@ -84,7 +94,13 @@ pub fn decode_vectors() -> Vec { two.operation_roots.push(block_ref(0x11, 576)); two.operation_roots.push(block_ref(0x22, 1024)); let two_bytes = two.encode(); - v.push((MAN, "accept", "-", "two_operation_roots", two_bytes.clone())); + v.push(row( + MAN, + "accept", + "-", + "two_operation_roots", + two_bytes.clone(), + )); // Locate the first root: id(16) + document_id(16) + lineage option tag(1) // + generation(8) + roots count(4). @@ -93,7 +109,7 @@ pub fn decode_vectors() -> Vec { let body = &reordered[16..]; let redone = ManifestId::derive(doc, 0, body); reordered[0..16].copy_from_slice(&redone.0.to_be_bytes()); - v.push(( + v.push(row( MAN, "reject", "non-canonical-vec-order", @@ -103,7 +119,7 @@ pub fn decode_vectors() -> Vec { let mut trailing = one_bytes.clone(); trailing.push(0); - v.push(( + v.push(row( MAN, "reject", "trailing-bytes", @@ -114,12 +130,18 @@ pub fn decode_vectors() -> Vec { // --- OperationIndex ---------------------------------------------------- const IDX: &str = "bundle.operation_index"; let empty_index = OperationIndex::build(&[]).expect("empty").encode(); - v.push((IDX, "accept", "-", "empty_index", empty_index.clone())); + v.push(row(IDX, "accept", "-", "empty_index", empty_index.clone())); let one_index = OperationIndex::build(&[(block_ref(0x11, 576), vec![([2; 16], 8)])]) .expect("one block") .encode(); - v.push((IDX, "accept", "-", "one_block_one_entry", one_index.clone())); + v.push(row( + IDX, + "accept", + "-", + "one_block_one_entry", + one_index.clone(), + )); let two_index = OperationIndex::build(&[ (block_ref(0x11, 576), vec![([1; 16], 8), ([3; 16], 40)]), @@ -127,7 +149,7 @@ pub fn decode_vectors() -> Vec { ]) .expect("two blocks") .encode(); - v.push((IDX, "accept", "-", "two_blocks", two_index.clone())); + v.push(row(IDX, "accept", "-", "two_blocks", two_index.clone())); // The lenient-compression class. `CompressionAlgorithm::None` used to ignore // this byte, so `[0, 0xFF]` and `[0, 0]` decoded alike: two byte strings, one @@ -135,7 +157,7 @@ pub fn decode_vectors() -> Vec { // (Push 5 / P3; `req:binfmt:compression-none-parameter`.) let mut lenient = one_index.clone(); lenient[4 + CHUNK_REF_COMPRESSION_PARAM] = 0xFF; - v.push(( + v.push(row( IDX, "reject", "lenient-sub-codec", @@ -144,7 +166,7 @@ pub fn decode_vectors() -> Vec { )); // Strict ascent, checked per-site (there is no guard here to catch it). - v.push(( + v.push(row( IDX, "reject", "non-canonical-vec-order", @@ -155,7 +177,7 @@ pub fn decode_vectors() -> Vec { // Entries are 24 bytes: id(16) + block(4) + offset(4). They begin after the // blocks and their count. let entries_at = 4 + 2 * CHUNK_REF_LEN + 4; - v.push(( + v.push(row( IDX, "reject", "non-canonical-vec-order", @@ -167,7 +189,7 @@ pub fn decode_vectors() -> Vec { let mut bad_ordinal = one_index.clone(); let ordinal_at = 4 + CHUNK_REF_LEN + 4 + 16; bad_ordinal[ordinal_at..ordinal_at + 4].copy_from_slice(&7u32.to_le_bytes()); - v.push(( + v.push(row( IDX, "reject", "block-ordinal-out-of-range", @@ -178,7 +200,7 @@ pub fn decode_vectors() -> Vec { // A block reference whose kind is not an operation-envelope block. let mut wrong_kind = one_index.clone(); wrong_kind[4 + 32] = ChunkKind::Snapshot.discriminant(); - v.push(( + v.push(row( IDX, "reject", "wrong-chunk-kind", @@ -188,7 +210,7 @@ pub fn decode_vectors() -> Vec { let mut idx_trailing = empty_index.clone(); idx_trailing.push(0); - v.push(( + v.push(row( IDX, "reject", "trailing-bytes", @@ -201,11 +223,11 @@ pub fn decode_vectors() -> Vec { let payloads: Vec> = vec![vec![0xAA; 8], vec![0xBB; 13]]; let packed = block::pack_operation_blocks(&payloads); let good = packed.first().expect("one block").clone(); - v.push((BLK, "accept", "-", "two_envelopes", good.clone())); + v.push(row(BLK, "accept", "-", "two_envelopes", good.clone())); let mut blk_trailing = good.clone(); blk_trailing.push(0); - v.push(( + v.push(row( BLK, "reject", "trailing-bytes", @@ -215,7 +237,7 @@ pub fn decode_vectors() -> Vec { let mut blk_truncated = good.clone(); blk_truncated.pop(); - v.push(( + v.push(row( BLK, "reject", "truncated", @@ -227,7 +249,7 @@ pub fn decode_vectors() -> Vec { // reject on the count, not pre-allocate for it. let mut huge = good.clone(); huge[0..4].copy_from_slice(&u32::MAX.to_le_bytes()); - v.push(( + v.push(row( BLK, "reject", "count-exceeds-remaining", diff --git a/crates/epiphany-layout-ir/src/barrier.rs b/crates/epiphany-layout-ir/src/barrier.rs index 6ae46f9..bbb5f02 100644 --- a/crates/epiphany-layout-ir/src/barrier.rs +++ b/crates/epiphany-layout-ir/src/barrier.rs @@ -1127,14 +1127,16 @@ mod tests { #[test] fn a_barrier_prohibiting_every_operation_tag_round_trips() { use epiphany_ops::OperationKindRegistryId; - let mut tags: Vec = (0u8..=30) - .filter(|d| *d != 16) - .map(|d| { - epiphany_determinism::CanonicalDecode::decode_canonical(&[d][..]) - .unwrap_or_else(|e| panic!("tag {d} must decode: {e:?}")) - }) - .collect(); + // From the production vocabulary, not a spelled range. `(0u8..=30)` would + // silently stop covering the newest tag the day one is appended — which + // is precisely how this surface broke. + let mut tags: Vec = OperationKindTag::PAYLOAD_FREE.to_vec(); tags.push(OperationKindTag::Registered(OperationKindRegistryId(7))); + assert!( + tags.len() > 30, + "the barrier round-trip must cover every tag, got {}", + tags.len() + ); let barrier = EditBarrier { scope: BarrierScope::WholeScore, diff --git a/crates/epiphany-ops/DECISIONS.md b/crates/epiphany-ops/DECISIONS.md index 9f88352..8f939a6 100644 --- a/crates/epiphany-ops/DECISIONS.md +++ b/crates/epiphany-ops/DECISIONS.md @@ -1569,3 +1569,45 @@ fails if one goes missing. Gated in the conformance suite as `[7d]`, and drift-locked: the committed file must equal `vectors::render()`, so a wire-format change lands in the diff. + +### P4 follow-up: one production-owned tag vocabulary, and the corpus ratified + +Two review findings, both real. + +**1. The tag-omission failure could recur.** After the P4 fix there were *still* +four hand-maintained lists of tags — a test-local `all_tags()`, a barrier test +spelling `0u8..=30`, a fuzz corpus naming five, a vector corpus naming four — +plus a malformed-bytes test asserting `31` rejects. A future tag 31 added to +`discriminant()` and omitted from `decode_canonical()` would have left every one +of them green, and the malformed test would have *locked* it, exactly as the `30` +version did. + +`operation_kind_tag_vocabulary!` is now the single source: it generates +`discriminant`, `from_discriminant`, and `OperationKindTag::PAYLOAD_FREE` from +one list. The generated `discriminant` match is **exhaustive over the enum**, so +a variant added to `OperationKindTag` and not to the macro **fails to compile** +(verified). Everything downstream reads `PAYLOAD_FREE`: the decoder, the fuzz +corpus (all 31 tags, not five), the conformance vectors (an accept vector per +tag, so 65 vectors not 37), and the edit-barrier round-trip. Every "one past the +vocabulary" constant is *computed*, never spelled — a spelled constant is the +trap that springs on whoever appends the next tag. + +Verified end to end by adding a hypothetical tag 31: +- to the enum only → **compile error**; +- to the enum and the macro → compiles, decodes, and every derived check passes + *because* they read `PAYLOAD_FREE` — while the committed corpus's drift lock + and its stale "one past the vocabulary" reject vector both fail, forcing the + new vectors into the diff. + +There is no path where a new tag leaves everything green. + +**2. The corpus called itself normative; the spec said it was deferred.** Binary +Format's §"About This Companion" listed the cross-implementation decoder test +among things the document does *not* cover, and its Golden Anchor Registry called +it "the deferred conformance harness". Both now ratify it: +`req:binfmt:decode-vectors` and §"The Decode Vector Corpus" state that a +conforming decoder MUST accept every `accept` vector, MUST reject every `reject` +vector, and MUST re-encode an accepted value to its own bytes — and that +**accepting a `reject` vector and then normalizing it is accepting it**. Binary +Format 0.8.0 → 0.9.0. The wire-format fuzzer stays an implementation deliverable. +The corpus header now cites the requirement rather than asserting one. diff --git a/crates/epiphany-ops/src/fuzz.rs b/crates/epiphany-ops/src/fuzz.rs index 96e83ac..e8b2d0f 100644 --- a/crates/epiphany-ops/src/fuzz.rs +++ b/crates/epiphany-ops/src/fuzz.rs @@ -707,16 +707,17 @@ fn build_decode_corpus(rng: &mut SplitMix64) -> DecodeCorpus { exactly those branches" ); - let tags = [ - OperationKindTag::InsertEvent, - OperationKindTag::Transpose, - OperationKindTag::TransposeInterval, - OperationKindTag::DeleteRepeatStructure, - OperationKindTag::Registered(crate::OperationKindRegistryId(0x0123_4567_89AB_CDEF)), - ] - .iter() - .map(|t| t.to_canonical_bytes()) - .collect(); + // EVERY tag, from the production vocabulary — not a hand-picked five. A tag + // named nowhere is a tag whose decoder is never exercised, which is how + // `TransposeInterval` encoded to a byte its own decoder rejected. + let tags = OperationKindTag::PAYLOAD_FREE + .iter() + .copied() + .chain(std::iter::once(OperationKindTag::Registered( + crate::OperationKindRegistryId(0x0123_4567_89AB_CDEF), + ))) + .map(|t| t.to_canonical_bytes()) + .collect(); DecodeCorpus { states, tags } } diff --git a/crates/epiphany-ops/src/payload.rs b/crates/epiphany-ops/src/payload.rs index e73e257..dedf68c 100644 --- a/crates/epiphany-ops/src/payload.rs +++ b/crates/epiphany-ops/src/payload.rs @@ -418,45 +418,88 @@ pub enum OperationKindTag { TransposeInterval, } -impl OperationKindTag { - fn discriminant(&self) -> u8 { - match self { - OperationKindTag::InsertEvent => 0, - OperationKindTag::DeleteEvent => 1, - OperationKindTag::ModifyEvent => 2, - OperationKindTag::RespellPitch => 3, - OperationKindTag::Transpose => 4, - OperationKindTag::CreateCrossCutting => 5, - OperationKindTag::DeleteCrossCutting => 6, - OperationKindTag::ModifyCrossCutting => 7, - OperationKindTag::ChangeRegionTimeModel => 8, - OperationKindTag::InsertRegion => 9, - OperationKindTag::DeleteRegion => 10, - OperationKindTag::InsertStaffInstance => 11, - OperationKindTag::DeleteStaffInstance => 12, - OperationKindTag::SetUserSystemBreak => 13, - OperationKindTag::SetUserPageBreak => 14, - OperationKindTag::DeclareTransaction => 15, - OperationKindTag::Registered(_) => 16, - OperationKindTag::InsertIdentifiedPitch => 17, - OperationKindTag::DeleteIdentifiedPitch => 18, - OperationKindTag::ModifyIdentifiedPitch => 19, - OperationKindTag::CreateVoice => 20, - OperationKindTag::DeleteVoice => 21, - OperationKindTag::SetMetadata => 22, - OperationKindTag::SetMetricGrid => 23, - // Phase-3 first tranche; appended past the golden-locked 0..=23. - OperationKindTag::InsertStaff => 24, - OperationKindTag::SetTimeSignature => 25, - OperationKindTag::SetTempoSegment => 26, - OperationKindTag::SetStaffLayout => 27, - // Schema-major-2 revision; appended past the Phase-3 24..=27. - OperationKindTag::CreateRepeatStructure => 28, - OperationKindTag::DeleteRepeatStructure => 29, - // Push 4a; appended past 29. - OperationKindTag::TransposeInterval => 30, +/// The discriminant of [`OperationKindTag::Registered`], the one tag that +/// carries a payload. It sits inside the payload-free range, so it is named +/// rather than derived. +pub const REGISTERED_TAG_DISCRIMINANT: u8 = 16; + +/// The single source of truth for the payload-free tag vocabulary. +/// +/// Generates the discriminant mapping, its inverse, and +/// [`OperationKindTag::PAYLOAD_FREE`] from one list. The generated +/// `discriminant` match is **exhaustive over the enum**, so a variant added to +/// `OperationKindTag` without being added here fails to compile — and because +/// the decoder, the fuzz corpus, the conformance vectors, and the edit-barrier +/// round-trip all read `PAYLOAD_FREE`, adding it here reaches every one of them. +/// +/// This exists because it did not. Push 4a added `TransposeInterval` to a +/// hand-written `discriminant` match and to nothing else: the decoder rejected +/// its own encoding, an edit barrier naming it could not be reopened, and four +/// separate hand-maintained lists — two of them asserting the tag was *unknown* +/// — stayed green (Push 5 / P4). +macro_rules! operation_kind_tag_vocabulary { + ($($variant:ident = $disc:literal),+ $(,)?) => { + impl OperationKindTag { + /// Every payload-free tag, in discriminant order. [`Registered`] + /// is excluded: it carries an id and has no bare encoding. + /// + /// [`Registered`]: OperationKindTag::Registered + pub const PAYLOAD_FREE: &'static [OperationKindTag] = + &[$(OperationKindTag::$variant),+]; + + /// The tag's one-byte wire discriminant + /// (`req:binfmt:kind-tag`; append-only). + pub fn discriminant(&self) -> u8 { + match self { + $(OperationKindTag::$variant => $disc,)+ + OperationKindTag::Registered(_) => REGISTERED_TAG_DISCRIMINANT, + } + } + + /// The payload-free tag for `discriminant`, or `None` if it names + /// no tag. `Registered` is never returned: it is decoded separately, + /// with its id. + fn from_discriminant(discriminant: u8) -> Option { + Some(match discriminant { + $($disc => OperationKindTag::$variant,)+ + _ => return None, + }) + } } - } + }; +} + +operation_kind_tag_vocabulary! { + InsertEvent = 0, + DeleteEvent = 1, + ModifyEvent = 2, + RespellPitch = 3, + Transpose = 4, + CreateCrossCutting = 5, + DeleteCrossCutting = 6, + ModifyCrossCutting = 7, + ChangeRegionTimeModel = 8, + InsertRegion = 9, + DeleteRegion = 10, + InsertStaffInstance = 11, + DeleteStaffInstance = 12, + SetUserSystemBreak = 13, + SetUserPageBreak = 14, + DeclareTransaction = 15, + InsertIdentifiedPitch = 17, + DeleteIdentifiedPitch = 18, + ModifyIdentifiedPitch = 19, + CreateVoice = 20, + DeleteVoice = 21, + SetMetadata = 22, + SetMetricGrid = 23, + InsertStaff = 24, + SetTimeSignature = 25, + SetTempoSegment = 26, + SetStaffLayout = 27, + CreateRepeatStructure = 28, + DeleteRepeatStructure = 29, + TransposeInterval = 30, } impl CanonicalEncode for OperationKindTag { @@ -481,7 +524,7 @@ impl CanonicalDecode for OperationKindTag { expected: 1, actual: 0, })?; - if tag == 16 { + if tag == REGISTERED_TAG_DISCRIMINANT { let arr: [u8; 16] = rest.try_into().map_err(|_| DecodeError::UnexpectedLength { expected: 17, actual: bytes.len(), @@ -496,43 +539,9 @@ impl CanonicalDecode for OperationKindTag { actual: bytes.len(), }); } - Ok(match tag { - 0 => OperationKindTag::InsertEvent, - 1 => OperationKindTag::DeleteEvent, - 2 => OperationKindTag::ModifyEvent, - 3 => OperationKindTag::RespellPitch, - 4 => OperationKindTag::Transpose, - 5 => OperationKindTag::CreateCrossCutting, - 6 => OperationKindTag::DeleteCrossCutting, - 7 => OperationKindTag::ModifyCrossCutting, - 8 => OperationKindTag::ChangeRegionTimeModel, - 9 => OperationKindTag::InsertRegion, - 10 => OperationKindTag::DeleteRegion, - 11 => OperationKindTag::InsertStaffInstance, - 12 => OperationKindTag::DeleteStaffInstance, - 13 => OperationKindTag::SetUserSystemBreak, - 14 => OperationKindTag::SetUserPageBreak, - 15 => OperationKindTag::DeclareTransaction, - 17 => OperationKindTag::InsertIdentifiedPitch, - 18 => OperationKindTag::DeleteIdentifiedPitch, - 19 => OperationKindTag::ModifyIdentifiedPitch, - 20 => OperationKindTag::CreateVoice, - 21 => OperationKindTag::DeleteVoice, - 22 => OperationKindTag::SetMetadata, - 23 => OperationKindTag::SetMetricGrid, - 24 => OperationKindTag::InsertStaff, - 25 => OperationKindTag::SetTimeSignature, - 26 => OperationKindTag::SetTempoSegment, - 27 => OperationKindTag::SetStaffLayout, - 28 => OperationKindTag::CreateRepeatStructure, - 29 => OperationKindTag::DeleteRepeatStructure, - // Push 4a. Omitting this made `to_canonical_bytes` and - // `decode_canonical` asymmetric: the tag encoded to `[30]` and - // would not read back, so an edit barrier that named - // `TransposeInterval` could be persisted and never reopened. - 30 => OperationKindTag::TransposeInterval, - _ => return Err(DecodeError::MalformedDomainTag), - }) + // The mapping is generated from the same list as `discriminant` + // (`operation_kind_tag_vocabulary!`), so encode and decode cannot drift. + OperationKindTag::from_discriminant(tag).ok_or(DecodeError::MalformedDomainTag) } } @@ -1902,71 +1911,58 @@ mod tests { assert_eq!(prim.tag(), OperationKindTag::RespellPitch); } - /// **Every** `OperationKindTag` variant, listed by name. + /// **Every** `OperationKindTag`, derived from the production vocabulary. /// - /// Enumerating *variants* is the point. The round-trip test used to - /// enumerate *discriminants* — `(0u8..30).map(decode_canonical)` — which - /// starts from bytes the decoder already knows and therefore cannot notice a - /// variant the decoder is missing. Push 4a added `TransposeInterval` to - /// `discriminant()` and not to `decode_canonical`, and that test stayed - /// green while the tag encoded to `[30]` and would not read back. - /// - /// Adding a variant without adding it here fails `the_tag_vocabulary_is_complete`. + /// Not a hand-written list. `OperationKindTag::PAYLOAD_FREE` is generated by + /// `operation_kind_tag_vocabulary!`, whose `discriminant` match is + /// exhaustive over the enum — so a variant that exists cannot be missing + /// here, and the fuzz corpus, the conformance vectors, and the edit-barrier + /// round-trip all read the same constant. fn all_tags() -> Vec { - let mut tags = vec![ - OperationKindTag::InsertEvent, - OperationKindTag::DeleteEvent, - OperationKindTag::ModifyEvent, - OperationKindTag::RespellPitch, - OperationKindTag::Transpose, - OperationKindTag::CreateCrossCutting, - OperationKindTag::DeleteCrossCutting, - OperationKindTag::ModifyCrossCutting, - OperationKindTag::ChangeRegionTimeModel, - OperationKindTag::InsertRegion, - OperationKindTag::DeleteRegion, - OperationKindTag::InsertStaffInstance, - OperationKindTag::DeleteStaffInstance, - OperationKindTag::SetUserSystemBreak, - OperationKindTag::SetUserPageBreak, - OperationKindTag::DeclareTransaction, - OperationKindTag::InsertIdentifiedPitch, - OperationKindTag::DeleteIdentifiedPitch, - OperationKindTag::ModifyIdentifiedPitch, - OperationKindTag::CreateVoice, - OperationKindTag::DeleteVoice, - OperationKindTag::SetMetadata, - OperationKindTag::SetMetricGrid, - OperationKindTag::InsertStaff, - OperationKindTag::SetTimeSignature, - OperationKindTag::SetTempoSegment, - OperationKindTag::SetStaffLayout, - OperationKindTag::CreateRepeatStructure, - OperationKindTag::DeleteRepeatStructure, - OperationKindTag::TransposeInterval, - ]; + let mut tags = OperationKindTag::PAYLOAD_FREE.to_vec(); tags.push(OperationKindTag::Registered(OperationKindRegistryId( 0x0102_0304_0506_0708_090A_0B0C_0D0E_0F10, ))); tags } + /// One past the payload-free vocabulary — computed, never spelled. Spelling + /// it is how `30` came to be asserted *unknown* while it was + /// `TransposeInterval`. + fn first_unknown_discriminant() -> u8 { + OperationKindTag::PAYLOAD_FREE + .iter() + .map(OperationKindTag::discriminant) + .max() + .expect("a non-empty vocabulary") + + 1 + } + /// A compile-time-ish completeness check: `all_tags` must match the /// vocabulary the decoder accepts, in both directions. #[test] fn the_tag_vocabulary_is_complete() { let tags = all_tags(); - assert_eq!(tags.len(), 31, "30 payload-less tags plus `Registered`"); + let unknown = first_unknown_discriminant(); - // Every payload-less discriminant 0..=30 except 16 decodes to a variant - // that is in the list. - for d in (0u8..=30).filter(|d| *d != 16) { + // The payload-free discriminants are exactly `0..unknown`, minus the one + // `Registered` occupies. No gaps: the vocabulary is dense and + // append-only. + for d in (0..unknown).filter(|d| *d != REGISTERED_TAG_DISCRIMINANT) { let decoded = OperationKindTag::decode_canonical(&[d]) .unwrap_or_else(|e| panic!("discriminant {d} must decode: {e:?}")); - assert!(tags.contains(&decoded), "{decoded:?} missing from all_tags"); + assert!( + tags.contains(&decoded), + "{decoded:?} missing from PAYLOAD_FREE" + ); } - // And 31 is one past the vocabulary. - assert!(OperationKindTag::decode_canonical(&[31]).is_err()); + assert_eq!( + tags.len(), + usize::from(unknown), + "{unknown} payload-free discriminants (one of them `Registered`'s), \ + plus the `Registered` value itself" + ); + assert!(OperationKindTag::decode_canonical(&[unknown]).is_err()); } #[test] @@ -2002,12 +1998,13 @@ mod tests { #[test] fn operation_kind_tag_decode_rejects_malformed_bytes() { use epiphany_determinism::DecodeError; - // Unknown discriminant (31 is one past the vocabulary): rejected, - // never normalized. This assertion named 30 until Push 5 / P4, by which - // time 30 was `TransposeInterval` — so the test was actively locking a - // bug in place rather than guarding against one. + // One past the vocabulary: rejected, never normalized. COMPUTED, not + // spelled. This assertion literally said `30` until Push 5 / P4, by + // which time 30 was `TransposeInterval` — so the test was locking a bug + // in place rather than guarding against one. A spelled constant here is + // a trap that springs on whoever appends the next tag. assert_eq!( - OperationKindTag::decode_canonical(&[31]), + OperationKindTag::decode_canonical(&[first_unknown_discriminant()]), Err(DecodeError::MalformedDomainTag) ); // Empty input. diff --git a/crates/epiphany-ops/src/vectors.rs b/crates/epiphany-ops/src/vectors.rs index 53477b8..7ba133d 100644 --- a/crates/epiphany-ops/src/vectors.rs +++ b/crates/epiphany-ops/src/vectors.rs @@ -29,13 +29,19 @@ use crate::{ /// `verdict` is `"accept"` or `"reject"`. An `accept` vector additionally /// asserts **injectivity**: the decoded value must re-encode to exactly these /// bytes. -pub type DecodeVector = ( - &'static str, - &'static str, - &'static str, - &'static str, - Vec, -); +pub type DecodeVector = (&'static str, &'static str, &'static str, String, Vec); + +/// One row. `name` is a `String` because the tag vectors derive theirs from the +/// production vocabulary rather than spelling them. +fn row( + surface: &'static str, + verdict: &'static str, + class: &'static str, + name: impl Into, + bytes: Vec, +) -> DecodeVector { + (surface, verdict, class, name.into(), bytes) +} /// Swaps the two equal-length records of `entry` bytes that begin at `first`. fn swap_records(bytes: &[u8], first: usize, entry: usize) -> Vec { @@ -75,7 +81,7 @@ pub fn decode_vectors() -> Vec { // --- MaterializedState ------------------------------------------------- const MS: &str = "ops.materialized_state"; let empty = MaterializedState::default().canonical_bytes(); - v.push((MS, "accept", "-", "empty_state", empty.clone())); + v.push(row(MS, "accept", "-", "empty_state", empty.clone())); // Two objects, canonically ordered. Swapping them is caught only by the // whole-state re-encode guard: `objects` is a BTreeMap, so the decoder @@ -91,8 +97,8 @@ pub fn decode_vectors() -> Vec { } .canonical_bytes(); let (at, entry) = count_and_entry(&empty, &two_objects); - v.push((MS, "accept", "-", "two_objects", two_objects.clone())); - v.push(( + v.push(row(MS, "accept", "-", "two_objects", two_objects.clone())); + v.push(row( MS, "reject", "non-canonical-map-order", @@ -117,8 +123,14 @@ pub fn decode_vectors() -> Vec { } .canonical_bytes(); let (at, entry) = count_and_entry(&empty, &two_anomalies); - v.push((MS, "accept", "-", "two_anomalies", two_anomalies.clone())); - v.push(( + v.push(row( + MS, + "accept", + "-", + "two_anomalies", + two_anomalies.clone(), + )); + v.push(row( MS, "reject", "non-canonical-vec-order", @@ -140,8 +152,8 @@ pub fn decode_vectors() -> Vec { } .canonical_bytes(); let (at, entry) = count_and_entry(&empty, &two_pending); - v.push((MS, "accept", "-", "two_pending", two_pending.clone())); - v.push(( + v.push(row(MS, "accept", "-", "two_pending", two_pending.clone())); + v.push(row( MS, "reject", "non-canonical-vec-order", @@ -151,7 +163,7 @@ pub fn decode_vectors() -> Vec { let mut trailing = empty.clone(); trailing.push(0); - v.push(( + v.push(row( MS, "reject", "trailing-bytes", @@ -161,7 +173,7 @@ pub fn decode_vectors() -> Vec { let mut truncated = empty.clone(); truncated.pop(); - v.push(( + v.push(row( MS, "reject", "truncated", @@ -173,7 +185,7 @@ pub fn decode_vectors() -> Vec { // pre-allocate on it, and must not loop toward EOF for a measurable time. let mut huge_count = empty.clone(); huge_count[0..4].copy_from_slice(&u32::MAX.to_le_bytes()); - v.push(( + v.push(row( MS, "reject", "count-exceeds-remaining", @@ -182,22 +194,49 @@ pub fn decode_vectors() -> Vec { )); // --- OperationKindTag -------------------------------------------------- + // + // EVERY tag gets an accept vector, generated from the production vocabulary. + // A hand-picked subset is how `TransposeInterval` shipped encoding to a byte + // its own decoder rejected: the corpus never named it. A new tag now lands in + // the committed file as a new line, and the drift lock forces it into the diff. const TAG: &str = "ops.operation_kind_tag"; - for (name, tag) in [ - ("insert_event", OperationKindTag::InsertEvent), - ("transpose_frozen", OperationKindTag::Transpose), - ("transpose_interval", OperationKindTag::TransposeInterval), - ( - "registered", - OperationKindTag::Registered(OperationKindRegistryId(0x0123_4567_89AB_CDEF)), - ), - ] { - v.push((TAG, "accept", "-", name, tag.to_canonical_bytes())); + for tag in OperationKindTag::PAYLOAD_FREE { + let name = format!("tag_{:02}", tag.discriminant()); + v.push(row(TAG, "accept", "-", name, tag.to_canonical_bytes())); } + v.push(row( + TAG, + "accept", + "-", + "registered", + OperationKindTag::Registered(OperationKindRegistryId(0x0123_4567_89AB_CDEF)) + .to_canonical_bytes(), + )); - v.push((TAG, "reject", "unknown-discriminant", "tag_200", vec![200])); - v.push((TAG, "reject", "truncated", "tag_empty", Vec::new())); - v.push(( + // One past the vocabulary, computed rather than spelled. + let unknown = OperationKindTag::PAYLOAD_FREE + .iter() + .map(OperationKindTag::discriminant) + .max() + .expect("a non-empty vocabulary") + + 1; + v.push(row( + TAG, + "reject", + "unknown-discriminant", + format!("tag_{unknown}_one_past_the_vocabulary"), + vec![unknown], + )); + + v.push(row( + TAG, + "reject", + "unknown-discriminant", + "tag_200", + vec![200], + )); + v.push(row(TAG, "reject", "truncated", "tag_empty", Vec::new())); + v.push(row( TAG, "reject", "trailing-bytes", @@ -208,7 +247,7 @@ pub fn decode_vectors() -> Vec { let mut short_registered = OperationKindTag::Registered(OperationKindRegistryId(1)).to_canonical_bytes(); short_registered.pop(); - v.push(( + v.push(row( TAG, "reject", "truncated", diff --git a/crates/epiphany-testkit/src/vectors.rs b/crates/epiphany-testkit/src/vectors.rs index ad3354b..71e5354 100644 --- a/crates/epiphany-testkit/src/vectors.rs +++ b/crates/epiphany-testkit/src/vectors.rs @@ -2,10 +2,11 @@ //! decode-hardening track). //! //! `spec/vectors/decode_vectors.txt` is a committed, human-diffable list of byte -//! strings and their normative accept/reject verdict. It is what a *second* -//! implementation is checked against — the reference implementation's fuzzers -//! prove its own decoders self-consistent, which says nothing about whether a -//! foreign decoder agrees with the format. +//! strings and their accept/reject verdict, ratified by the Binary Format +//! companion's `req:binfmt:decode-vectors` (§"The Decode Vector Corpus"). It is +//! what a *second* implementation is checked against — the reference +//! implementation's fuzzers prove its own decoders self-consistent, which says +//! nothing about whether a foreign decoder agrees with the format. //! //! Two properties are enforced here: //! @@ -36,11 +37,16 @@ const HEADER: &str = "\ # # # -# `verdict` is `accept` or `reject`, and is the ONLY normative column besides -# the bytes. A conforming decoder must accept every `accept` vector and reject -# every `reject` vector. An accepted value must additionally re-encode to -# exactly the vector's bytes: canonical decode is injective, which is what -# content-addressing rests on. +# NORMATIVE: Binary Format companion, `req:binfmt:decode-vectors` +# (section: The Decode Vector Corpus). A conforming decoder MUST accept every `accept` +# vector for a surface it implements and MUST reject every `reject` vector, and +# a value decoded from an `accept` vector MUST re-encode to exactly its bytes. +# +# Accepting a `reject` vector and then normalizing it IS accepting it, and does +# not satisfy the requirement. Canonical decode is injective: distinct byte +# strings denote distinct values, which is what content-addressing rests on. +# +# `verdict` and the bytes are the only normative columns. # # `class` names why a `reject` vector is rejected. It is informative only — # implementations need not agree on error taxonomy. `-` where not applicable. diff --git a/spec/binary_format.pdf b/spec/binary_format.pdf index 00d3065..748e573 100644 Binary files a/spec/binary_format.pdf and b/spec/binary_format.pdf differ diff --git a/spec/binary_format.tex b/spec/binary_format.tex index 476da5b..3224260 100644 --- a/spec/binary_format.tex +++ b/spec/binary_format.tex @@ -235,7 +235,7 @@ {\Large\scshape\color{epiphanyslate}Binary Format}\\[6pt] {\large\itshape\color{epiphanyslate}A companion to the Core Specification}\\[14pt] {\color{epiphanygold}\rule{3in}{0.8pt}}\\[24pt] - {\normalsize\color{epiphanyink}Version 0.8.0 --- Strict CompressionAlgorithm decode (the non-zero None parameter is rejected, not ignored)}\\[4pt] + {\normalsize\color{epiphanyink}Version 0.9.0 --- The decode vector corpus is ratified (the cross-implementation decoder test is no longer deferred)}\\[4pt] {\small\color{epiphanyslate}Normative for the byte layouts it defines} \vfill \end{titlepage} @@ -275,13 +275,16 @@ This document does \emph{not} cover: companion's; \item the Chapter-4 tuning-catalog values (pitch-space and tuning-system registries), which are undelivered Track-C work and have no wire form yet; - \item the required conformance harnesses (the cross-implementation decoder - test and the wire-format fuzzer). Those are \emph{implementation} - deliverables tracked in the Phase-2 process documents, not part of this - document's normative text; Chapter~\ref{ch:goldens} records the golden - anchors that any such harness must reproduce. + \item the wire-format fuzzer, an \emph{implementation} deliverable tracked + in the Phase-2 process documents rather than part of this document's + normative text; Chapter~\ref{ch:goldens} records the golden anchors any + such harness must reproduce. \end{itemize} +It \emph{does} cover the cross-implementation decoder test, which is no longer +deferred: Section~\ref{sec:goldens:decode-vectors} ratifies the literal-byte +vector corpus and the accept/reject verdict a conforming decoder owes it. + \section{Relationship to the Core Specification and the Operation Catalog} \label{sec:about:relationship} @@ -3121,9 +3124,48 @@ Each anchor names its crate file, then the test on a second line. Struct \emph{bodies} in Chapter~\ref{ch:values} are round-trip-locked rather than literal-byte-locked: their byte identity follows from the frozen positional rule plus the literal-byte locks on every discriminant and leaf -they embed. A future cross-implementation decoder test (the deferred -conformance harness) should add literal-byte vectors for the representative -layouts of Section~\ref{sec:values:representative}. +they embed. + +\section{The Decode Vector Corpus} +\label{sec:goldens:decode-vectors} + +The cross-implementation decoder test is \textbf{delivered}, as a corpus of +literal byte strings each paired with the verdict a decoder owes it. The +reference implementation carries it at \texttt{spec/vectors/decode\_vectors.txt} +and regenerates it from a committed generator, so a change to any wire form +moves the file. + +Each line is \texttt{surface\ \ verdict\ \ class\ \ name\ \ hex}. The +\texttt{verdict} is \texttt{accept} or \texttt{reject}. The \texttt{class} +names \emph{why} a rejected vector is rejected and is \textbf{informative +only}: implementations need not agree on an error taxonomy. + +\begin{requirement} + \label{req:binfmt:decode-vectors} + A conforming decoder \MUST{} accept every \texttt{accept} vector for a + surface it implements, and \MUST{} reject every \texttt{reject} vector for + that surface. A value decoded from an \texttt{accept} vector \MUST{} + re-encode to exactly that vector's bytes. + + Accepting a \texttt{reject} vector and then normalizing it is + \emph{accepting} it, and does not satisfy this requirement. Canonical decode + is injective: distinct byte strings denote distinct values, which is what + content-addressing rests on. +\end{requirement} + +\begin{rationale} + The corpus is not a restatement of this document; it is what a second + implementation can be run against. Every rejection class in it is one the + reference implementation shipped a bug in, or one whose check no injectivity + fuzzer can see: a whole-value re-encode guard catches the fields a decoder + \emph{normalizes} and is blind to order-preserving sequence fields, which need + per-site order checks; and a guard on an outer value can \emph{mask} a lenient + inner codec rather than fix it (Push~5). + + A future revision should extend the corpus to the representative struct + layouts of Section~\ref{sec:values:representative}, which remain round-trip + locked rather than literal-byte locked. +\end{rationale} % =========================================================================== \chapter{Revision History} @@ -3239,6 +3281,16 @@ layouts of Section~\ref{sec:values:representative}. having no whole-value re-encode guard, accepted both. Found by the wire-decode fuzzer. Supersedes the core specification's ``ignored on read''; no wire layout changed and no conforming writer is affected. \\ + \today & Golden anchors & 0.9.0 --- Ratifies the decode vector corpus + (Push~5, P4): \texttt{req:binfmt:decode-vectors} and + Section~\ref{sec:goldens:decode-vectors}. The cross-implementation decoder + test moves out of the \sectionsc{About This Companion} exclusion list and out + of the \sectionsc{Golden Anchor Registry}'s ``deferred conformance harness'' + note: literal byte vectors with accept/reject verdicts now exist + (\texttt{spec/vectors/decode\_vectors.txt}), and a conforming decoder owes them + the verdict. Accepting a \texttt{reject} vector and normalizing it is + \emph{accepting} it. The wire-format fuzzer remains an implementation + deliverable. No wire layout changed. \\ \bottomrule \end{longtable} diff --git a/spec/vectors/decode_vectors.txt b/spec/vectors/decode_vectors.txt index c31d4ab..61ad350 100644 --- a/spec/vectors/decode_vectors.txt +++ b/spec/vectors/decode_vectors.txt @@ -9,11 +9,16 @@ # # # -# `verdict` is `accept` or `reject`, and is the ONLY normative column besides -# the bytes. A conforming decoder must accept every `accept` vector and reject -# every `reject` vector. An accepted value must additionally re-encode to -# exactly the vector's bytes: canonical decode is injective, which is what -# content-addressing rests on. +# NORMATIVE: Binary Format companion, `req:binfmt:decode-vectors` +# (section: The Decode Vector Corpus). A conforming decoder MUST accept every `accept` +# vector for a surface it implements and MUST reject every `reject` vector, and +# a value decoded from an `accept` vector MUST re-encode to exactly its bytes. +# +# Accepting a `reject` vector and then normalizing it IS accepting it, and does +# not satisfy the requirement. Canonical decode is injective: distinct byte +# strings denote distinct values, which is what content-addressing rests on. +# +# `verdict` and the bytes are the only normative columns. # # `class` names why a `reject` vector is rejected. It is informative only — # implementations need not agree on error taxonomy. `-` where not applicable. @@ -33,10 +38,38 @@ ops.materialized_state reject truncated empty_state_truncated 000000000000000000 ops.materialized_state reject count-exceeds-remaining effects_count_u32_max ffffffff00000000000000000000000000000000000000000000000000000000 # ops.operation_kind_tag -ops.operation_kind_tag accept - insert_event 00 -ops.operation_kind_tag accept - transpose_frozen 04 -ops.operation_kind_tag accept - transpose_interval 1e +ops.operation_kind_tag accept - tag_00 00 +ops.operation_kind_tag accept - tag_01 01 +ops.operation_kind_tag accept - tag_02 02 +ops.operation_kind_tag accept - tag_03 03 +ops.operation_kind_tag accept - tag_04 04 +ops.operation_kind_tag accept - tag_05 05 +ops.operation_kind_tag accept - tag_06 06 +ops.operation_kind_tag accept - tag_07 07 +ops.operation_kind_tag accept - tag_08 08 +ops.operation_kind_tag accept - tag_09 09 +ops.operation_kind_tag accept - tag_10 0a +ops.operation_kind_tag accept - tag_11 0b +ops.operation_kind_tag accept - tag_12 0c +ops.operation_kind_tag accept - tag_13 0d +ops.operation_kind_tag accept - tag_14 0e +ops.operation_kind_tag accept - tag_15 0f +ops.operation_kind_tag accept - tag_17 11 +ops.operation_kind_tag accept - tag_18 12 +ops.operation_kind_tag accept - tag_19 13 +ops.operation_kind_tag accept - tag_20 14 +ops.operation_kind_tag accept - tag_21 15 +ops.operation_kind_tag accept - tag_22 16 +ops.operation_kind_tag accept - tag_23 17 +ops.operation_kind_tag accept - tag_24 18 +ops.operation_kind_tag accept - tag_25 19 +ops.operation_kind_tag accept - tag_26 1a +ops.operation_kind_tag accept - tag_27 1b +ops.operation_kind_tag accept - tag_28 1c +ops.operation_kind_tag accept - tag_29 1d +ops.operation_kind_tag accept - tag_30 1e ops.operation_kind_tag accept - registered 1000000000000000000123456789abcdef +ops.operation_kind_tag reject unknown-discriminant tag_31_one_past_the_vocabulary 1f ops.operation_kind_tag reject unknown-discriminant tag_200 c8 ops.operation_kind_tag reject truncated tag_empty - ops.operation_kind_tag reject trailing-bytes insert_event_trailing 0000