From f105b53599754c7a88aeee9230f37063c1c3b783 Mon Sep 17 00:00:00 2001 From: Levi Neuwirth Date: Sun, 21 Jun 2026 19:26:17 -0400 Subject: [PATCH] Item 6 (part 2): operation-block summaries (D-B, C/D integration) Chapter 8's OperationEnvelopeBlock carries dvv_summary/min_stamp/max_stamp so a reader can select or skip a block by causal frontier / stamp range without decoding it. These are semantic (ops-computed); the bundle carries them opaquely. Bundle (Agent D): - OperationBlockSummary { dvv_summary: FrontierBytes, min_stamp, max_stamp } and Manifest.operation_block_summaries: BTreeMap, keyed by the block's chunk id, encoded/decoded in canonical (ChunkId-ascending) order and accessible via Manifest::operation_block_summary. Optional and non-canonical; preserved across reopen by the manifest round-trip. - Round-trip + selectability test. Testkit (Agent F, the C/D integration point): - roundtrip::operation_block_summary computes the summary from envelopes using ops (causal frontier + min/max OperationStamp canonical bytes). - assert_operation_block_summary_survives_storage commits a real operation block + its summary, reopens, and selects the summary by block id without decoding the payload. Wired into acceptance + the conformance suite. bundle DECISIONS updated (summary metadata now carried, not omitted); fixed a stale "pending item 5" doc on criterion 4 (the whole-score codec has landed). Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/epiphany-bundle/DECISIONS.md | 23 +++-- crates/epiphany-bundle/src/lib.rs | 4 +- crates/epiphany-bundle/src/manifest.rs | 91 +++++++++++++++++++ .../examples/conformance_suite.rs | 2 + crates/epiphany-testkit/src/roundtrip.rs | 82 ++++++++++++++++- crates/epiphany-testkit/tests/acceptance.rs | 24 ++++- 6 files changed, 209 insertions(+), 17 deletions(-) diff --git a/crates/epiphany-bundle/DECISIONS.md b/crates/epiphany-bundle/DECISIONS.md index 28a8649..59f094e 100644 --- a/crates/epiphany-bundle/DECISIONS.md +++ b/crates/epiphany-bundle/DECISIONS.md @@ -163,14 +163,21 @@ yet enforced so a later integration knows where to extend. is re-verified on read). Whole-history dedup needs the same body-wide content index as the deferred GC engine. -- **Operation-envelope block summary metadata is omitted.** Chapter 8's - `OperationEnvelopeBlock` carries `dvv_summary`, `min_stamp`, and `max_stamp`. - These are *semantic* — a DVV and `OperationStamp`s computed by reading the - envelopes, which belong to `epiphany-ops` (Agent C). The bundle treats a block - as opaque envelope bytes, so it cannot compute them. At C/D integration these - become opaque, ops-supplied fields prefixed to the block payload; v0 stores - only the envelopes (`block::encode_block`). `read_operation_block` does enforce - the chunk kind and the active profile's maximum block size. +- **Operation-envelope block summary metadata is carried (M4 follow-up).** + Chapter 8's `OperationEnvelopeBlock` carries `dvv_summary`, `min_stamp`, and + `max_stamp`. These are *semantic* — a DVV and `OperationStamp`s computed by + reading the envelopes, which belong to `epiphany-ops` (Agent C). The bundle + still treats a block as opaque envelope bytes and cannot compute them, but the + manifest now carries an `OperationBlockSummary { dvv_summary, min_stamp, + max_stamp }` per block, keyed by the block's `ChunkId` + (`Manifest::operation_block_summaries` / `operation_block_summary`), as + **opaque ops-supplied bytes** in canonical (ChunkId-ascending) order. This lets + a reader select or skip a block by causal frontier / stamp range without + decoding it. The C/D integration point — ops computes the summary, the bundle + carries it — is exercised end to end by Agent F + (`roundtrip::operation_block_summary` + + `assert_operation_block_summary_survives_storage`). `read_operation_block` + still enforces the chunk kind and the active profile's maximum block size. - **Schema negotiation is major-gate only.** A canonical chunk or manifest at an unsupported schema *major* is refused (`BundleError::UnsupportedSchemaVersion`); diff --git a/crates/epiphany-bundle/src/lib.rs b/crates/epiphany-bundle/src/lib.rs index d1d7ec1..439f4ed 100644 --- a/crates/epiphany-bundle/src/lib.rs +++ b/crates/epiphany-bundle/src/lib.rs @@ -84,8 +84,8 @@ pub use ids::{ WallClockDuration, WallClockTime, }; pub use manifest::{ - BlobRef, ExtensionDeclaration, Manifest, ProfileConstraints, ProfileDeclaration, - RetentionPolicy, SnapshotRef, + BlobRef, ExtensionDeclaration, Manifest, OperationBlockSummary, ProfileConstraints, + ProfileDeclaration, RetentionPolicy, SnapshotRef, }; pub use store::{BlockStore, CrashPoint, FaultStore, MemStore, Tear}; pub use superblock::{ diff --git a/crates/epiphany-bundle/src/manifest.rs b/crates/epiphany-bundle/src/manifest.rs index ddbc5f6..a42d015 100644 --- a/crates/epiphany-bundle/src/manifest.rs +++ b/crates/epiphany-bundle/src/manifest.rs @@ -18,6 +18,8 @@ //! fields are deliberately distinct (QUICKSTART: *"these are distinct; do not //! merge them"*): exactly one canonical base, plus any number of caches. +use std::collections::BTreeMap; + use crate::chunk::{ChunkRef, CompressionAlgorithm}; use crate::codec::{DecodeError, Reader, Writer}; use crate::ids::{ @@ -322,6 +324,40 @@ impl ExtensionDeclaration { } /// The manifest itself. +/// Summary metadata for one operation-envelope block (Chapter 8: an +/// `OperationEnvelopeBlock`'s `dvv_summary` / `min_stamp` / `max_stamp`). +/// +/// These are *semantic* values — a causal frontier (DVV) and operation-stamp +/// range computed by reading the block's envelopes — which belong to the +/// operation layer (Agent C), not the bundle. The bundle treats them as **opaque +/// bytes**, computed and interpreted by `epiphany-ops`, and carries them keyed by +/// the block's chunk id so a reader can select or skip a block by causal frontier +/// or stamp range **without decoding its envelopes** (the point of a summary). +#[derive(Clone, PartialEq, Eq, Debug, Default)] +pub struct OperationBlockSummary { + /// The causal frontier (DVV) the block covers — opaque ops-computed bytes. + pub dvv_summary: FrontierBytes, + /// The block's minimum operation stamp, canonical bytes — opaque to the bundle. + pub min_stamp: Vec, + /// The block's maximum operation stamp, canonical bytes — opaque to the bundle. + pub max_stamp: Vec, +} + +impl OperationBlockSummary { + fn encode(&self, w: &mut Writer) { + self.dvv_summary.encode(w); + w.put_var_bytes(&self.min_stamp); + w.put_var_bytes(&self.max_stamp); + } + fn decode(r: &mut Reader) -> Result { + Ok(OperationBlockSummary { + dvv_summary: FrontierBytes::decode(r)?, + min_stamp: r.get_var_bytes()?, + max_stamp: r.get_var_bytes()?, + }) + } +} + #[derive(Clone, PartialEq, Eq, Debug)] pub struct Manifest { /// Logical-work identity. @@ -336,6 +372,11 @@ pub struct Manifest { /// Operation-envelope blocks defining the canonical document (canonical /// root). pub operation_roots: Vec, + /// Per-operation-block summaries (Chapter 8: `dvv_summary`/`min_stamp`/ + /// `max_stamp`), keyed by the block's chunk id. Opaque, ops-supplied metadata + /// that lets a reader select blocks without decoding them; non-canonical and + /// optional (a block need not have an entry). + pub operation_block_summaries: BTreeMap, /// Optional operation index (non-canonical accelerator). pub operation_index_root: Option, /// The active canonical base snapshot, if pruning has occurred (canonical @@ -367,6 +408,7 @@ impl Manifest { manifest_id: ManifestId::default(), generation: 0, operation_roots: Vec::new(), + operation_block_summaries: BTreeMap::new(), operation_index_root: None, canonical_base: None, acceleration_snapshots: Vec::new(), @@ -390,6 +432,13 @@ impl Manifest { ) } + /// The summary recorded for the given operation block, if any. Lets a reader + /// select or skip a block by causal frontier / stamp range without decoding + /// its envelopes (Chapter 8: `OperationEnvelopeBlock` summary metadata). + pub fn operation_block_summary(&self, block: ChunkId) -> Option<&OperationBlockSummary> { + self.operation_block_summaries.get(&block) + } + /// The first profile declaration in canonical order, or `None` if none is /// declared. pub fn canonical_first_profile(&self) -> Option { @@ -422,6 +471,14 @@ impl Manifest { let op_roots = sorted_dedup_chunk_refs(&self.operation_roots); w.put_seq(&op_roots, |w, c| c.encode(w)); + // Per-block summaries, in canonical (BTreeMap = ChunkId-ascending) order. + let summaries: Vec<(&ChunkId, &OperationBlockSummary)> = + self.operation_block_summaries.iter().collect(); + w.put_seq(&summaries, |w, entry| { + w.put_bytes(entry.0.as_bytes()); + entry.1.encode(w); + }); + w.put_opt(&self.operation_index_root, |w, c| c.encode(w)); w.put_opt(&self.canonical_base, |w, s| s.encode(w)); @@ -483,6 +540,13 @@ impl Manifest { let lineage_id = r.get_opt(LineageId::decode)?; let generation = r.get_u64()?; let operation_roots = r.get_seq(ChunkRef::decode)?; + let summary_entries = r.get_seq(|r| { + let id = ChunkId(ContentHash(r.take_array::<32>()?)); + let summary = OperationBlockSummary::decode(r)?; + Ok((id, summary)) + })?; + let operation_block_summaries: BTreeMap = + summary_entries.into_iter().collect(); let operation_index_root = r.get_opt(ChunkRef::decode)?; let canonical_base = r.get_opt(SnapshotRef::decode)?; let acceleration_snapshots = r.get_seq(SnapshotRef::decode)?; @@ -499,6 +563,7 @@ impl Manifest { manifest_id: stored_id, generation, operation_roots, + operation_block_summaries, operation_index_root, canonical_base, acceleration_snapshots, @@ -644,6 +709,32 @@ mod tests { use crate::chunk::{chunk_id, ChunkKind}; use crate::ids::SnapshotId; + #[test] + fn operation_block_summaries_round_trip_and_are_selectable() { + let mut m = Manifest::empty(DocumentId([5; 16])); + let block = ChunkId(ContentHash([7; 32])); + m.operation_block_summaries.insert( + block, + OperationBlockSummary { + dvv_summary: FrontierBytes::from_bytes(vec![1, 2, 3]), + min_stamp: vec![10, 11], + max_stamp: vec![20, 21], + }, + ); + // The summary survives the canonical encode/decode of the manifest... + let decoded = Manifest::decode(&m.encode()).expect("manifest decodes"); + let summary = decoded + .operation_block_summary(block) + .expect("summary preserved"); + assert_eq!(summary.dvv_summary.as_bytes(), &[1, 2, 3]); + assert_eq!(summary.min_stamp, vec![10, 11]); + assert_eq!(summary.max_stamp, vec![20, 21]); + // ...and a reader selects by block id without touching any block payload. + assert!(decoded + .operation_block_summary(ChunkId(ContentHash([8; 32]))) + .is_none()); + } + #[test] fn semver_orders_numerically_not_byte_wise() { // SemVer integers are little-endian; a byte-order sort would put 256.0.0 diff --git a/crates/epiphany-testkit/examples/conformance_suite.rs b/crates/epiphany-testkit/examples/conformance_suite.rs index a9abc4f..d4f4aa6 100644 --- a/crates/epiphany-testkit/examples/conformance_suite.rs +++ b/crates/epiphany-testkit/examples/conformance_suite.rs @@ -49,6 +49,8 @@ fn main() { roundtrip::assert_reduction_serialization_stable(&session, seed); let (score, frontier) = convergence::materialized_score(seed.wrapping_add(101)); roundtrip::assert_score_serialization_stable(&score, &frontier, seed); + let envs = generators::operation_envelopes(&mut rng, 24, 3, 8, 8); + roundtrip::assert_operation_block_summary_survives_storage(&envs, seed.wrapping_add(202)); } roundtrip::assert_content_mutation_changes_serialization(); diff --git a/crates/epiphany-testkit/src/roundtrip.rs b/crates/epiphany-testkit/src/roundtrip.rs index 084ea35..5b1d85b 100644 --- a/crates/epiphany-testkit/src/roundtrip.rs +++ b/crates/epiphany-testkit/src/roundtrip.rs @@ -31,9 +31,9 @@ use std::fmt::Debug; use epiphany_bundle::{ - Bundle, ChunkKind, CommitContext, DocumentId, FileUuid, FixedHeader, FrontierBytes, Manifest, - MemStore, ProfileId, ReductionAlgorithmVersion, SchemaVersion, SlotParse, SnapshotId, - SnapshotRef, StagedChunk, Superblock, + pack_operation_blocks, Bundle, ChunkKind, CommitContext, DocumentId, FileUuid, FixedHeader, + FrontierBytes, Manifest, MemStore, OperationBlockSummary, ProfileId, ReductionAlgorithmVersion, + SchemaVersion, SlotParse, SnapshotId, SnapshotRef, StagedChunk, Superblock, }; use epiphany_core::Score; use epiphany_determinism::{CanonicalDecode, CanonicalEncode}; @@ -470,11 +470,87 @@ pub fn assert_header_decode_rejects_corruption(header: &FixedHeader) { ); } +/// The ops-computed summary of an operation block (Chapter 8: an +/// `OperationEnvelopeBlock`'s `dvv_summary`/`min_stamp`/`max_stamp`). This is the +/// **C/D integration point**: the operation layer (Agent C) computes the +/// semantic summary by reading the envelopes — the causal frontier they cover +/// and the canonical bytes of the minimum and maximum operation stamps — and the +/// bundle (Agent D) carries it opaquely, keyed by the block's chunk id, so a +/// reader can select a block by frontier/stamp range without decoding it. +pub fn operation_block_summary(envelopes: &[OperationEnvelope]) -> OperationBlockSummary { + let stamp_bytes = |e: &OperationEnvelope| e.stamp.to_canonical_bytes(); + OperationBlockSummary { + dvv_summary: FrontierBytes::from_bytes(generators::frontier_bytes(envelopes)), + min_stamp: envelopes + .iter() + .min_by_key(|e| e.stamp.reduction_tuple()) + .map(stamp_bytes) + .unwrap_or_default(), + max_stamp: envelopes + .iter() + .max_by_key(|e| e.stamp.reduction_tuple()) + .map(stamp_bytes) + .unwrap_or_default(), + } +} + +/// Asserts an ops-computed [`operation_block_summary`] survives a real bundle +/// commit + reopen and is selectable by the block's chunk id without decoding the +/// block payload (Chapter 8 operation-block summary metadata, C/D integration). +pub fn assert_operation_block_summary_survives_storage(envelopes: &[OperationEnvelope], seed: u64) { + let summary = operation_block_summary(envelopes); + assert!( + !summary.dvv_summary.as_bytes().is_empty() + && !summary.min_stamp.is_empty() + && !summary.max_stamp.is_empty(), + "a non-empty envelope set must produce a non-vacuous summary" + ); + + let mut rng = Rng::new(seed); + let uuid = FileUuid(rng.array16()); + let doc = DocumentId(rng.array16()); + let mut bundle = + Bundle::create(MemStore::new(), uuid, Manifest::empty(doc)).expect("create bundle"); + // A real operation block (opaque payload bytes) carrying the summary. + let blocks: Vec = pack_operation_blocks(&[rng.byte_vec(4, 64)]) + .into_iter() + .map(StagedChunk::operation_block) + .collect(); + bundle + .commit(&blocks, |ctx| { + let mut m = ctx.previous_manifest.clone(); + let root = ctx.new_chunks[0]; + m.operation_roots.push(root); + m.operation_block_summaries.insert(root.id, summary.clone()); + m + }) + .expect("commit operation block + summary"); + + // Reopen and select the summary by block id — no block payload is decoded. + let image = bundle.into_store().into_bytes(); + let reopened = Bundle::open(MemStore::from_bytes(image)).expect("reopen bundle"); + let root_id = reopened.manifest().operation_roots[0].id; + assert_eq!( + reopened.manifest().operation_block_summary(root_id), + Some(&summary), + "the ops-computed block summary must survive storage and be selectable" + ); +} + #[cfg(test)] mod tests { use super::*; use epiphany_bundle::encode_block; + #[test] + fn operation_block_summaries_survive_storage_and_select() { + let mut rng = Rng::new(0x05_5077_5044_0B0B); + for seed in 0..16u64 { + let envelopes = generators::operation_envelopes(&mut rng, 24, 3, 8, 8); + assert_operation_block_summary_survives_storage(&envelopes, seed.wrapping_add(1)); + } + } + #[test] fn corpus_round_trips() { run_roundtrip_corpus(60_000, 0x00C0_FFEE_1234_5678); diff --git a/crates/epiphany-testkit/tests/acceptance.rs b/crates/epiphany-testkit/tests/acceptance.rs index d19e7b4..d1b4e48 100644 --- a/crates/epiphany-testkit/tests/acceptance.rs +++ b/crates/epiphany-testkit/tests/acceptance.rs @@ -95,10 +95,10 @@ fn criterion_3_equivocation() { /// manifest/header — including decoder rejection of corruption. /// /// The **full-Score** byte round-trip is split out below: its bookkeeping -/// projection ([`reducer_bookkeeping_serialization`]) and its reproducibility -/// ([`full_score_materialization_is_reproducible`]) are exercised now; the -/// whole-`Score` byte codec is pending item 5 (Agent B) — -/// [`criterion_4_full_score_byte_roundtrip`]. +/// projection ([`reducer_bookkeeping_serialization`]), its reproducibility +/// ([`full_score_materialization_is_reproducible`]), and — now that item 5's +/// whole-`Score` codec has landed — the real byte-level round-trip +/// ([`criterion_4_full_score_byte_roundtrip`]). #[test] fn criterion_4_canonical_serialization_stability() { roundtrip::run_roundtrip_corpus(100_000, 0x00C0_FFEE_1234_5678); @@ -168,6 +168,22 @@ fn criterion_4_full_score_byte_roundtrip() { } } +/// **Operation-block summaries (Chapter 8, C/D integration).** An ops-computed +/// block summary (causal frontier + min/max operation stamp) survives a real +/// bundle commit + reopen and is selectable by block id without decoding the +/// block payload. +#[test] +fn operation_block_summaries_survive_storage() { + let mut rng = Rng::new(0x0B5_5044_0B0B_0B0B); + for seed in 0..16u64 { + let envelopes = generators::operation_envelopes(&mut rng, 24, 3, 8, 8); + roundtrip::assert_operation_block_summary_survives_storage( + &envelopes, + seed.wrapping_add(1), + ); + } +} + /// Criterion 5 — **Reduction determinism.** A randomized 1,000-envelope set, /// reduced 10 times in 10 different orders, produces byte-identical materialized /// states *and* an identical canonical reduction order (Appendix D's