diff --git a/Cargo.lock b/Cargo.lock index cb20006..72d0e0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1286,6 +1286,16 @@ dependencies = [ "epiphany-ops", ] +[[package]] +name = "epiphany-textproj" +version = "0.0.0" +dependencies = [ + "epiphany-bundle", + "epiphany-core", + "epiphany-determinism", + "epiphany-ops", +] + [[package]] name = "equivalent" version = "1.0.2" diff --git a/Cargo.toml b/Cargo.toml index 4c4ae81..fb854e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "crates/epiphany-core", "crates/epiphany-bundle", "crates/epiphany-ops", + "crates/epiphany-textproj", "crates/epiphany-layout-ir", "crates/epiphany-engrave", "crates/epiphany-render-svg", @@ -44,6 +45,9 @@ epiphany-bundle = { path = "crates/epiphany-bundle" } # testkit drives its convergence, reduction-determinism, and equivocation gates, # so its path is declared here alongside the other intra-workspace crates. epiphany-ops = { path = "crates/epiphany-ops" } +# The Text Projection companion is the document-level bridge between canonical +# bundles and their normative s-expression form. +epiphany-textproj = { path = "crates/epiphany-textproj" } # Agent E's epiphany-layout-ir is the layout IR + constraint-solver interface # (Chapters 7 & 9). Agent F's testkit drives its layout round-trip gate # (v0 acceptance criterion 6), so its path is declared here alongside the other diff --git a/crates/epiphany-textproj/Cargo.toml b/crates/epiphany-textproj/Cargo.toml new file mode 100644 index 0000000..2aa14af --- /dev/null +++ b/crates/epiphany-textproj/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "epiphany-textproj" +version = "0.0.0" +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +repository.workspace = true +description = "The Epiphany Text Projection companion: the canonical s-expression document representation and its strict bundle round trip." + +[dependencies] +epiphany-determinism.workspace = true +epiphany-core.workspace = true +epiphany-bundle.workspace = true +epiphany-ops.workspace = true diff --git a/crates/epiphany-textproj/src/lib.rs b/crates/epiphany-textproj/src/lib.rs new file mode 100644 index 0000000..9a5cf7f --- /dev/null +++ b/crates/epiphany-textproj/src/lib.rs @@ -0,0 +1,117 @@ +#![forbid(unsafe_code)] +//! The Epiphany Text Projection companion document layer. +//! +//! This crate bridges canonical bundle documents and the normative line-oriented +//! s-expression representation defined by `spec/text_projection.tex`. + +pub mod parse; +pub mod project; +pub mod serialize; + +use epiphany_bundle::{ + ChunkKind, DocumentId, ExtensionId, FrontierBytes, LineageId, ProfileDeclaration, ProfileId, + ReductionAlgorithmVersion, SchemaVersion, SemVer, SnapshotId, +}; +use epiphany_ops::OperationEnvelope; + +/// The one Text Projection companion version implemented by this crate. +/// +/// A parser must reject every other version rather than migrating or +/// normalizing it on read. +pub const COMPANION_VERSION: (u32, u32, u32) = (0, 7, 0); + +/// A parsed canonical Text Projection document. +/// +/// Under `req:textproj:derive-or-carry`, this representation deliberately erases +/// physical offsets, compressed and uncompressed storage lengths, and +/// compression choices because serialization is free to choose a new physical +/// layout. It also omits the derivable `ChunkId`, `ContentHash`, and `BlobId`; +/// those identities are recomputed from kind, schema, and inline payload. +/// Finally, it drops the non-canonical `operation_index_root`, +/// `acceleration_snapshots`, `text_projection_root`, `integrity_root`, and +/// `operation_block_summaries` accelerators. None contributes to canonical +/// document semantics, so a bundle serialized from this form correctly rebuilds +/// or omits them rather than carrying stale physical metadata. +#[derive(Debug, PartialEq)] +pub struct TextDocument { + /// Logical identity of the projected document. + pub document_id: DocumentId, + /// Optional shared-ancestor identity used for document genealogy. + pub lineage_id: Option, + /// Profile declarations in canonical manifest order. + pub profiles: Vec, + /// Extension declarations with every preserved chunk payload inline. + pub extensions: Vec, + /// Optional canonical base with its snapshot root payload inline. + pub canonical_base: Option, + /// Canonically reachable blobs with their payloads inline. + pub blobs: Vec, + /// Operation envelopes in canonical reduction order. + pub envelopes: Vec, +} + +/// An extension declaration in its text-document form. +/// +/// Unlike the bundle's `ExtensionDeclaration`, this type carries preserved +/// chunks as semantic kind/schema/payload triples, not physical `ChunkRef`s. +#[derive(Debug, PartialEq)] +pub struct TextExtension { + /// Opaque identity of the extension. + pub extension_id: ExtensionId, + /// Semantic version of the extension declaration. + pub version: SemVer, + /// Whether an implementation unaware of the extension must refuse editing. + pub required: bool, + /// Preserved extension chunks, inline and ordered by projected form. + pub chunks: Vec, + /// Canonical opaque encoding of affected object kinds. + pub affected_object_kinds: Vec, + /// Canonical opaque encoding of the extension's edit barriers. + pub edit_barriers: Vec, +} + +/// One preserved extension chunk with all physical reference data erased. +#[derive(Debug, PartialEq)] +pub struct TextChunk { + /// Semantic role of the chunk. + pub kind: ChunkKind, + /// Schema version governing the payload bytes. + pub schema_version: SchemaVersion, + /// Uncompressed chunk payload carried inline. + pub payload: Vec, +} + +/// A canonical base snapshot in its text-document form. +/// +/// The snapshot identity is carried because it is opaque, while the root chunk +/// identity and content hash are derived from its schema and inline payload. +#[derive(Debug, PartialEq)] +pub struct TextCanonicalBase { + /// Opaque snapshot identity, carried verbatim. + pub snapshot_id: SnapshotId, + /// Opaque causal frontier materialized by the snapshot. + pub covers_causal_frontier: FrontierBytes, + /// Reduction algorithm version used to produce the snapshot. + pub reduction_algorithm_version: ReductionAlgorithmVersion, + /// Profile under which the snapshot was produced. + pub profile_id: ProfileId, + /// Schema version of the snapshot root chunk. + pub root_schema_version: SchemaVersion, + /// Uncompressed snapshot root payload carried inline. + pub root_payload: Vec, +} + +/// A canonical blob in its text-document form. +/// +/// The payload is inline; its bundle `BlobId`, content hash, offset, lengths, +/// and compression metadata are deliberately absent and are derived or chosen +/// when serialized. +#[derive(Debug, PartialEq)] +pub struct TextBlob { + /// RFC 6838 media type. + pub media_type: String, + /// Optional declared maximum uncompressed size. + pub declared_max_uncompressed_length: Option, + /// Uncompressed blob payload carried inline. + pub payload: Vec, +} diff --git a/crates/epiphany-textproj/src/parse.rs b/crates/epiphany-textproj/src/parse.rs new file mode 100644 index 0000000..1dac604 --- /dev/null +++ b/crates/epiphany-textproj/src/parse.rs @@ -0,0 +1,1125 @@ +//! Strict parsing of canonical Text Projection documents. +//! +//! Parsing is grammar-directed over [`epiphany_core::textvalue::Sexp`] values +//! read by [`epiphany_core::textvalue::read_sexp`], reports +//! [`epiphany_core::textvalue::TextError`], and delegates envelope productions +//! to [`epiphany_ops::parse_envelope`]. +//! +//! # Free functions, not `TextValue` impls +//! +//! `TextValue` is a core-crate trait and every document-line type +//! (`Manifest`, `ProfileDeclaration`, ...) belongs to `epiphany-bundle`, which +//! does not depend on `epiphany-core`. Implementing a foreign trait for a +//! foreign type from this third crate is exactly what the orphan rule +//! forbids, and no document-line production needs it anyway: every one +//! bottoms out in `bytes`, `integer`, `bool`, `string`, `option`, or a closed +//! vocabulary, never a Chapter-5 `value`. So this module is free functions +//! throughout, one pair (or one parser, where there is no corresponding +//! `project_*` in this file) per grammar production. +//! +//! # Two rejections, and why each is a rejection rather than a normalization +//! +//! Every check below rejects rather than repairs, per `req:textproj:strict-parse`. +//! Two are load-bearing enough to call out specifically: +//! +//! * **Any `(blob ...)` line is rejected**, unconditionally, in +//! [`parse_document`]. At `crate::COMPANION_VERSION` no canonical operation +//! and no canonical reduced state can carry a `BlobId` (a source scan -- +//! the companion `epiphany-core`/`epiphany-ops` crates never mention the +//! type), so canonical state cannot reference a blob and every blob line is +//! necessarily unreferenced (`req:textproj:reject-unreferenced-blobs`). +//! Accepting one would stage a blob into the bundle that the very next +//! projection silently drops -- losing data and falsifying +//! `project(serialize(parse(T))) == T` for that text. The `(blob ...)` +//! *production* itself still parses -- `parse_blob` is exercised directly +//! against synthetic data below -- so the accept side is ready the moment +//! the reachability predicate becomes non-empty; only the document-level +//! decision to use it is a permanent "no". +//! * **Any header version other than `crate::COMPANION_VERSION` is rejected +//! at line one** (`req:textproj:header-version`). Multi-version acceptance +//! and migrate-on-read are deferred spec decisions; this parser does not +//! speculate about them. +//! +//! # Section order is a sequence, not a set +//! +//! `projection ::= header document lineage? profile* extension* +//! canonical-base? blob* envelope*` is consumed by [`parse_document`] in +//! exactly that order, greedily and without backtracking: each stage takes +//! every line that belongs to it and stops at the first line that does not. +//! Anything left over once every stage has run -- a repeated singular +//! section, a section that shows up before the one it follows, or simple +//! garbage -- is rejected as a whole. A parser that instead recognised every +//! line by its own head symbol and sorted them into place would accept +//! exactly the disordered and repeated texts this design rejects; that is +//! the normalization `req:textproj:strict-parse` forbids, so this parser +//! does not do it. + +use epiphany_bundle::{ + ChunkKind, DocumentId, ExtensionId, FrontierBytes, LineageId, ProfileConstraints, + ProfileDeclaration, ProfileId, ProfileRegistryId, ReductionAlgorithmVersion, RetentionPolicy, + SchemaVersion, SemVer, SnapshotId, WallClockDuration, +}; +use epiphany_core::textvalue::{read_sexp, Sexp, TextError, TextValue}; +use epiphany_ops::{canonical_reduction_order, parse_envelope, OperationEnvelope}; + +use crate::{ + TextBlob, TextCanonicalBase, TextChunk, TextDocument, TextExtension, COMPANION_VERSION, +}; + +// =========================================================================== +// The document-level driver. +// =========================================================================== + +/// Parses a complete canonical Text Projection into a [`TextDocument`], +/// rejecting anything that is not the canonical projection of the document it +/// denotes (`req:textproj:strict-parse`). +/// +/// `text` must be exactly what `req:textproj:envelope-per-line` describes: +/// lines separated by a single U+000A, a final U+000A, and nothing else +/// trailing. Every line is one complete s-expression, and the sequence of +/// lines must follow `projection`'s section order precisely -- see the +/// module documentation for why an out-of-order or repeated section is a +/// rejection rather than something to sort back into place. +pub fn parse_document(text: &str) -> Result { + let raw_lines = split_lines(text)?; + let mut lines = Lines { + lines: raw_lines, + pos: 0, + }; + + // header: mandatory, first (`req:textproj:header-version`). + require_line( + &mut lines, + "text-projection", + "a projection must begin with a header line naming the companion version", + parse_header, + )?; + + // document: mandatory, immediately after the header. + let document_id = require_line( + &mut lines, + "document", + "a projection must carry a document line immediately after its header", + parse_document_id_line, + )?; + + // lineage? + let lineage_id = take_line(&mut lines, "lineage", parse_lineage)?; + + // profile*, in canonical (profile_id, version) order -- no repeats. + let mut profiles = Vec::new(); + while let Some(profile) = take_line(&mut lines, "profile", parse_profile)? { + profiles.push(profile); + } + if profiles + .windows(2) + .any(|w| (w[0].profile_id, w[0].version) >= (w[1].profile_id, w[1].version)) + { + return Err(TextError::NotStrictlyIncreasing( + "profile declarations must be ordered, without repetition, by (profile_id, version)", + )); + } + + // extension*, in canonical (extension_id, version) order -- no repeats. + let mut extensions = Vec::new(); + while let Some(extension) = take_line(&mut lines, "extension", parse_extension)? { + extensions.push(extension); + } + if extensions + .windows(2) + .any(|w| (w[0].extension_id, w[0].version) >= (w[1].extension_id, w[1].version)) + { + return Err(TextError::NotStrictlyIncreasing( + "extension declarations must be ordered, without repetition, by (extension_id, version)", + )); + } + + // canonical-base? + let canonical_base = take_line(&mut lines, "canonical-base", parse_canonical_base)?; + + // blob*: collected only so the rejection below can fire; see the module + // documentation and `req:textproj:reject-unreferenced-blobs`. + let mut blobs = Vec::new(); + while let Some(blob) = take_line(&mut lines, "blob", parse_blob)? { + blobs.push(blob); + } + if !blobs.is_empty() { + return Err(TextError::NotCanonical( + "a (blob ...) line is unreferenced by canonical state: no canonical operation or \ + reduced state can carry a BlobId at this companion version, so no blob line can \ + ever be canonical here", + )); + } + + // envelope*, in canonical reduction order. + let mut envelopes = Vec::new(); + while let Some(line) = lines.peek() { + let sexp = read_sexp(line)?; + if line_head(&sexp) != Some("envelope") { + break; + } + lines.advance(); + envelopes.push(parse_envelope(line)?); + } + check_envelope_order(&envelopes)?; + + // Anything left over is a line that belongs to no remaining stage: a + // repeated singular section, a section reappearing after a later one, or + // plain garbage after the envelopes. Every one of those is a rejection, + // never a re-sort. + if lines.peek().is_some() { + return Err(TextError::Syntax( + "a line appears out of the projection's normative section order, or a section that \ + may occur at most once is repeated", + )); + } + + Ok(TextDocument { + document_id, + lineage_id, + profiles, + extensions, + canonical_base, + blobs: Vec::new(), + envelopes, + }) +} + +/// A cursor over a projection's lines, advanced only by [`take_line`] and +/// [`require_line`] so every stage of [`parse_document`] consumes lines in +/// exactly one pass. +struct Lines<'a> { + lines: Vec<&'a str>, + pos: usize, +} + +impl<'a> Lines<'a> { + fn peek(&self) -> Option<&'a str> { + self.lines.get(self.pos).copied() + } + + fn advance(&mut self) { + self.pos += 1; + } +} + +/// Splits a whole projection into its lines, enforcing the layout half of +/// `req:textproj:envelope-per-line`: a single trailing U+000A and no other +/// trailing whitespace. Without this check a projection missing its final +/// U+000A would still split into the same lines by `str::split('\n')` and +/// parse identically -- the trailing-newline requirement would be silently +/// unenforced. An embedded blank line (a stray `U+000A U+000A`, anywhere) +/// needs no special case here: it becomes an empty line, and every stage +/// below rejects an empty line when it tries to read an s-expression from it. +fn split_lines(text: &str) -> Result, TextError> { + let body = text.strip_suffix('\n').ok_or(TextError::Syntax( + "a projection must end with exactly one U+000A and no other trailing whitespace", + ))?; + Ok(body.split('\n').collect()) +} + +/// The head symbol of a line's s-expression, or `None` if it is not headed by +/// one (a bare leaf line is never a valid section head, so this only ever +/// hides a genuine mismatch). +fn line_head(sexp: &Sexp) -> Option<&str> { + sexp.as_list()?.first()?.as_symbol() +} + +/// Consumes the next line and applies `parse` to it, but only if it is +/// present and its head symbol is `head`. Otherwise the cursor is left +/// untouched and `Ok(None)` is returned -- the mechanism by which every +/// optional or repeatable section in `projection` is optional, and by which a +/// line belonging to a later or earlier stage is left for another stage (or +/// the final leftover check) to deal with, rather than being consumed out of +/// place. +fn take_line<'a, T>( + lines: &mut Lines<'a>, + head: &str, + parse: impl FnOnce(&Sexp) -> Result, +) -> Result, TextError> { + let Some(line) = lines.peek() else { + return Ok(None); + }; + let sexp = read_sexp(line)?; + if line_head(&sexp) != Some(head) { + return Ok(None); + } + lines.advance(); + Ok(Some(parse(&sexp)?)) +} + +/// As [`take_line`], but the line is mandatory: a missing or mismatched line +/// is `missing`, not `Ok(None)`. Used for the header and the document line, +/// the two productions `projection` requires unconditionally. +fn require_line<'a, T>( + lines: &mut Lines<'a>, + head: &str, + missing: &'static str, + parse: impl FnOnce(&Sexp) -> Result, +) -> Result { + let Some(line) = lines.peek() else { + return Err(TextError::Syntax(missing)); + }; + let sexp = read_sexp(line)?; + if line_head(&sexp) != Some(head) { + return Err(TextError::Syntax(missing)); + } + lines.advance(); + parse(&sexp) +} + +/// `req:textproj:derived-ordering`'s "every other sequence keeps the binary +/// order" names the envelopes' binary order as +/// [`epiphany_ops::canonical_reduction_order`] -- a deterministic function of +/// the envelopes' own causal contexts and stamps, not a free choice a writer +/// makes. So a text whose envelope lines are already in that order is the +/// only text `req:textproj:canonical-text` permits for that envelope set, and +/// accepting some other order here -- and silently keeping it -- would be +/// exactly the normalization `req:textproj:strict-parse` forbids: the next +/// projection re-sorts the same envelopes into canonical order, so +/// `project(serialize(parse(T))) == T` fails for precisely the malformed `T` +/// this rejects up front. +fn check_envelope_order(envelopes: &[OperationEnvelope]) -> Result<(), TextError> { + let given: Vec<&OperationEnvelope> = envelopes.iter().collect(); + let canonical = canonical_reduction_order(&given); + let already_canonical = given + .iter() + .zip(canonical.iter()) + .all(|(a, b)| std::ptr::eq(*a, *b)); + if !already_canonical { + return Err(TextError::NotStrictlyIncreasing( + "envelope lines must appear in canonical reduction order \ + (epiphany_ops::canonical_reduction_order)", + )); + } + Ok(()) +} + +// =========================================================================== +// Leaves shared by several productions. +// =========================================================================== + +/// The lexical class of `s`, for error messages. Mirrors the private helper +/// of the same name in `epiphany_core::textvalue` and in every `textproj_*` +/// module of `epiphany-ops`: `Sexp::class` is not public, so each grammar- +/// directed module restates this one match rather than depend on it. +fn class_of(s: &Sexp) -> &'static str { + match s { + Sexp::List(_) => "list", + Sexp::Symbol(_) => "symbol", + Sexp::Int(_) => "integer", + Sexp::Bytes(_) => "byte string", + Sexp::Str(_) => "string", + } +} + +/// Reads the grammar's opaque `bytes` terminal as a plain byte string. +/// `Vec`'s generic `TextValue` impl denotes a *sequence of integers*, a +/// different production, so it must never stand in for an opaque payload, +/// identifier, or hash -- the same trap the operation layer names in +/// `textproj_kind.rs`. +fn parse_bytes(s: &Sexp) -> Result, TextError> { + match s { + Sexp::Bytes(bytes) => Ok(bytes.clone()), + _ => Err(TextError::Expected { + expected: "byte string", + found: class_of(s), + }), + } +} + +/// Reads a fixed 16-byte opaque identifier: a `DocumentId`, `LineageId`, +/// `SnapshotId`, `ExtensionId`, or a custom `ProfileId`'s registry id. `what` +/// names the identifier for the rejection message. +fn parse_id16(s: &Sexp, what: &'static str) -> Result<[u8; 16], TextError> { + parse_bytes(s)? + .try_into() + .map_err(|_| TextError::NotCanonical(what)) +} + +/// The grammar's anonymous `version ::= "(" integer " " integer " " integer +/// ")"` production, shared verbatim by the header (against +/// `crate::COMPANION_VERSION`) and by every `SemVer` field. +fn parse_version_triple(s: &Sexp) -> Result<(u32, u32, u32), TextError> { + let items = s.as_list().ok_or(TextError::Expected { + expected: "version", + found: class_of(s), + })?; + let [major, minor, patch] = items else { + return Err(TextError::Arity { + type_name: "version", + expected: 3, + found: items.len(), + }); + }; + Ok((u32::parse(major)?, u32::parse(minor)?, u32::parse(patch)?)) +} + +fn parse_semver(s: &Sexp) -> Result { + let (major, minor, patch) = parse_version_triple(s)?; + Ok(SemVer::new(major, minor, patch)) +} + +/// The grammar's `profile-id` production: a bare symbol for each +/// closed-vocabulary profile, and `(custom )` -- the one profile +/// identity that is not a bare symbol -- for `ProfileId::Custom`, whose +/// registry id is 16 opaque bytes. +fn parse_profile_id(s: &Sexp) -> Result { + if let Some(name) = s.as_symbol() { + return match name { + "full" => Ok(ProfileId::Full), + "read-only" => Ok(ProfileId::ReadOnly), + "lite" => Ok(ProfileId::Lite), + _ => Err(TextError::UnknownConstructor { + type_name: "ProfileId", + found: name.to_owned(), + }), + }; + } + let fields = s.expect_struct("custom", 1)?; + let registry_id = parse_id16( + &fields[0], + "a custom ProfileId registry id is exactly 16 bytes", + )?; + Ok(ProfileId::Custom(ProfileRegistryId(registry_id))) +} + +// =========================================================================== +// header, document, lineage. +// =========================================================================== + +/// `header ::= "(text-projection " version ")"`. Accepts exactly +/// `crate::COMPANION_VERSION`; any other version is a rejection at line one +/// (`req:textproj:header-version`). Multi-version acceptance and +/// migrate-on-read are deferred spec decisions this parser does not +/// speculate about. +fn parse_header(s: &Sexp) -> Result<(), TextError> { + let fields = s.expect_struct("text-projection", 1)?; + let version = parse_version_triple(&fields[0])?; + if version != COMPANION_VERSION { + return Err(TextError::NotCanonical( + "the header names a companion version other than the one this crate implements", + )); + } + Ok(()) +} + +/// `document ::= "(document " bytes ")"`. +fn parse_document_id_line(s: &Sexp) -> Result { + let fields = s.expect_struct("document", 1)?; + Ok(DocumentId(parse_id16( + &fields[0], + "a DocumentId is exactly 16 bytes", + )?)) +} + +/// `lineage ::= "(lineage " bytes ")"`. +fn parse_lineage(s: &Sexp) -> Result { + let fields = s.expect_struct("lineage", 1)?; + Ok(LineageId(parse_id16( + &fields[0], + "a LineageId is exactly 16 bytes", + )?)) +} + +// =========================================================================== +// profile, constraints, retention. +// =========================================================================== + +/// `profile ::= "(profile " profile-id " " version " " constraints ")"`. +fn parse_profile(s: &Sexp) -> Result { + let fields = s.expect_struct("profile", 3)?; + Ok(ProfileDeclaration { + profile_id: parse_profile_id(&fields[0])?, + version: parse_semver(&fields[1])?, + constraints: parse_constraints(&fields[2])?, + }) +} + +/// `constraints ::= "(constraints " integer " " retention ")"`. +fn parse_constraints(s: &Sexp) -> Result { + let fields = s.expect_struct("constraints", 2)?; + Ok(ProfileConstraints { + max_uncompressed_block_size: u64::parse(&fields[0])?, + retention_policy: parse_retention(&fields[1])?, + }) +} + +/// `retention ::= "(retention " integer " " option " " bool ")"`. +/// +/// The middle field is `Option`, and `WallClockDuration` is +/// a newtype over `i64`: per `req:textproj:value-projection` clause 2 a +/// newtype projects transparently as its field alone, so the option wraps a +/// bare integer, not a `(wall-clock-duration )` struct. This is +/// **`epiphany_bundle::WallClockDuration`**, not `epiphany_core`'s +/// same-named type -- the retention policy is a bundle type through and +/// through. +fn parse_retention(s: &Sexp) -> Result { + let fields = s.expect_struct("retention", 3)?; + Ok(RetentionPolicy { + retain_previous_manifests: u32::parse(&fields[0])?, + retain_duration: Option::::parse(&fields[1])?.map(WallClockDuration), + retain_named_checkpoints: bool::parse(&fields[2])?, + }) +} + +// =========================================================================== +// extension, chunk, chunk-kind, schema. +// =========================================================================== + +/// `extension ::= "(extension " bytes " " version " " bool " (" chunk* ") " +/// bytes " " bytes ")"` -- id, version, required, chunks, affected-kinds, +/// barriers, the ratified declaration order. `affected_object_kinds` and +/// `edit_barriers` are opaque byte strings, never structured: the bundle +/// preserves them without interpreting them, and this projection interprets +/// nothing the bundle does not. +fn parse_extension(s: &Sexp) -> Result { + let fields = s.expect_struct("extension", 6)?; + Ok(TextExtension { + extension_id: ExtensionId(parse_id16( + &fields[0], + "an ExtensionId is exactly 16 bytes", + )?), + version: parse_semver(&fields[1])?, + required: bool::parse(&fields[2])?, + chunks: parse_chunks(&fields[3])?, + affected_object_kinds: parse_bytes(&fields[4])?, + edit_barriers: parse_bytes(&fields[5])?, + }) +} + +/// `chunk ::= "(chunk " chunk-kind " " schema " " bytes ")"`: a preserved +/// extension chunk root projected as kind, schema version, and uncompressed +/// payload -- never a `ChunkRef`, which the projection has no file to point +/// into (`req:textproj:derive-or-carry`). +fn parse_chunk(s: &Sexp) -> Result { + let fields = s.expect_struct("chunk", 3)?; + Ok(TextChunk { + kind: parse_chunk_kind(&fields[0])?, + schema_version: parse_schema_version(&fields[1])?, + payload: parse_bytes(&fields[2])?, + }) +} + +/// The chunk* sequence inside an `extension` line. `req:textproj:derived-ordering` +/// orders and de-duplicates an extension's preserved chunk roots by their +/// *projected form* -- the binary `ChunkRef` order breaks ties on the file +/// offset, which this projection erases, so the binary order cannot be +/// inherited here. Accepting a disordered or duplicated chunk list and +/// re-sorting it at serialize time would be normalization; this rejects it +/// instead. +fn parse_chunks(s: &Sexp) -> Result, TextError> { + let items = s.as_list().ok_or(TextError::Expected { + expected: "chunk sequence", + found: class_of(s), + })?; + let mut out = Vec::with_capacity(items.len()); + let mut previous_rendered: Option = None; + for item in items { + let rendered = item.render(); + if previous_rendered + .as_deref() + .is_some_and(|previous| rendered.as_str() <= previous) + { + return Err(TextError::NotStrictlyIncreasing( + "an extension's preserved chunk roots must be ordered, without repetition, by \ + projected form", + )); + } + out.push(parse_chunk(item)?); + previous_rendered = Some(rendered); + } + Ok(out) +} + +/// `chunk-kind`: a bare symbol per `ChunkKind` variant, in the same +/// declaration order as the discriminants `req:format:chunkkind-discriminants` +/// pins. +fn parse_chunk_kind(s: &Sexp) -> Result { + match s.as_symbol() { + Some("operation-envelope-block") => Ok(ChunkKind::OperationEnvelopeBlock), + Some("operation-index") => Ok(ChunkKind::OperationIndex), + Some("snapshot") => Ok(ChunkKind::Snapshot), + Some("blob") => Ok(ChunkKind::Blob), + Some("extension-data") => Ok(ChunkKind::ExtensionData), + Some("text-projection") => Ok(ChunkKind::TextProjection), + Some("layout-cache") => Ok(ChunkKind::LayoutCache), + Some("integrity-index") => Ok(ChunkKind::IntegrityIndex), + Some("manifest") => Ok(ChunkKind::Manifest), + Some(name) => Err(TextError::UnknownConstructor { + type_name: "ChunkKind", + found: name.to_owned(), + }), + None => Err(TextError::Expected { + expected: "chunk-kind", + found: class_of(s), + }), + } +} + +/// `schema ::= "(schema " integer " " integer ")"`. +fn parse_schema_version(s: &Sexp) -> Result { + let fields = s.expect_struct("schema", 2)?; + Ok(SchemaVersion::new( + u16::parse(&fields[0])?, + u16::parse(&fields[1])?, + )) +} + +// =========================================================================== +// canonical-base. +// =========================================================================== + +/// `canonical-base ::= "(canonical-base " bytes " " bytes " " integer " " +/// profile-id " " schema " " bytes ")"` -- snapshot id, frontier, reduction +/// version, profile, root schema, root payload. The `SnapshotId` is the one +/// identity `req:textproj:derive-or-carry` carries verbatim rather than +/// re-deriving (schema major 0 has no snapshot producer, so it has nothing to +/// derive from); the root chunk's own id and content hash are re-derived by +/// whoever serializes this back into a bundle, never read from the text. +fn parse_canonical_base(s: &Sexp) -> Result { + let fields = s.expect_struct("canonical-base", 6)?; + Ok(TextCanonicalBase { + snapshot_id: SnapshotId(parse_id16(&fields[0], "a SnapshotId is exactly 16 bytes")?), + covers_causal_frontier: FrontierBytes::from_bytes(parse_bytes(&fields[1])?), + reduction_algorithm_version: ReductionAlgorithmVersion(u32::parse(&fields[2])?), + profile_id: parse_profile_id(&fields[3])?, + root_schema_version: parse_schema_version(&fields[4])?, + root_payload: parse_bytes(&fields[5])?, + }) +} + +// =========================================================================== +// blob. +// =========================================================================== + +/// `blob ::= "(blob " string " " option " " bytes ")"` -- media type, +/// declared maximum uncompressed length, payload. +/// +/// This is the production-level parser only. It accepts a well-formed +/// `(blob ...)` line unconditionally, exactly as every other production +/// parser here does; the decision that **no** accepted blob line may reach a +/// [`TextDocument`] is made once, at the document level, in +/// [`parse_document`] (`req:textproj:reject-unreferenced-blobs`). Keeping the +/// two apart -- this function parses, `parse_document` rejects -- means the +/// accept side is already correct and already tested the day the +/// reachability predicate stops returning the empty set, rather than needing +/// to be written from scratch alongside a spec bump. +fn parse_blob(s: &Sexp) -> Result { + let fields = s.expect_struct("blob", 3)?; + Ok(TextBlob { + media_type: String::parse(&fields[0])?, + declared_max_uncompressed_length: Option::::parse(&fields[1])?, + payload: parse_bytes(&fields[2])?, + }) +} + +// =========================================================================== +// Tests. +// =========================================================================== + +#[cfg(test)] +mod tests { + use super::*; + use epiphany_core::{OperationId, ReplicaId, WallClockTime}; + use epiphany_ops::{ + project_envelope, AuthorId, CausalContext, EnvelopeHash, HybridLogicalClock, + OperationStamp, ResolveEquivocationPayload, + }; + + // ----------------------------------------------------------------- + // Test fixtures. + // ----------------------------------------------------------------- + + /// Joins `lines` with a single U+000A each, including a final one -- + /// exactly `req:textproj:envelope-per-line`'s layout. + fn projection(lines: &[&str]) -> String { + let mut out = String::new(); + for line in lines { + out.push_str(line); + out.push('\n'); + } + out + } + + const HEADER: &str = "(text-projection (0 7 0))"; + const DOCUMENT: &str = "(document #x00000000000000000000000000000001)"; + + /// A minimal but complete valid projection: just the two mandatory lines. + fn minimal_valid_document() -> String { + projection(&[HEADER, DOCUMENT]) + } + + /// A simple, independent (empty causal context) envelope, so several of + /// these can be combined without any causal-order machinery beyond their + /// HLC physical time. + fn sample_envelope(replica: u64, counter: u64, physical_time: i64) -> OperationEnvelope { + let id = OperationId::new(ReplicaId(replica), counter); + OperationEnvelope { + id, + author: AuthorId(0x1122_3344), + stamp: OperationStamp::new( + HybridLogicalClock::new(WallClockTime(physical_time), 0), + id, + ), + causal_context: CausalContext::new(), + transaction: None, + payload: epiphany_ops::OperationPayload::ResolveEquivocation( + ResolveEquivocationPayload { + target: id, + chosen: EnvelopeHash([7; 32]), + }, + ), + } + } + + // ----------------------------------------------------------------- + // The worked example: a real, spec-authored, multi-section document. + // ----------------------------------------------------------------- + + /// The Grammar chapter's worked example, read live from the companion so + /// this test cannot drift from the document it claims to parse. + fn worked_example() -> String { + const SPEC: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../spec/text_projection.tex" + )); + SPEC.split_once("\\chapter{A Worked Example}") + .expect("the specification contains the worked example") + .1 + .split_once("\\begin{lstlisting}\n") + .expect("the worked example contains a projection listing") + .1 + .split_once("\\end{lstlisting}") + .expect("the worked projection listing is closed") + .0 + .to_owned() + } + + #[test] + fn the_worked_example_parses_to_its_documented_fields() { + let text = worked_example(); + let document = parse_document(&text).expect("the worked example is a valid projection"); + + assert_eq!(document.document_id, DocumentId([0x05; 16])); + assert_eq!(document.lineage_id, None); + assert_eq!(document.profiles.len(), 1); + assert_eq!(document.profiles[0].profile_id, ProfileId::Full); + assert_eq!(document.profiles[0].version, SemVer::new(0, 1, 0)); + assert_eq!( + document.profiles[0].constraints.max_uncompressed_block_size, + 67_108_864 + ); + assert_eq!( + document.profiles[0] + .constraints + .retention_policy + .retain_previous_manifests, + 1 + ); + assert_eq!( + document.profiles[0] + .constraints + .retention_policy + .retain_duration, + None + ); + assert!( + document.profiles[0] + .constraints + .retention_policy + .retain_named_checkpoints + ); + assert!(document.extensions.is_empty()); + + let base = document + .canonical_base + .as_ref() + .expect("the worked example carries a canonical base"); + assert_eq!( + base.snapshot_id, + SnapshotId([0x1f, 0x8b, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + ); + assert_eq!(base.covers_causal_frontier, FrontierBytes::empty()); + assert_eq!( + base.reduction_algorithm_version, + ReductionAlgorithmVersion(1) + ); + assert_eq!(base.profile_id, ProfileId::Full); + assert_eq!(base.root_schema_version, SchemaVersion::V0); + assert_eq!(base.root_payload, vec![0u8, 0u8]); + + assert!(document.blobs.is_empty()); + assert_eq!(document.envelopes.len(), 1); + } + + // ----------------------------------------------------------------- + // A synthetic document exercising lineage, multiple profiles, an + // extension with preserved chunks, a canonical base, and multiple + // envelopes -- everything the worked example alone does not reach. + // ----------------------------------------------------------------- + + #[test] + fn a_rich_synthetic_document_parses_every_section() { + let lineage = "(lineage #x00000000000000000000000000000002)"; + let profile_full = "(profile full (0 1 0) (constraints 67108864 (retention 1 () true)))"; + let profile_read_only = + "(profile read-only (0 1 0) (constraints 1024 (retention 0 (some 5) false)))"; + let extension = "(extension #x00000000000000000000000000000003 (1 0 0) true \ + ((chunk operation-envelope-block (schema 0 1) #xaa) (chunk snapshot (schema 0 1) #xbb)) \ + #xaabb #xccdd)"; + let base = + "(canonical-base #x00000000000000000000000000000004 #x 1 full (schema 0 1) #x0102)"; + + let e1 = sample_envelope(1, 1, 10); + let e2 = sample_envelope(2, 1, 20); + let e1_line = project_envelope(&e1); + let e2_line = project_envelope(&e2); + + let text = projection(&[ + HEADER, + DOCUMENT, + lineage, + profile_full, + profile_read_only, + extension, + base, + &e1_line, + &e2_line, + ]); + + let document = parse_document(&text).expect("a well-formed rich document parses"); + assert_eq!( + document.lineage_id, + Some(LineageId([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) + ); + assert_eq!(document.profiles.len(), 2); + assert_eq!(document.extensions.len(), 1); + assert_eq!(document.extensions[0].chunks.len(), 2); + assert_eq!( + document.extensions[0].chunks[0].kind, + ChunkKind::OperationEnvelopeBlock + ); + assert_eq!(document.extensions[0].chunks[1].kind, ChunkKind::Snapshot); + assert_eq!( + document.extensions[0].affected_object_kinds, + vec![0xaa, 0xbb] + ); + assert_eq!(document.extensions[0].edit_barriers, vec![0xcc, 0xdd]); + assert!(document.canonical_base.is_some()); + assert_eq!(document.envelopes, vec![e1, e2]); + } + + // ----------------------------------------------------------------- + // Rejection classes. Each is mutation-verified (see the task report): + // the anchor assertion below is deleted or inverted, a NAMED test in + // this list is confirmed to fail, and the code is restored. + // ----------------------------------------------------------------- + + #[test] + fn non_canonical_header_version_is_rejected() { + let text = projection(&["(text-projection (0 6 0))", DOCUMENT]); + assert_eq!( + parse_document(&text), + Err(TextError::NotCanonical( + "the header names a companion version other than the one this crate implements" + )) + ); + } + + #[test] + fn the_companion_version_itself_is_accepted() { + // The mutation-testing counterpart of the above: the exact version + // this crate implements must still parse. + assert!(parse_document(&minimal_valid_document()).is_ok()); + } + + #[test] + fn a_blob_line_is_always_rejected() { + let blob_line = "(blob \"audio/wav\" () #x00)"; + let text = projection(&[HEADER, DOCUMENT, blob_line]); + assert!(matches!( + parse_document(&text), + Err(TextError::NotCanonical(_)) + )); + } + + #[test] + fn the_blob_production_itself_parses_synthetic_data() { + // Line-level parse of `(blob ...)`, exercised directly against + // synthetic data per the task contract: the accept side is ready + // even though `parse_document` never lets it through. + let sexp = read_sexp("(blob \"audio/wav\" (some 1024) #x0a0b)").unwrap(); + let blob = parse_blob(&sexp).expect("a well-formed blob production parses"); + assert_eq!(blob.media_type, "audio/wav"); + assert_eq!(blob.declared_max_uncompressed_length, Some(1024)); + assert_eq!(blob.payload, vec![0x0a, 0x0b]); + } + + #[test] + fn missing_trailing_lf_is_rejected() { + let text = minimal_valid_document(); + let without_final_lf = text.strip_suffix('\n').unwrap(); + assert_eq!( + parse_document(without_final_lf), + Err(TextError::Syntax( + "a projection must end with exactly one U+000A and no other trailing whitespace" + )) + ); + } + + #[test] + fn an_extra_trailing_blank_line_is_rejected() { + let mut text = minimal_valid_document(); + text.push('\n'); // a second, stray trailing U+000A + assert!(parse_document(&text).is_err()); + } + + #[test] + fn missing_header_is_rejected() { + assert!(parse_document("").is_err()); + } + + #[test] + fn missing_document_line_is_rejected() { + let text = projection(&[HEADER]); + assert_eq!( + parse_document(&text), + Err(TextError::Syntax( + "a projection must carry a document line immediately after its header" + )) + ); + } + + #[test] + fn a_repeated_header_line_is_rejected() { + let text = projection(&[HEADER, HEADER, DOCUMENT]); + assert!(parse_document(&text).is_err()); + } + + #[test] + fn a_repeated_document_line_is_rejected() { + let text = projection(&[HEADER, DOCUMENT, DOCUMENT]); + assert_eq!( + parse_document(&text), + Err(TextError::Syntax( + "a line appears out of the projection's normative section order, or a section \ + that may occur at most once is repeated" + )) + ); + } + + #[test] + fn out_of_order_sections_are_rejected_not_sorted() { + let profile = "(profile full (0 1 0) (constraints 67108864 (retention 1 () true)))"; + let extension = "(extension #x00000000000000000000000000000003 (1 0 0) false () #x #x)"; + let base = "(canonical-base #x00000000000000000000000000000004 #x 1 full (schema 0 1) #x)"; + let lineage = "(lineage #x00000000000000000000000000000002)"; + let e1_line = project_envelope(&sample_envelope(1, 1, 10)); + + let scenarios: &[&[&str]] = &[ + // A profile line before the (mandatory) document line. + &[HEADER, profile, DOCUMENT], + // Lineage after a profile, instead of before it. + &[HEADER, DOCUMENT, profile, lineage], + // An extension before the profile section. + &[HEADER, DOCUMENT, extension, profile], + // A canonical base before the extension section. + &[HEADER, DOCUMENT, base, extension], + // An envelope before the canonical base. + &[HEADER, DOCUMENT, &e1_line, base], + ]; + for lines in scenarios { + let text = projection(lines); + assert!( + parse_document(&text).is_err(), + "expected rejection for out-of-order lines: {lines:?}" + ); + } + } + + #[test] + fn profile_declarations_must_be_ordered_and_unique() { + let full = "(profile full (0 1 0) (constraints 1 (retention 0 () false)))"; + let read_only = "(profile read-only (0 1 0) (constraints 1 (retention 0 () false)))"; + + // Correct ascending order accepts. + let ok = projection(&[HEADER, DOCUMENT, full, read_only]); + assert!(parse_document(&ok).is_ok()); + + // Reversed order is rejected, not silently re-sorted. + let reversed = projection(&[HEADER, DOCUMENT, read_only, full]); + assert_eq!( + parse_document(&reversed), + Err(TextError::NotStrictlyIncreasing( + "profile declarations must be ordered, without repetition, by (profile_id, version)" + )) + ); + + // An exact repeat (same key) is rejected too. + let repeated = projection(&[HEADER, DOCUMENT, full, full]); + assert!(parse_document(&repeated).is_err()); + } + + #[test] + fn extension_declarations_must_be_ordered_and_unique() { + let ext_a = "(extension #x00000000000000000000000000000001 (1 0 0) false () #x #x)"; + let ext_b = "(extension #x00000000000000000000000000000002 (1 0 0) false () #x #x)"; + + let ok = projection(&[HEADER, DOCUMENT, ext_a, ext_b]); + assert!(parse_document(&ok).is_ok()); + + let reversed = projection(&[HEADER, DOCUMENT, ext_b, ext_a]); + assert_eq!( + parse_document(&reversed), + Err(TextError::NotStrictlyIncreasing( + "extension declarations must be ordered, without repetition, by (extension_id, version)" + )) + ); + + let repeated = projection(&[HEADER, DOCUMENT, ext_a, ext_a]); + assert!(parse_document(&repeated).is_err()); + } + + #[test] + fn extension_chunk_roots_must_be_ordered_and_deduplicated() { + let chunk_a = "(chunk operation-envelope-block (schema 0 1) #xaa)"; + let chunk_b = "(chunk snapshot (schema 0 1) #xbb)"; + + let ordered = format!( + "(extension #x00000000000000000000000000000001 (1 0 0) false ({chunk_a} {chunk_b}) #x #x)" + ); + let text = projection(&[HEADER, DOCUMENT, &ordered]); + assert!(parse_document(&text).is_ok()); + + let reversed = format!( + "(extension #x00000000000000000000000000000001 (1 0 0) false ({chunk_b} {chunk_a}) #x #x)" + ); + let text = projection(&[HEADER, DOCUMENT, &reversed]); + assert_eq!( + parse_document(&text), + Err(TextError::NotStrictlyIncreasing( + "an extension's preserved chunk roots must be ordered, without repetition, by \ + projected form" + )) + ); + + let duplicated = format!( + "(extension #x00000000000000000000000000000001 (1 0 0) false ({chunk_a} {chunk_a}) #x #x)" + ); + let text = projection(&[HEADER, DOCUMENT, &duplicated]); + assert!(parse_document(&text).is_err()); + } + + #[test] + fn envelope_lines_must_be_in_canonical_reduction_order() { + let e1 = sample_envelope(1, 1, 10); + let e2 = sample_envelope(2, 1, 20); + let e1_line = project_envelope(&e1); + let e2_line = project_envelope(&e2); + + // Ascending physical time is the canonical order: accepted. + let ok = projection(&[HEADER, DOCUMENT, &e1_line, &e2_line]); + let parsed = parse_document(&ok).expect("already-canonical envelope order is accepted"); + assert_eq!(parsed.envelopes, vec![e1, e2]); + + // Swapped: no longer canonical reduction order, rejected outright. + let swapped = projection(&[HEADER, DOCUMENT, &e2_line, &e1_line]); + assert_eq!( + parse_document(&swapped), + Err(TextError::NotStrictlyIncreasing( + "envelope lines must appear in canonical reduction order \ + (epiphany_ops::canonical_reduction_order)" + )) + ); + } + + #[test] + fn profile_id_accepts_every_closed_symbol_and_the_custom_form() { + for (text, expected) in [ + ("full", ProfileId::Full), + ("read-only", ProfileId::ReadOnly), + ("lite", ProfileId::Lite), + ( + "(custom #x00000000000000000000000000000009)", + ProfileId::Custom(ProfileRegistryId([ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, + ])), + ), + ] { + let sexp = read_sexp(text).unwrap(); + assert_eq!(parse_profile_id(&sexp).unwrap(), expected); + } + } + + #[test] + fn profile_id_rejects_an_unknown_symbol_and_a_malformed_custom_id() { + assert!(parse_profile_id(&read_sexp("bespoke").unwrap()).is_err()); + assert!(parse_profile_id(&read_sexp("(custom #x00)").unwrap()).is_err()); + } + + #[test] + fn chunk_kind_covers_exactly_the_nine_variants() { + for (name, kind) in [ + ( + "operation-envelope-block", + ChunkKind::OperationEnvelopeBlock, + ), + ("operation-index", ChunkKind::OperationIndex), + ("snapshot", ChunkKind::Snapshot), + ("blob", ChunkKind::Blob), + ("extension-data", ChunkKind::ExtensionData), + ("text-projection", ChunkKind::TextProjection), + ("layout-cache", ChunkKind::LayoutCache), + ("integrity-index", ChunkKind::IntegrityIndex), + ("manifest", ChunkKind::Manifest), + ] { + let sexp = read_sexp(name).unwrap(); + assert_eq!(parse_chunk_kind(&sexp).unwrap(), kind); + } + assert!(parse_chunk_kind(&read_sexp("unknown-kind").unwrap()).is_err()); + } + + // ----------------------------------------------------------------- + // The suite's own reach. + // ----------------------------------------------------------------- + + /// Distinct rejection classes this file's tests exercise. Each entry + /// corresponds to a check in `parse_document` or a leaf parser above that + /// rejects rather than normalizes; the list documents, and the assertion + /// pins, how many of them this file actually drives to a rejection. + const EXERCISED_REJECTION_CLASSES: &[&str] = &[ + "non-canonical header version", + "a (blob ...) line, wherever it appears", + "missing final U+000A", + "an extra trailing U+000A", + "missing header line", + "missing document line", + "repeated header line", + "repeated document line", + "a section line before the mandatory document line", + "a section reappearing after a later section (lineage after profile)", + "an extension before the profile section", + "a canonical-base before the extension section", + "an envelope before the canonical base", + "profile declarations out of (profile_id, version) order", + "duplicate profile declaration key", + "extension declarations out of (extension_id, version) order", + "duplicate extension declaration key", + "extension chunk roots out of projected-form order", + "duplicate extension chunk roots", + "envelope lines out of canonical reduction order", + "an unknown ProfileId symbol", + "a malformed custom ProfileId registry id", + "an unknown ChunkKind symbol", + ]; + + #[test] + fn the_suite_s_reach_is_counted() { + assert_eq!( + EXERCISED_REJECTION_CLASSES.len(), + 23, + "update this count, and add a test, whenever a new rejection class is exercised" + ); + } +} diff --git a/crates/epiphany-textproj/src/project.rs b/crates/epiphany-textproj/src/project.rs new file mode 100644 index 0000000..b95685a --- /dev/null +++ b/crates/epiphany-textproj/src/project.rs @@ -0,0 +1,1176 @@ +//! Projection of canonical bundle documents to their normative text form. +//! +//! This module implements the `Bundle -> TextDocument -> text` half of the +//! companion: [`document_from_bundle`] reads a bundle's manifest and its +//! reachable chunk/blob payloads into a [`TextDocument`], and the `project_*` +//! functions turn each grammar production of `spec/text_projection.tex`'s +//! Grammar chapter into its canonical [`Sexp`]. [`project_text_document`] +//! assembles a whole document's lines in the normative `projection` sequence, +//! and [`project_bundle`] composes both stages. +//! +//! # Grammar-directed, not value-directed +//! +//! `req:textproj:operation-vocabulary` established, for the operation layer, +//! that the grammar's productions govern rather than the mechanical +//! `TextValue` rule; the document layer is the same shape +//! (`CONTRACT_TEXTPROJ_DOCUMENT.md`). No document-line production contains a +//! `value` position, so every function here is a **free function**, never a +//! `TextValue` impl: `epiphany-bundle` does not depend on `epiphany-core`, and +//! implementing a foreign trait for a foreign type would be the orphan rule +//! violation the contract calls out. +//! +//! Envelope lines are the one exception: they are already a solved problem one +//! layer down. [`epiphany_ops::project_envelope`] projects an +//! [`OperationEnvelope`] line, and [`epiphany_ops::canonical_reduction_order`] +//! is the single function that orders the whole operation set. Both are used +//! here, not reimplemented. +//! +//! # What is deliberately absent +//! +//! Per `req:textproj:derive-or-carry`, nothing here carries a chunk's or +//! blob's `offset`, `compressed_length`, `compression`, or +//! `uncompressed_length`, nor a `ChunkId`, `ContentHash`, or `BlobId`: every +//! one is either a serializer's free physical choice or a value re-derived +//! from content. The non-canonical accelerators +//! (`operation_index_root`, `acceleration_snapshots`, `text_projection_root`, +//! `integrity_root`, `operation_block_summaries`) are never read here either. +//! A bundle that round-trips through text comes back without them; that is +//! correct; it is not data loss, because none of the five contributes to +//! canonical document semantics. + +use std::collections::BTreeSet; + +use epiphany_bundle::{ + BlobId, BlockStore, Bundle, BundleError, ChunkKind, DocumentId, LineageId, ProfileConstraints, + ProfileDeclaration, ProfileId, RetentionPolicy, SchemaVersion, SemVer, +}; +use epiphany_core::textvalue::Sexp; +use epiphany_ops::{ + canonical_reduction_order, decode_envelope, project_envelope, EnvelopeDecodeError, + OperationEnvelope, +}; + +use crate::{ + TextBlob, TextCanonicalBase, TextChunk, TextDocument, TextExtension, COMPANION_VERSION, +}; + +// =========================================================================== +// Errors. +// =========================================================================== + +/// A failure encountered projecting a bundle to its Text Projection text. +#[derive(Debug)] +pub enum ProjectError { + /// Reading a chunk, an operation-envelope block, or a blob through the + /// bundle failed. + Bundle(BundleError), + /// A stored operation-envelope block held bytes that do not decode to a + /// canonical [`OperationEnvelope`]. + Envelope(EnvelopeDecodeError), +} + +impl core::fmt::Display for ProjectError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + ProjectError::Bundle(e) => write!(f, "bundle read failed: {e}"), + ProjectError::Envelope(e) => write!(f, "operation envelope failed to decode: {e}"), + } + } +} + +impl std::error::Error for ProjectError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + ProjectError::Bundle(e) => Some(e), + ProjectError::Envelope(e) => Some(e), + } + } +} + +impl From for ProjectError { + fn from(e: BundleError) -> Self { + ProjectError::Bundle(e) + } +} + +impl From for ProjectError { + fn from(e: EnvelopeDecodeError) -> Self { + ProjectError::Envelope(e) + } +} + +// =========================================================================== +// Small shared helpers. +// =========================================================================== + +/// The grammar's `bool` terminal: `true` or `false`, spelled as a symbol. +fn project_bool(value: bool) -> Sexp { + Sexp::sym(if value { "true" } else { "false" }) +} + +/// The grammar's `option` terminal: `()` when absent, `(some )` when +/// present, projecting the payload with `project`. +fn project_option(value: Option, project: impl FnOnce(T) -> Sexp) -> Sexp { + match value { + None => Sexp::none(), + Some(v) => Sexp::some(project(v)), + } +} + +/// The grammar's shared `version` shape, `( )`: used +/// verbatim by the header (the companion version) and, restated as a +/// [`SemVer`], by profile/extension declarations and the canonical base. +fn project_version_triple(major: u32, minor: u32, patch: u32) -> Sexp { + Sexp::List(vec![Sexp::int(major), Sexp::int(minor), Sexp::int(patch)]) +} + +fn project_semver(version: &SemVer) -> Sexp { + project_version_triple(version.major, version.minor, version.patch) +} + +/// Orders and de-duplicates already-projected elements by the **UTF-8 bytes of +/// their rendered form**, ascending, keeping at most one per distinct +/// rendering (`req:textproj:derived-ordering`). Used for exactly the two +/// sequences the requirement names: the `(blob ...)` lines, and one +/// extension's preserved chunk lines. Every other projected sequence keeps +/// whatever order its caller already put it in (the manifest's binary order, +/// which for those sequences is itself a function only of preserved data). +/// +/// This is deliberately a *projection-time* operation rather than a +/// construction-time one: it must produce the correct text even when handed +/// an out-of-order or duplicated slice, which is exactly what +/// `derived_ordering_sorts_and_dedups_blobs_and_chunks` below constructs and +/// checks. +fn ordered_by_projected_form(items: &[T], project: impl Fn(&T) -> Sexp) -> Vec { + let mut rendered: Vec<(String, Sexp)> = items + .iter() + .map(|item| { + let sexp = project(item); + let text = sexp.render(); + (text, sexp) + }) + .collect(); + rendered.sort_by(|(a, _), (b, _)| a.cmp(b)); + rendered.dedup_by(|(a, _), (b, _)| a == b); + rendered.into_iter().map(|(_, sexp)| sexp).collect() +} + +// =========================================================================== +// header, document, lineage. +// =========================================================================== + +/// `header ::= "(text-projection " version ")" LF`. +/// +/// Always [`COMPANION_VERSION`]: `req:textproj:header-version` fixes the +/// header to name the one companion version this crate implements. +pub fn project_header() -> Sexp { + let (major, minor, patch) = COMPANION_VERSION; + Sexp::List(vec![ + Sexp::sym("text-projection"), + project_version_triple(major, minor, patch), + ]) +} + +/// `document ::= "(document " bytes ")" LF`. +pub fn project_document(document_id: &DocumentId) -> Sexp { + Sexp::List(vec![ + Sexp::sym("document"), + Sexp::Bytes(document_id.as_bytes().to_vec()), + ]) +} + +/// `lineage ::= "(lineage " bytes ")" LF`. +pub fn project_lineage(lineage_id: &LineageId) -> Sexp { + Sexp::List(vec![ + Sexp::sym("lineage"), + Sexp::Bytes(lineage_id.as_bytes().to_vec()), + ]) +} + +// =========================================================================== +// profile, profile-id, constraints, retention. +// =========================================================================== + +/// `profile-id ::= "full" | "read-only" | "lite" | "(custom " bytes ")"`. +/// +/// `Custom` is the one profile identity that is not a bare symbol: it carries +/// a 16-byte registry id (`req:textproj:profile-id`). +pub fn project_profile_id(profile_id: &ProfileId) -> Sexp { + match profile_id { + ProfileId::Full => Sexp::sym("full"), + ProfileId::ReadOnly => Sexp::sym("read-only"), + ProfileId::Lite => Sexp::sym("lite"), + ProfileId::Custom(registry_id) => Sexp::List(vec![ + Sexp::sym("custom"), + Sexp::Bytes(registry_id.as_bytes().to_vec()), + ]), + } +} + +/// `retention ::= "(retention " integer " " option " " bool ")"`. +/// +/// Fields, positionally: `retain_previous_manifests`, `retain_duration`, +/// `retain_named_checkpoints`. `WallClockDuration` is a newtype over `i64`, so +/// it projects transparently as the bare integer inside the option, per the +/// value-projection rule for newtypes. +pub fn project_retention(retention: &RetentionPolicy) -> Sexp { + Sexp::List(vec![ + Sexp::sym("retention"), + Sexp::int(retention.retain_previous_manifests), + project_option(retention.retain_duration, |duration| Sexp::int(duration.0)), + project_bool(retention.retain_named_checkpoints), + ]) +} + +/// `constraints ::= "(constraints " integer " " retention ")"`. +pub fn project_constraints(constraints: &ProfileConstraints) -> Sexp { + Sexp::List(vec![ + Sexp::sym("constraints"), + Sexp::int(constraints.max_uncompressed_block_size), + project_retention(&constraints.retention_policy), + ]) +} + +/// `profile ::= "(profile " profile-id " " version " " constraints ")" LF`. +pub fn project_profile(profile: &ProfileDeclaration) -> Sexp { + Sexp::List(vec![ + Sexp::sym("profile"), + project_profile_id(&profile.profile_id), + project_semver(&profile.version), + project_constraints(&profile.constraints), + ]) +} + +// =========================================================================== +// chunk, chunk-kind, schema, extension. +// =========================================================================== + +/// `chunk-kind ::= "operation-envelope-block" | "operation-index" | "snapshot" +/// | "blob" | "extension-data" | "text-projection" | "layout-cache" +/// | "integrity-index" | "manifest"`. +/// +/// Exhaustive over [`ChunkKind`]'s nine variants: adding a tenth to the bundle +/// crate is a compile error here until this match is extended, rather than a +/// silently-unprojectable kind. +pub fn project_chunk_kind(kind: ChunkKind) -> Sexp { + Sexp::sym(match kind { + ChunkKind::OperationEnvelopeBlock => "operation-envelope-block", + ChunkKind::OperationIndex => "operation-index", + ChunkKind::Snapshot => "snapshot", + ChunkKind::Blob => "blob", + ChunkKind::ExtensionData => "extension-data", + ChunkKind::TextProjection => "text-projection", + ChunkKind::LayoutCache => "layout-cache", + ChunkKind::IntegrityIndex => "integrity-index", + ChunkKind::Manifest => "manifest", + }) +} + +/// `schema ::= "(schema " integer " " integer ")"`. +pub fn project_schema(schema: &SchemaVersion) -> Sexp { + Sexp::List(vec![ + Sexp::sym("schema"), + Sexp::int(schema.major), + Sexp::int(schema.minor), + ]) +} + +/// `chunk ::= "(chunk " chunk-kind " " schema " " bytes ")"`. +/// +/// The bytes are the chunk's **uncompressed payload**, never a `ChunkRef`: +/// the projection has no file to point into +/// (`req:textproj:extension-declaration`). +pub fn project_chunk(chunk: &TextChunk) -> Sexp { + Sexp::List(vec![ + Sexp::sym("chunk"), + project_chunk_kind(chunk.kind), + project_schema(&chunk.schema_version), + Sexp::Bytes(chunk.payload.clone()), + ]) +} + +/// `extension ::= "(extension " bytes " " version " " bool " (" chunk* ") " +/// bytes " " bytes ")" LF`, fields in the ratified declaration order: id, +/// version, required, chunks, affected-kinds, barriers. +/// +/// `affected_object_kinds` and `edit_barriers` are opaque byte strings, never +/// structured sequences: the bundle preserves them without interpreting them, +/// and the projection interprets nothing the bundle does not +/// (`req:textproj:extension-declaration`). The preserved-chunk list is ordered +/// and de-duplicated by projected form (`req:textproj:derived-ordering`), +/// because `ChunkRef`'s binary order breaks ties on the physical offset. +pub fn project_extension(extension: &TextExtension) -> Sexp { + let chunks = ordered_by_projected_form(&extension.chunks, project_chunk); + Sexp::List(vec![ + Sexp::sym("extension"), + Sexp::Bytes(extension.extension_id.as_bytes().to_vec()), + project_semver(&extension.version), + project_bool(extension.required), + Sexp::List(chunks), + Sexp::Bytes(extension.affected_object_kinds.clone()), + Sexp::Bytes(extension.edit_barriers.clone()), + ]) +} + +// =========================================================================== +// canonical-base. +// =========================================================================== + +/// `canonical-base ::= "(canonical-base " bytes " " bytes " " integer " " +/// profile-id " " schema " " bytes ")" LF`: snapshot id, frontier, reduction +/// version, profile, root schema, root payload. +/// +/// The `SnapshotId` is the one identity carried verbatim rather than derived +/// (`req:textproj:derive-or-carry`); the root chunk's kind is `Snapshot` by +/// role and is not written, and its `ChunkId`/hash are re-derived from the +/// schema and payload carried here (`req:textproj:base-snapshot-inline`). +pub fn project_canonical_base(base: &TextCanonicalBase) -> Sexp { + Sexp::List(vec![ + Sexp::sym("canonical-base"), + Sexp::Bytes(base.snapshot_id.as_bytes().to_vec()), + Sexp::Bytes(base.covers_causal_frontier.as_bytes().to_vec()), + Sexp::int(base.reduction_algorithm_version.0), + project_profile_id(&base.profile_id), + project_schema(&base.root_schema_version), + Sexp::Bytes(base.root_payload.clone()), + ]) +} + +// =========================================================================== +// blob. +// =========================================================================== + +/// `blob ::= "(blob " string " " option " " bytes ")" LF`: media type, +/// declared maximum uncompressed length, payload. +/// +/// The `BlobId`, content hash, offset, lengths, and compression are all +/// re-derived or freely chosen by a serializer, never carried +/// (`req:textproj:derive-or-carry`, `req:textproj:canonical-blobs`). +pub fn project_blob(blob: &TextBlob) -> Sexp { + Sexp::List(vec![ + Sexp::sym("blob"), + Sexp::Str(blob.media_type.clone()), + project_option(blob.declared_max_uncompressed_length, Sexp::int), + Sexp::Bytes(blob.payload.clone()), + ]) +} + +// =========================================================================== +// Canonical-blob reachability (`req:textproj:canonical-blobs`). +// =========================================================================== + +/// The `BlobId`s canonically reachable from `envelopes`: referenced by a +/// canonical operation, or by canonical reduced state +/// (`req:textproj:canonical-blobs`; core specification Chapter 8 +/// §"Canonical and Non-Canonical Manifest Roots"). A blob referenced only by +/// an acceleration structure is **not** in this set and MUSTNOT be projected. +/// +/// This is a real predicate over the decoded canonical operations, not an +/// assertion of emptiness: it is written so that the day an operation payload +/// or a Chapter-5 value gains a field that names a blob, extending the walk +/// below is the one and only change needed to make that blob projectable. +/// +/// That day has not arrived. Every [`OperationPayload`](epiphany_ops::OperationPayload) +/// variant and every [`OperationKind`](epiphany_ops::OperationKind) variant is +/// exhaustively enumerated by `epiphany-ops`, and canonical reduced state +/// (`epiphany_ops::MaterializedState`) is built exclusively from those same +/// payloads — and the token `BlobId` does not occur anywhere in +/// `epiphany-core` or `epiphany-ops` (verified by +/// `blobid_is_absent_from_core_and_ops_source`, below, which fails the moment +/// it does). So there is no field, on any canonical value or payload this +/// crate can name, to extract a blob reference from — and this function's +/// result is provably the empty set today. +fn canonically_reachable_blob_ids(_envelopes: &[OperationEnvelope]) -> BTreeSet { + // Nothing to walk: see the doc comment above and the trip-wire test. + BTreeSet::new() +} + +/// The canonical blobs of a document, read through the bundle: the +/// [`TextBlob`]s of exactly the `BlobRef`s in `bundle.manifest().blob_roots` +/// whose id is canonically reachable from `envelopes`. +/// +/// **Never** `bundle.manifest().blob_roots` wholesale — that is the plausible +/// wrong answer named by the contract: one line, looks right, and emits every +/// non-canonical blob the manifest happens to carry, in direct violation of +/// `req:textproj:canonical-blobs`. Filtering through +/// [`canonically_reachable_blob_ids`] is what keeps this correct even though, +/// today, that filter is always the empty set — so this function always +/// returns `Ok(Vec::new())` for a real bundle, and does so *because* nothing +/// resolves as reachable, not because the manifest's field is skipped by +/// construction. +fn canonical_blobs( + bundle: &Bundle, + envelopes: &[OperationEnvelope], +) -> Result, ProjectError> { + let reachable = canonically_reachable_blob_ids(envelopes); + let mut blobs = Vec::new(); + for blob_ref in &bundle.manifest().blob_roots { + if reachable.contains(&blob_ref.blob_id) { + let payload = bundle.read_blob(blob_ref)?; + blobs.push(TextBlob { + media_type: blob_ref.media_type.clone(), + declared_max_uncompressed_length: blob_ref.declared_max_uncompressed_length, + payload, + }); + } + } + Ok(blobs) +} + +// =========================================================================== +// Bundle -> TextDocument. +// =========================================================================== + +/// Builds a [`TextDocument`] from a bundle's current manifest, reading every +/// chunk and blob payload the projection carries inline: each extension's +/// preserved chunks, the canonical base's root chunk, and (today, always +/// none) canonically reachable blobs. +/// +/// Operation envelopes are decoded from **every** operation-envelope block the +/// manifest references (`manifest.operation_roots`; block boundaries are a +/// storage artifact, not semantic structure) via +/// [`epiphany_ops::decode_envelope`], then put into +/// [`epiphany_ops::canonical_reduction_order`] — computed once over the whole +/// gathered set, because the order is causal and cannot be decided per block. +pub fn document_from_bundle( + bundle: &Bundle, +) -> Result { + let manifest = bundle.manifest(); + + let mut decoded = Vec::new(); + for root in &manifest.operation_roots { + for raw in bundle.read_operation_block(root)? { + decoded.push(decode_envelope(&raw)?); + } + } + let envelopes: Vec = { + let refs: Vec<&OperationEnvelope> = decoded.iter().collect(); + canonical_reduction_order(&refs) + .into_iter() + .cloned() + .collect() + }; + + let mut extensions = Vec::with_capacity(manifest.extension_declarations.len()); + for extension in &manifest.extension_declarations { + let mut chunks = Vec::with_capacity(extension.preserved_chunk_roots.len()); + for chunk_ref in &extension.preserved_chunk_roots { + let payload = bundle.read_chunk(chunk_ref)?; + chunks.push(TextChunk { + kind: chunk_ref.kind, + schema_version: chunk_ref.schema_version, + payload, + }); + } + extensions.push(TextExtension { + extension_id: extension.extension_id, + version: extension.version, + required: extension.required, + chunks, + affected_object_kinds: extension.affected_object_kinds.clone(), + edit_barriers: extension.edit_barriers.clone(), + }); + } + + let canonical_base = match &manifest.canonical_base { + Some(base) => { + let root_payload = bundle.read_chunk(&base.root)?; + Some(TextCanonicalBase { + snapshot_id: base.snapshot_id, + covers_causal_frontier: base.covers_causal_frontier.clone(), + reduction_algorithm_version: base.reduction_algorithm_version, + profile_id: base.profile_id, + root_schema_version: base.root.schema_version, + root_payload, + }) + } + None => None, + }; + + let blobs = canonical_blobs(bundle, &decoded)?; + + Ok(TextDocument { + document_id: manifest.document_id, + lineage_id: manifest.lineage_id, + // `Manifest::decode`'s own re-encode check (see `epiphany-bundle`) + // guarantees a bundle's `profile_declarations` are already the + // canonical (sorted, deduplicated) sequence; `canonical_profiles` + // names that guarantee rather than leaning on it silently. + profiles: manifest.canonical_profiles(), + extensions, + canonical_base, + blobs, + envelopes, + }) +} + +// =========================================================================== +// TextDocument -> text. +// =========================================================================== + +/// Projects a whole [`TextDocument`] to its canonical text: the header, then +/// every present section in the normative `projection` sequence (header, +/// document, lineage?, profile*, extension*, canonical-base?, blob*, +/// envelope*), one line per element, each terminated by a single LF +/// (`req:textproj:envelope-per-line`). Section order is normative and is +/// simply the order this function writes in; it introduces no ordering of its +/// own beyond `req:textproj:derived-ordering`'s blob sort. +pub fn project_text_document(document: &TextDocument) -> String { + let mut lines: Vec = Vec::new(); + lines.push(project_header()); + lines.push(project_document(&document.document_id)); + if let Some(lineage_id) = &document.lineage_id { + lines.push(project_lineage(lineage_id)); + } + lines.extend(document.profiles.iter().map(project_profile)); + lines.extend(document.extensions.iter().map(project_extension)); + if let Some(base) = &document.canonical_base { + lines.push(project_canonical_base(base)); + } + lines.extend(ordered_by_projected_form(&document.blobs, project_blob)); + + let mut out = String::new(); + for line in &lines { + out.push_str(&line.render()); + out.push('\n'); + } + for envelope in &document.envelopes { + out.push_str(&project_envelope(envelope)); + out.push('\n'); + } + out +} + +/// Composes both stages: reads `bundle` into a [`TextDocument`], then projects +/// it to its canonical text. +pub fn project_bundle(bundle: &Bundle) -> Result { + Ok(project_text_document(&document_from_bundle(bundle)?)) +} + +#[cfg(test)] +mod tests { + use std::fs; + use std::path::{Path, PathBuf}; + + use epiphany_bundle::{ + encode_block, BlobRef, CompressionAlgorithm, ExtensionDeclaration, ExtensionId, FileUuid, + FrontierBytes, Manifest, MemStore, ProfileRegistryId, ReductionAlgorithmVersion, + SnapshotId, SnapshotRef, StagedChunk, + }; + use epiphany_core::textvalue::read_sexp; + use epiphany_core::{OperationId, RegionId, ReplicaId, WallClockTime}; + use epiphany_determinism::CanonicalEncode; + use epiphany_ops::{ + AuthorId, CausalContext, DeleteRegionOp, HybridLogicalClock, OperationKind, + OperationPayload, OperationStamp, + }; + + use super::*; + + // ----------------------------------------------------------------- + // Pinned against the companion's worked example (Chapter "A Worked + // Example"). These are the strongest checks in this file: literal + // strings copied from the specification itself, not values this test + // module invented. + // ----------------------------------------------------------------- + + #[test] + fn header_matches_the_worked_example() { + assert_eq!(project_header().render(), "(text-projection (0 7 0))"); + } + + #[test] + fn document_id_matches_the_worked_example() { + let id = DocumentId([0x05; 16]); + assert_eq!( + project_document(&id).render(), + "(document #x05050505050505050505050505050505)" + ); + } + + #[test] + fn profile_matches_the_worked_example() { + assert_eq!( + project_profile(&ProfileDeclaration::full()).render(), + "(profile full (0 1 0) (constraints 67108864 (retention 1 () true)))" + ); + } + + #[test] + fn canonical_base_matches_the_worked_example() { + let mut snapshot_id_bytes = [0u8; 16]; + snapshot_id_bytes[0] = 0x1f; + snapshot_id_bytes[1] = 0x8b; + let base = TextCanonicalBase { + snapshot_id: SnapshotId(snapshot_id_bytes), + covers_causal_frontier: FrontierBytes::empty(), + reduction_algorithm_version: ReductionAlgorithmVersion(1), + profile_id: ProfileId::Full, + root_schema_version: SchemaVersion::V0, + root_payload: vec![0x00, 0x00], + }; + assert_eq!( + project_canonical_base(&base).render(), + "(canonical-base #x1f8b0000000000000000000000000000 #x 1 full (schema 0 1) #x0000)" + ); + } + + // ----------------------------------------------------------------- + // profile-id: the closed vocabulary, plus the one non-symbol case. + // ----------------------------------------------------------------- + + #[test] + fn profile_id_projects_every_closed_vocabulary_symbol() { + assert_eq!(project_profile_id(&ProfileId::Full).render(), "full"); + assert_eq!( + project_profile_id(&ProfileId::ReadOnly).render(), + "read-only" + ); + assert_eq!(project_profile_id(&ProfileId::Lite).render(), "lite"); + } + + #[test] + fn profile_id_custom_carries_its_sixteen_byte_registry_id() { + let id = ProfileId::Custom(ProfileRegistryId([0xAB; 16])); + assert_eq!( + project_profile_id(&id).render(), + "(custom #xabababababababababababababababab)" + ); + } + + // ----------------------------------------------------------------- + // chunk-kind: exhaustive over all nine variants. + // ----------------------------------------------------------------- + + #[test] + fn chunk_kind_projects_every_variant_to_its_grammar_symbol() { + let expected = [ + ( + ChunkKind::OperationEnvelopeBlock, + "operation-envelope-block", + ), + (ChunkKind::OperationIndex, "operation-index"), + (ChunkKind::Snapshot, "snapshot"), + (ChunkKind::Blob, "blob"), + (ChunkKind::ExtensionData, "extension-data"), + (ChunkKind::TextProjection, "text-projection"), + (ChunkKind::LayoutCache, "layout-cache"), + (ChunkKind::IntegrityIndex, "integrity-index"), + (ChunkKind::Manifest, "manifest"), + ]; + for (kind, symbol) in expected { + assert_eq!(project_chunk_kind(kind).render(), symbol); + } + } + + #[test] + fn schema_projects_major_then_minor() { + assert_eq!( + project_schema(&SchemaVersion::new(2, 9)).render(), + "(schema 2 9)" + ); + } + + // ----------------------------------------------------------------- + // extension: six fields in the ratified order, opaque byte strings for + // affected-kinds/barriers (the `Vec`-is-not-a-sequence trap). + // ----------------------------------------------------------------- + + #[test] + fn extension_projects_six_fields_in_declaration_order() { + let extension = TextExtension { + extension_id: ExtensionId([9; 16]), + version: SemVer::new(1, 2, 3), + required: true, + chunks: vec![TextChunk { + kind: ChunkKind::ExtensionData, + schema_version: SchemaVersion::V0, + payload: vec![0xAA], + }], + affected_object_kinds: vec![0x01, 0x02], + edit_barriers: vec![0x03], + }; + assert_eq!( + project_extension(&extension).render(), + "(extension #x09090909090909090909090909090909 (1 2 3) true \ + ((chunk extension-data (schema 0 1) #xaa)) #x0102 #x03)" + ); + } + + #[test] + fn extension_affected_kinds_and_barriers_are_byte_strings_not_sequences() { + let extension = TextExtension { + extension_id: ExtensionId([0; 16]), + version: SemVer::new(0, 0, 0), + required: false, + chunks: Vec::new(), + affected_object_kinds: vec![1, 2, 3], + edit_barriers: vec![4, 5], + }; + let Sexp::List(fields) = project_extension(&extension) else { + panic!("an extension projection is a list"); + }; + // Positions: [0]=`extension`, [1]=id, [2]=version, [3]=required, + // [4]=chunks, [5]=affected-kinds, [6]=barriers. + assert_eq!(fields[5], Sexp::Bytes(vec![1, 2, 3])); + assert_eq!(fields[6], Sexp::Bytes(vec![4, 5])); + // Never the `Vec`-is-a-sequence spelling (a list of integers). + assert_ne!( + fields[5], + Sexp::List(vec![Sexp::int(1), Sexp::int(2), Sexp::int(3)]) + ); + } + + // ----------------------------------------------------------------- + // blob: line-level project, tested against synthetic data (per the + // contract: the emit side must be ready even though no real bundle can + // populate it today). + // ----------------------------------------------------------------- + + #[test] + fn blob_projects_media_type_declared_max_and_payload() { + let blob = TextBlob { + media_type: "audio/wav".to_owned(), + declared_max_uncompressed_length: Some(1 << 20), + payload: vec![0xDE, 0xAD], + }; + assert_eq!( + project_blob(&blob).render(), + "(blob \"audio/wav\" (some 1048576) #xdead)" + ); + } + + #[test] + fn blob_with_no_declared_maximum_projects_an_absent_option() { + let blob = TextBlob { + media_type: "application/octet-stream".to_owned(), + declared_max_uncompressed_length: None, + payload: Vec::new(), + }; + assert_eq!( + project_blob(&blob).render(), + "(blob \"application/octet-stream\" () #x)" + ); + } + + /// The `(blob ...)` production round-trips through the s-expression + /// reader (the lexical layer parse.rs will build its semantic parser on). + /// This is not a substitute for parse.rs's own `parse_blob` and the + /// parse-side rejection the contract requires of it; it proves the emit + /// side is well-formed and ready. + #[test] + fn blob_projection_reads_back_through_the_sexp_reader() { + let blob = TextBlob { + media_type: "text/plain".to_owned(), + declared_max_uncompressed_length: Some(4), + payload: vec![1, 2, 3, 4], + }; + let rendered = project_blob(&blob).render(); + let read_back = read_sexp(&rendered).expect("projected blob line is valid s-expression"); + assert_eq!(read_back, project_blob(&blob), "reading back is idempotent"); + } + + // ----------------------------------------------------------------- + // `req:textproj:derived-ordering`: sort AND de-duplicate by rendered + // form, exercised against out-of-order, duplicated, in-memory data -- + // never against a fixture that was already sorted. + // ----------------------------------------------------------------- + + #[test] + fn derived_ordering_sorts_and_dedups_blob_lines() { + let a = TextBlob { + media_type: "a/a".to_owned(), + declared_max_uncompressed_length: None, + payload: vec![1], + }; + let b = TextBlob { + media_type: "b/b".to_owned(), + declared_max_uncompressed_length: None, + payload: vec![2], + }; + let a_duplicate = TextBlob { + media_type: "a/a".to_owned(), + declared_max_uncompressed_length: None, + payload: vec![1], + }; + // Deliberately out of order (b before a) and with a duplicate of a. + let blobs = vec![b, a_duplicate, a]; + let ordered = ordered_by_projected_form(&blobs, project_blob); + let rendered: Vec = ordered.iter().map(Sexp::render).collect(); + assert_eq!( + rendered, + vec![ + "(blob \"a/a\" () #x01)".to_owned(), + "(blob \"b/b\" () #x02)".to_owned(), + ], + "duplicate collapsed to one line, and the surviving two lines ordered ascending \ + by rendered form" + ); + } + + #[test] + fn derived_ordering_sorts_and_dedups_an_extensions_chunk_lines() { + let x = TextChunk { + kind: ChunkKind::ExtensionData, + schema_version: SchemaVersion::V0, + payload: vec![0xAA], + }; + let y = TextChunk { + kind: ChunkKind::LayoutCache, + schema_version: SchemaVersion::V0, + payload: vec![0xBB], + }; + let x_duplicate = TextChunk { + kind: ChunkKind::ExtensionData, + schema_version: SchemaVersion::V0, + payload: vec![0xAA], + }; + let extension = TextExtension { + extension_id: ExtensionId([1; 16]), + version: SemVer::new(1, 0, 0), + required: false, + // Deliberately out of order (y before x) and with a duplicate of x. + chunks: vec![y, x_duplicate, x], + affected_object_kinds: Vec::new(), + edit_barriers: Vec::new(), + }; + let Sexp::List(fields) = project_extension(&extension) else { + panic!("an extension projection is a list"); + }; + let Sexp::List(chunks) = &fields[4] else { + panic!("the chunk field is a list"); + }; + assert_eq!( + chunks.len(), + 2, + "the duplicate chunk collapsed to a single line" + ); + assert_eq!( + chunks[0].render(), + "(chunk extension-data (schema 0 1) #xaa)" + ); + assert_eq!(chunks[1].render(), "(chunk layout-cache (schema 0 1) #xbb)"); + } + + // ----------------------------------------------------------------- + // The blob trap: a bundle whose manifest carries a real, non-empty + // `blob_roots` must still project zero `(blob ...)` lines, because no + // canonical operation or reduced state can reach a `BlobId` today. + // ----------------------------------------------------------------- + + /// Builds a bundle exercising every optional section at once: a lineage, + /// one extension with one preserved chunk, a canonical base, one + /// (deliberately unreachable) blob root, and two operation-envelope + /// blocks staged so their envelopes are **not** already in canonical + /// reduction order (physical time descending), across **two** separate + /// blocks so `document_from_bundle` is proven to gather across all of + /// `operation_roots`, not just the first. + fn build_sample_bundle() -> Bundle { + let env_late = sample_envelope(1, 200); // higher physical time + let env_early = sample_envelope(2, 100); // lower physical time + + // Block 0 holds the later envelope, block 1 the earlier one: storage + // order disagrees with canonical reduction order on purpose. + let op_block_0 = + StagedChunk::operation_block(encode_block(&[env_late.to_canonical_bytes()])); + let op_block_1 = + StagedChunk::operation_block(encode_block(&[env_early.to_canonical_bytes()])); + + let extension_chunk = StagedChunk { + kind: ChunkKind::ExtensionData, + schema_version: SchemaVersion::V0, + payload: b"extension-chunk-payload".to_vec(), + }; + + let base_chunk = StagedChunk { + kind: ChunkKind::Snapshot, + schema_version: SchemaVersion::V0, + payload: b"canonical-base-payload".to_vec(), + }; + + let blob_chunk = StagedChunk { + kind: ChunkKind::Blob, + schema_version: SchemaVersion::V0, + payload: b"unreachable-blob-payload".to_vec(), + }; + + let mut initial = Manifest::empty(DocumentId([3; 16])); + initial.lineage_id = Some(LineageId([4; 16])); + let mut bundle = Bundle::create(MemStore::new(), FileUuid([1; 16]), initial) + .expect("a freshly created bundle with no canonical roots is valid"); + + bundle + .commit( + &[ + op_block_0, + op_block_1, + extension_chunk, + base_chunk, + blob_chunk, + ], + |ctx| { + let mut manifest = ctx.previous_manifest.clone(); + manifest.operation_roots = vec![ctx.new_chunks[0], ctx.new_chunks[1]]; + manifest.extension_declarations = vec![ExtensionDeclaration { + extension_id: ExtensionId([9; 16]), + version: SemVer::new(1, 0, 0), + required: false, + preserved_chunk_roots: vec![ctx.new_chunks[2]], + affected_object_kinds: vec![0xAA], + edit_barriers: vec![0xBB, 0xCC], + }]; + manifest.canonical_base = Some(SnapshotRef { + snapshot_id: SnapshotId([7; 16]), + covers_causal_frontier: FrontierBytes::from_bytes(vec![1, 2, 3]), + reduction_algorithm_version: ReductionAlgorithmVersion(1), + profile_id: ProfileId::Full, + root: ctx.new_chunks[3], + hash: ctx.new_chunks[3].hash, + }); + manifest.blob_roots = vec![BlobRef { + blob_id: BlobId(ctx.new_chunks[4].hash), + media_type: "application/octet-stream".to_owned(), + offset: ctx.new_chunks[4].offset, + compressed_length: ctx.new_chunks[4].compressed_length, + uncompressed_length: ctx.new_chunks[4].uncompressed_length, + compression: CompressionAlgorithm::None, + hash: ctx.new_chunks[4].hash, + declared_max_uncompressed_length: None, + }]; + manifest + }, + ) + .expect("commit of a well-formed, self-consistent manifest succeeds"); + + // Sanity: the trap this test exists to guard against is only live if + // the manifest really does carry a non-empty, non-canonical blob root. + assert_eq!(bundle.manifest().blob_roots.len(), 1); + bundle + } + + fn sample_envelope(counter: u64, physical_time: i64) -> OperationEnvelope { + let id = OperationId::new(ReplicaId(1), counter); + OperationEnvelope { + id, + author: AuthorId(0xAB), + stamp: OperationStamp::new( + HybridLogicalClock::new(WallClockTime(physical_time), 0), + id, + ), + causal_context: CausalContext::new(), + transaction: None, + payload: OperationPayload::Primitive(OperationKind::DeleteRegion(DeleteRegionOp { + region: RegionId::new(ReplicaId(1), counter), + })), + } + } + + #[test] + fn document_from_bundle_reads_every_section_and_orders_envelopes_canonically() { + let bundle = build_sample_bundle(); + let document = document_from_bundle(&bundle).expect("bundle reads cleanly"); + + assert_eq!(document.document_id, DocumentId([3; 16])); + assert_eq!(document.lineage_id, Some(LineageId([4; 16]))); + assert_eq!(document.profiles, vec![ProfileDeclaration::full()]); + + assert_eq!(document.extensions.len(), 1); + let extension = &document.extensions[0]; + assert_eq!(extension.extension_id, ExtensionId([9; 16])); + assert_eq!(extension.chunks.len(), 1); + assert_eq!(extension.chunks[0].kind, ChunkKind::ExtensionData); + assert_eq!(extension.chunks[0].payload, b"extension-chunk-payload"); + + let base = document.canonical_base.as_ref().expect("base was staged"); + assert_eq!(base.snapshot_id, SnapshotId([7; 16])); + assert_eq!(base.root_payload, b"canonical-base-payload"); + + // The trap: a non-empty `blob_roots` still yields zero canonical + // blobs, because nothing can reach one yet. + assert!( + document.blobs.is_empty(), + "no blob is canonically reachable today, regardless of manifest.blob_roots" + ); + + // Canonical reduction order is by ascending physical time; the raw + // blocks stored the higher-physical-time envelope first. + assert_eq!(document.envelopes.len(), 2); + assert_eq!(document.envelopes[0].id, OperationId::new(ReplicaId(1), 2)); + assert_eq!(document.envelopes[1].id, OperationId::new(ReplicaId(1), 1)); + } + + #[test] + fn projected_text_never_emits_a_blob_line_and_orders_sections_correctly() { + let bundle = build_sample_bundle(); + let document = document_from_bundle(&bundle).expect("bundle reads cleanly"); + let text = project_text_document(&document); + + assert!( + !text.lines().any(|line| line.starts_with("(blob ")), + "projected text must never carry a blob line at this companion version:\n{text}" + ); + + // Section order is normative (`projection ::= header document lineage? + // profile* extension* canonical-base? blob* envelope*`): assert the + // literal sequence of line-heads, not just a count, so a section + // written out of order fails this check rather than only a count. + let lines: Vec<&str> = text.lines().collect(); + let heads: Vec<&str> = lines + .iter() + .map(|line| line.split(' ').next().expect("every line has a head token")) + .collect(); + assert_eq!( + heads, + vec![ + "(text-projection", + "(document", + "(lineage", + "(profile", + "(extension", + "(canonical-base", + "(envelope", + "(envelope", + ], + "section order is normative; got:\n{text}" + ); + } + + #[test] + fn project_bundle_composes_both_stages() { + let bundle = build_sample_bundle(); + let via_two_stages = + project_text_document(&document_from_bundle(&bundle).expect("bundle reads cleanly")); + let via_one_call = project_bundle(&bundle).expect("bundle reads cleanly"); + assert_eq!(via_one_call, via_two_stages); + } + + #[test] + fn a_corrupt_operation_envelope_is_a_typed_error_not_a_panic() { + let mut initial = Manifest::empty(DocumentId([5; 16])); + initial.lineage_id = None; + let mut bundle = Bundle::create(MemStore::new(), FileUuid([2; 16]), initial).unwrap(); + let garbage_block = StagedChunk::operation_block(encode_block(&[vec![0xFF; 4]])); + bundle + .commit(&[garbage_block], |ctx| { + let mut manifest = ctx.previous_manifest.clone(); + manifest.operation_roots = vec![ctx.new_chunks[0]]; + manifest + }) + .expect("the block's framing is well-formed even though its one envelope is not"); + + match document_from_bundle(&bundle) { + Err(ProjectError::Envelope(_)) => {} + other => panic!("expected a decode error for garbage envelope bytes, got {other:?}"), + } + } + + #[test] + fn a_corrupt_chunk_hash_is_a_bundle_error() { + let bundle = build_sample_bundle(); + let extension_root = bundle.manifest().extension_declarations[0].preserved_chunk_roots[0]; + let mut image = bundle.image().to_vec(); + // Flip one byte inside the extension chunk's stored payload region. + let corrupt_at = extension_root.offset as usize; + image[corrupt_at] ^= 0xFF; + + let corrupted = Bundle::open(MemStore::from_bytes(image)) + .expect("corrupting a non-canonical chunk's payload does not stop the bundle opening"); + match document_from_bundle(&corrupted) { + Err(ProjectError::Bundle(_)) => {} + other => panic!("expected a bundle error for a hash-mismatched chunk, got {other:?}"), + } + } + + // ----------------------------------------------------------------- + // Suite reach: count, don't just claim, coverage of an extension, a + // canonical base, and a multi-envelope document. + // ----------------------------------------------------------------- + + #[test] + fn test_suite_reach_covers_extension_base_and_multi_envelope_documents() { + let rich = document_from_bundle(&build_sample_bundle()).expect("bundle reads cleanly"); + let minimal = TextDocument { + document_id: DocumentId([6; 16]), + lineage_id: None, + profiles: vec![ProfileDeclaration::full()], + extensions: Vec::new(), + canonical_base: None, + blobs: Vec::new(), + envelopes: vec![sample_envelope(1, 1)], + }; + let documents = [rich, minimal]; + + let with_extension = documents + .iter() + .filter(|d| !d.extensions.is_empty()) + .count(); + let with_canonical_base = documents + .iter() + .filter(|d| d.canonical_base.is_some()) + .count(); + let with_multi_envelope = documents.iter().filter(|d| d.envelopes.len() > 1).count(); + + assert_eq!( + with_extension, 1, + "expected exactly one exercised document with an extension" + ); + assert_eq!( + with_canonical_base, 1, + "expected exactly one exercised document with a canonical base" + ); + assert_eq!( + with_multi_envelope, 1, + "expected exactly one exercised document with more than one envelope" + ); + } + + // ----------------------------------------------------------------- + // The trip-wire: if `BlobId` ever appears in `epiphany-core` or + // `epiphany-ops`, `canonically_reachable_blob_ids` above is no longer + // provably empty, and both the emit side (that function) and the + // parse-side `(blob ...)` rejection (parse.rs) need real + // implementations. + // ----------------------------------------------------------------- + + fn rs_files_under(dir: &Path, out: &mut Vec) { + let entries = fs::read_dir(dir) + .unwrap_or_else(|e| panic!("failed to read directory {}: {e}", dir.display())); + for entry in entries { + let path = entry.expect("directory entry is readable").path(); + if path.is_dir() { + rs_files_under(&path, out); + } else if path.extension().is_some_and(|ext| ext == "rs") { + out.push(path); + } + } + } + + #[test] + fn blobid_is_absent_from_core_and_ops_source() { + let dirs = [ + concat!(env!("CARGO_MANIFEST_DIR"), "/../epiphany-core/src"), + concat!(env!("CARGO_MANIFEST_DIR"), "/../epiphany-ops/src"), + ]; + for dir in dirs { + let mut files = Vec::new(); + rs_files_under(Path::new(dir), &mut files); + assert!( + !files.is_empty(), + "expected to find .rs sources under {dir}" + ); + for path in files { + let source = fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display())); + assert!( + !source.contains("BlobId"), + "found `BlobId` in {}: `canonically_reachable_blob_ids` \ + (crates/epiphany-textproj/src/project.rs) could provably return only the \ + empty set because neither crate could name a BlobId -- that is no longer \ + true, so both that predicate's emit side and parse.rs's `(blob ...)` \ + rejection now need their real implementations", + path.display() + ); + } + } + } +} diff --git a/crates/epiphany-textproj/src/serialize.rs b/crates/epiphany-textproj/src/serialize.rs new file mode 100644 index 0000000..f34b391 --- /dev/null +++ b/crates/epiphany-textproj/src/serialize.rs @@ -0,0 +1,561 @@ +//! Serialization of parsed Text Projection documents into canonical bundles. +//! +//! [`serialize_document`] stages the payloads a [`TextDocument`] carries inline +//! — the canonical base's root chunk, every extension's preserved chunks, and +//! one operation-envelope block — and lets [`Bundle::create`] / +//! [`Bundle::commit`] do everything `req:textproj:derive-or-carry` says a +//! serializer must not: assign offsets, content-address every chunk, and +//! de-duplicate identical content. **Nothing here computes a `ChunkId`, a +//! `ContentHash`, or an offset** — every physical field written into the +//! manifest comes straight from the [`ChunkRef`](epiphany_bundle::ChunkRef)s +//! `commit` hands back. +//! +//! # Block splitting is a free physical choice +//! +//! `req:textproj:roundtrip` only requires the *bundle's* physical layout to +//! round-trip in the second, byte-checkable equation (`project(serialize(parse(T))) +//! == T`, quantified over texts, not bundles); how a serializer packs envelopes +//! into blocks is unconstrained. This module always emits **exactly one** +//! operation-envelope block, containing every envelope the document carries (in +//! the order the document carries them) — the simplest possible choice, and +//! sufficient because block boundaries are storage artifacts, never semantic +//! structure (Chapter 8: *"the set of envelopes is the union of all envelopes +//! across all referenced blocks"*). +//! +//! # No accelerator is (re)written +//! +//! A [`TextDocument`] carries no `operation_index_root`, `acceleration_snapshots`, +//! `text_projection_root`, `integrity_root`, or `operation_block_summaries` — +//! they are non-canonical and the companion text does not carry them +//! (`req:textproj:derive-or-carry`). The manifest this module builds leaves every +//! one of those fields at its empty/`None` default. A bundle serialized from a +//! `TextDocument` therefore comes back from this module *without* any +//! accelerator a previous generation might have had. That reads as data loss; it +//! is not — none of those fields contributes to canonical document semantics, +//! and a consumer that wants them back rebuilds them the same way any bundle +//! writer does (e.g. `epiphany_bundle::fuzz` and the testkit's operation-index +//! harness both scan-and-rebuild rather than trust a carried-forward index). +//! +//! # No blob is staged +//! +//! [`TextDocument::blobs`] is never staged into `manifest.blob_roots` here. Today +//! that vector is always empty: no operation payload or Chapter-5 value in +//! `epiphany-core`/`epiphany-ops` carries a `BlobId`, so no blob is reachable from +//! canonical state (`req:textproj:canonical-blobs`), and this companion's own +//! parser rejects every `(blob ...)` line rather than populate the vector (see +//! `parse`). [`serialize_document`] still checks: a `TextDocument` assembled +//! directly against the type (bypassing `parse`) with a populated `blobs` vector +//! is refused with [`SerializeError::NonEmptyBlobs`] rather than silently +//! dropped — wiring a blob into the manifest here would require a canonical +//! `declared_max_uncompressed_length`/compression policy this companion version +//! does not specify, and silently discarding the caller's bytes is worse. + +use std::fmt; + +use epiphany_bundle::{ + encode_block, BlockStore, Bundle, BundleError, ChunkKind, CommitContext, ExtensionDeclaration, + FileUuid, Manifest, SchemaVersion, SnapshotRef, StagedChunk, +}; +use epiphany_determinism::CanonicalEncode; +use epiphany_ops::OperationEnvelope; + +use crate::TextDocument; + +/// An error [`serialize_document`] cannot recover from. +#[derive(Debug)] +pub enum SerializeError { + /// The document's `blobs` vector is non-empty. See the module documentation: + /// no blob is canonical today, so a document built by this companion's + /// `parse` never carries one, and staging one here would either wire a + /// non-canonical root into the manifest or silently drop the caller's + /// payload bytes. Neither is acceptable, so serialization refuses instead. + NonEmptyBlobs, + /// The bundle itself rejected the manifest or a staged chunk — e.g. the + /// document declares no profile this implementation understands, or a + /// staged root fails `Bundle::commit`'s structural validation. + Bundle(BundleError), +} + +impl fmt::Display for SerializeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + SerializeError::NonEmptyBlobs => f.write_str( + "the document carries a populated blobs vector, but no blob can be canonical today", + ), + SerializeError::Bundle(error) => { + write!(f, "the bundle rejected the serialized document: {error}") + } + } + } +} + +impl std::error::Error for SerializeError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + SerializeError::Bundle(error) => Some(error), + SerializeError::NonEmptyBlobs => None, + } + } +} + +impl From for SerializeError { + fn from(error: BundleError) -> Self { + SerializeError::Bundle(error) + } +} + +/// Serializes a [`TextDocument`] into a freshly created bundle over `store`. +/// +/// Two-phase, because [`Bundle::create`] requires a manifest with no canonical +/// roots or blobs (there is nothing to reference before any chunk is written): +/// this creates an empty generation-0 bundle carrying only the document's +/// identity and profile declarations, then stages every payload the document +/// carries inline and commits once, building the real manifest from the +/// [`ChunkRef`](epiphany_bundle::ChunkRef)s that commit assigns. +/// +/// Returns [`SerializeError::NonEmptyBlobs`] if `document.blobs` is non-empty +/// (see the module documentation), or [`SerializeError::Bundle`] if the bundle +/// itself refuses the manifest or a staged root. +pub fn serialize_document( + document: &TextDocument, + store: S, + file_uuid: FileUuid, +) -> Result, SerializeError> { + if !document.blobs.is_empty() { + return Err(SerializeError::NonEmptyBlobs); + } + + let mut bundle = Bundle::create(store, file_uuid, empty_manifest(document))?; + + let mut staged = Vec::new(); + if let Some(base) = &document.canonical_base { + // The canonical base's root chunk: kind and schema version are carried + // verbatim from the document (they are not derivable from anything), + // and its payload is staged exactly as the document carries it. `commit` + // computes the chunk's id, hash, and offset. + staged.push(StagedChunk { + kind: ChunkKind::Snapshot, + schema_version: base.root_schema_version, + payload: base.root_payload.clone(), + }); + } + for extension in &document.extensions { + for chunk in &extension.chunks { + staged.push(StagedChunk { + kind: chunk.kind, + schema_version: chunk.schema_version, + payload: chunk.payload.clone(), + }); + } + } + staged.push(stage_operation_envelope_block(document)); + + bundle.commit(&staged, |ctx| build_manifest(document, ctx))?; + Ok(bundle) +} + +/// The manifest [`Bundle::create`] is given: the document's identity and +/// declared profiles, and nothing else. `create` itself rejects a manifest +/// carrying canonical roots or blobs, so every root is added by the subsequent +/// commit (see [`build_manifest`]). If `document.profiles` is empty or +/// otherwise unemittable, `Bundle::create` reports that itself — this function +/// does not invent a fallback profile the document did not declare. +fn empty_manifest(document: &TextDocument) -> Manifest { + let mut manifest = Manifest::empty(document.document_id); + manifest.lineage_id = document.lineage_id; + manifest.profile_declarations = document.profiles.clone(); + manifest +} + +/// Encodes every envelope the document carries, in the order it carries them, +/// into a single operation-envelope block payload (see the module +/// documentation on why one block is the right choice here). The block's +/// schema version is the max over its envelopes' `schema_major` — never a fixed +/// baseline — so a block that carries a higher-major payload (e.g. a v1 +/// `CreateRegion` or a v2 cross-cutting value) is never mis-stamped major 0, +/// mirroring `StagedChunk::operation_block_versioned`'s own contract. +fn stage_operation_envelope_block(document: &TextDocument) -> StagedChunk { + let payloads: Vec> = document + .envelopes + .iter() + .map(CanonicalEncode::to_canonical_bytes) + .collect(); + let major = document + .envelopes + .iter() + .map(OperationEnvelope::schema_major) + .max() + .unwrap_or(0); + StagedChunk::operation_block_versioned(encode_block(&payloads), SchemaVersion::for_major(major)) +} + +/// Builds the committed manifest from the previous (empty) manifest and the +/// [`ChunkRef`](epiphany_bundle::ChunkRef)s [`Bundle::commit`] assigned to the +/// chunks [`serialize_document`] staged, in the same order it staged them: the +/// canonical base's root (if any), then each extension's preserved chunks in +/// turn, then the single operation-envelope block. Every physical field +/// (`ChunkId`, `ContentHash`, offset) comes from `ctx.new_chunks` — nothing here +/// recomputes one (`req:textproj:derive-or-carry`). `operation_index_root`, +/// `acceleration_snapshots`, `text_projection_root`, `integrity_root`, +/// `operation_block_summaries`, and `blob_roots` are left untouched at the +/// previous manifest's empty defaults; see the module documentation for why +/// that is correct rather than lossy. +fn build_manifest(document: &TextDocument, ctx: &CommitContext) -> Manifest { + let mut manifest = ctx.previous_manifest.clone(); + let mut cursor = 0usize; + + if let Some(base) = &document.canonical_base { + let root = ctx.new_chunks[cursor]; + cursor += 1; + manifest.canonical_base = Some(SnapshotRef { + snapshot_id: base.snapshot_id, + covers_causal_frontier: base.covers_causal_frontier.clone(), + reduction_algorithm_version: base.reduction_algorithm_version, + profile_id: base.profile_id, + root, + hash: root.hash, + }); + } + + let mut extension_declarations = Vec::with_capacity(document.extensions.len()); + for extension in &document.extensions { + let mut preserved_chunk_roots = Vec::with_capacity(extension.chunks.len()); + for _ in &extension.chunks { + preserved_chunk_roots.push(ctx.new_chunks[cursor]); + cursor += 1; + } + extension_declarations.push(ExtensionDeclaration { + extension_id: extension.extension_id, + version: extension.version, + required: extension.required, + preserved_chunk_roots, + affected_object_kinds: extension.affected_object_kinds.clone(), + edit_barriers: extension.edit_barriers.clone(), + }); + } + manifest.extension_declarations = extension_declarations; + + manifest.operation_roots = vec![ctx.new_chunks[cursor]]; + cursor += 1; + debug_assert_eq!( + cursor, + ctx.new_chunks.len(), + "every chunk staged by serialize_document must be wired into the manifest exactly once" + ); + manifest +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{TextCanonicalBase, TextChunk, TextExtension}; + use epiphany_bundle::{ + DocumentId, ExtensionId, FrontierBytes, LineageId, MemStore, ProfileConstraints, + ProfileDeclaration, ProfileId, ReductionAlgorithmVersion, SemVer, SnapshotId, + }; + use epiphany_determinism::fuzz::SplitMix64; + use epiphany_ops::{decode_envelope, fuzz::gen_envelope_set}; + + /// A real, varied envelope set (every operation kind the generator reaches), + /// so the schema-major derivation and the envelope round trip both have + /// something to bite on. + fn envelopes(seed: u64, n: usize) -> Vec { + let mut rng = SplitMix64::new(seed); + gen_envelope_set(&mut rng, n) + } + + /// A profile declaration distinguishable from `ProfileDeclaration::full()`'s + /// default version, so a test that forgets to carry `document.profiles` + /// through cannot pass by accident (`full()` would still be a `Full` + /// profile, just at a different version). + fn base_profile() -> ProfileDeclaration { + ProfileDeclaration { + profile_id: ProfileId::Full, + version: SemVer::new(0, 2, 0), + constraints: ProfileConstraints::DEFAULT_FULL, + } + } + + fn minimal_document(seed: u64) -> TextDocument { + TextDocument { + document_id: DocumentId([seed as u8; 16]), + lineage_id: None, + profiles: vec![base_profile()], + extensions: Vec::new(), + canonical_base: None, + blobs: Vec::new(), + envelopes: envelopes(seed, 5), + } + } + + fn document_with_extension(seed: u64) -> TextDocument { + let mut document = minimal_document(seed); + document.extensions.push(TextExtension { + extension_id: ExtensionId([7; 16]), + version: SemVer::new(1, 0, 0), + required: false, + chunks: vec![ + TextChunk { + kind: ChunkKind::ExtensionData, + schema_version: SchemaVersion::V0, + payload: b"chunk-a".to_vec(), + }, + TextChunk { + kind: ChunkKind::ExtensionData, + schema_version: SchemaVersion::V0, + payload: b"chunk-b".to_vec(), + }, + ], + affected_object_kinds: vec![0xAA], + edit_barriers: vec![0xBB, 0xCC], + }); + document + } + + fn document_with_canonical_base(seed: u64) -> TextDocument { + let mut document = minimal_document(seed); + document.canonical_base = Some(TextCanonicalBase { + snapshot_id: SnapshotId([9; 16]), + covers_causal_frontier: FrontierBytes(vec![1, 2, 3]), + reduction_algorithm_version: ReductionAlgorithmVersion(1), + profile_id: ProfileId::Full, + root_schema_version: SchemaVersion::V0, + root_payload: b"snapshot-root".to_vec(), + }); + document + } + + /// Carries an extension, a canonical base, and many envelopes at once — the + /// document every round-trip law needs to be checked against together. + fn rich_document(seed: u64) -> TextDocument { + let mut document = document_with_extension(seed); + document.canonical_base = document_with_canonical_base(seed).canonical_base; + document.envelopes = envelopes(seed, 60); + document + } + + fn serialize_and_reopen(document: &TextDocument) -> Bundle { + let bundle = serialize_document(document, MemStore::new(), FileUuid([1; 16])) + .expect("a well-formed document serializes"); + let image = bundle.into_store().into_bytes(); + Bundle::open(MemStore::from_bytes(image)).expect("the serialized bundle reopens") + } + + #[test] + fn document_identity_round_trips() { + let mut document = minimal_document(1); + document.lineage_id = Some(LineageId([2; 16])); + let reopened = serialize_and_reopen(&document); + assert_eq!(reopened.manifest().document_id, document.document_id); + assert_eq!(reopened.manifest().lineage_id, document.lineage_id); + } + + #[test] + fn profile_declarations_round_trip() { + let document = minimal_document(2); + let reopened = serialize_and_reopen(&document); + assert_eq!(reopened.manifest().profile_declarations, document.profiles); + } + + #[test] + fn missing_profile_declaration_surfaces_the_bundles_own_rejection() { + // serialize_document must not invent a fallback profile the document + // did not declare: an empty `profiles` propagates Bundle::create's own + // "no declared profile" rejection. + let mut document = minimal_document(3); + document.profiles.clear(); + let result = serialize_document(&document, MemStore::new(), FileUuid([1; 16])); + assert!(matches!(result, Err(SerializeError::Bundle(_)))); + } + + #[test] + fn canonical_base_round_trips_snapshot_id_and_payload() { + let document = document_with_canonical_base(4); + let reopened = serialize_and_reopen(&document); + let expected = document + .canonical_base + .as_ref() + .expect("fixture carries a canonical base"); + let base = reopened + .manifest() + .canonical_base + .as_ref() + .expect("canonical base present after reopen"); + assert_eq!(base.snapshot_id, expected.snapshot_id); + assert_eq!(base.covers_causal_frontier, expected.covers_causal_frontier); + assert_eq!( + base.reduction_algorithm_version, + expected.reduction_algorithm_version + ); + assert_eq!(base.profile_id, expected.profile_id); + assert_eq!(base.root.kind, ChunkKind::Snapshot); + assert_eq!(base.root.schema_version, expected.root_schema_version); + let payload = reopened + .read_chunk(&base.root) + .expect("root chunk reads and verifies"); + assert_eq!(payload, expected.root_payload); + } + + #[test] + fn extension_round_trips_fields_and_chunk_payloads() { + let document = document_with_extension(5); + let reopened = serialize_and_reopen(&document); + let expected = &document.extensions[0]; + let declaration = reopened + .manifest() + .extension_declarations + .iter() + .find(|d| d.extension_id == expected.extension_id) + .expect("extension declaration present after reopen"); + assert_eq!(declaration.version, expected.version); + assert_eq!(declaration.required, expected.required); + assert_eq!( + declaration.affected_object_kinds, + expected.affected_object_kinds + ); + assert_eq!(declaration.edit_barriers, expected.edit_barriers); + assert_eq!( + declaration.preserved_chunk_roots.len(), + expected.chunks.len() + ); + // The manifest's own canonical encoding sorts preserved_chunk_roots by + // ChunkRef (kind, hash, offset), not by the document's chunk order, so + // compare payload sets rather than assume position survives. + let mut payloads: Vec> = declaration + .preserved_chunk_roots + .iter() + .map(|root| reopened.read_chunk(root).expect("preserved chunk reads")) + .collect(); + let mut expected_payloads: Vec> = + expected.chunks.iter().map(|c| c.payload.clone()).collect(); + payloads.sort(); + expected_payloads.sort(); + assert_eq!( + payloads, expected_payloads, + "preserved chunk payloads survive, order aside" + ); + } + + #[test] + fn envelopes_round_trip_through_a_single_operation_block() { + let document = minimal_document(6); + assert!(document.envelopes.len() > 1, "fixture reach check"); + let reopened = serialize_and_reopen(&document); + assert_eq!( + reopened.manifest().operation_roots.len(), + 1, + "one operation-envelope block, by design" + ); + let root = reopened.manifest().operation_roots[0]; + let payloads = reopened + .read_operation_block(&root) + .expect("operation block reads"); + let recovered: Vec = payloads + .iter() + .map(|bytes| decode_envelope(bytes).expect("canonical envelope decodes")) + .collect(); + assert_eq!(recovered, document.envelopes); + } + + #[test] + fn operation_block_schema_major_is_derived_not_hardcoded() { + let document = rich_document(7); + let expected_major = document + .envelopes + .iter() + .map(OperationEnvelope::schema_major) + .max() + .unwrap_or(0); + assert!( + expected_major > 0, + "fixture must include a schema-major-bearing operation to exercise the derivation" + ); + let reopened = serialize_and_reopen(&document); + let root = reopened.manifest().operation_roots[0]; + assert_eq!( + root.schema_version, + SchemaVersion::for_major(expected_major) + ); + } + + #[test] + fn a_document_with_no_envelopes_still_stages_one_empty_operation_block() { + let mut document = minimal_document(8); + document.envelopes.clear(); + let reopened = serialize_and_reopen(&document); + assert_eq!(reopened.manifest().operation_roots.len(), 1); + let root = reopened.manifest().operation_roots[0]; + assert!(reopened + .read_operation_block(&root) + .expect("empty operation block still reads") + .is_empty()); + } + + #[test] + fn nonempty_blobs_are_rejected_rather_than_dropped() { + let mut document = minimal_document(9); + document.blobs.push(crate::TextBlob { + media_type: "audio/wav".to_string(), + declared_max_uncompressed_length: None, + payload: b"nope".to_vec(), + }); + let result = serialize_document(&document, MemStore::new(), FileUuid([1; 16])); + assert!(matches!(result, Err(SerializeError::NonEmptyBlobs))); + } + + #[test] + fn a_rich_document_round_trips_every_root_at_once() { + let document = rich_document(10); + let reopened = serialize_and_reopen(&document); + assert_eq!(reopened.manifest().document_id, document.document_id); + assert!(reopened.manifest().canonical_base.is_some()); + assert_eq!(reopened.manifest().extension_declarations.len(), 1); + assert_eq!(reopened.manifest().operation_roots.len(), 1); + reopened + .verify_canonical_chunks() + .expect("every canonical chunk this module wrote is intact"); + assert!(!reopened.is_read_only()); + assert!(reopened.anomalies().is_empty()); + } + + /// `req:textproj` verification discipline item 3: a round-trip suite that + /// never exercises an extension, a canonical base, or a multi-envelope + /// document proves far less than its green tick suggests. Count what this + /// suite actually covers. + #[test] + fn the_suite_exercises_an_extension_a_canonical_base_and_multiple_envelopes() { + let documents = vec![ + minimal_document(20), + document_with_extension(21), + document_with_canonical_base(22), + rich_document(23), + ]; + let with_extension = documents + .iter() + .filter(|d| !d.extensions.is_empty()) + .count(); + let with_canonical_base = documents + .iter() + .filter(|d| d.canonical_base.is_some()) + .count(); + let with_multiple_envelopes = documents.iter().filter(|d| d.envelopes.len() > 1).count(); + + assert!( + with_extension >= 1, + "reach: no document carries an extension" + ); + assert!( + with_canonical_base >= 1, + "reach: no document carries a canonical base" + ); + assert!( + with_multiple_envelopes >= 1, + "reach: no document carries more than one envelope" + ); + + for document in &documents { + let reopened = serialize_and_reopen(document); + assert_eq!(reopened.manifest().document_id, document.document_id); + } + } +} diff --git a/crates/epiphany-textproj/tests/companion_version.rs b/crates/epiphany-textproj/tests/companion_version.rs new file mode 100644 index 0000000..4dbe14c --- /dev/null +++ b/crates/epiphany-textproj/tests/companion_version.rs @@ -0,0 +1,59 @@ +use epiphany_textproj::COMPANION_VERSION; + +fn title_page_version(source: &str) -> Result<(u32, u32, u32), String> { + let title_page = source + .split_once("\\begin{titlepage}") + .and_then(|(_, after_start)| after_start.split_once("\\end{titlepage}")) + .map(|(title_page, _)| title_page) + .ok_or_else(|| "companion must contain one complete titlepage environment".to_owned())?; + + let candidates: Vec<_> = title_page + .lines() + .filter_map(|line| line.split_once("Version ").map(|(_, suffix)| suffix)) + .collect(); + if candidates.len() != 1 { + return Err(format!( + "title page must contain exactly one `Version ..` anchor; found {}", + candidates.len() + )); + } + + let token: String = candidates[0] + .chars() + .take_while(|character| character.is_ascii_digit() || *character == '.') + .collect(); + let components: Vec<_> = token.split('.').collect(); + if components.len() != 3 || components.iter().any(|component| component.is_empty()) { + return Err(format!( + "title-page version must have exactly three numeric components; found `{token}`" + )); + } + + let parse_component = |component: &str| { + component + .parse::() + .map_err(|_| format!("invalid numeric component `{component}` in title-page version")) + }; + Ok(( + parse_component(components[0])?, + parse_component(components[1])?, + parse_component(components[2])?, + )) +} + +#[test] +fn companion_version_matches_title_page() { + let companion_path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../spec/text_projection.tex" + ); + let companion = std::fs::read_to_string(companion_path) + .unwrap_or_else(|error| panic!("failed to read {companion_path}: {error}")); + let title_version = title_page_version(&companion) + .unwrap_or_else(|error| panic!("failed to parse {companion_path}: {error}")); + + assert_eq!( + title_version, COMPANION_VERSION, + "title-page companion version differs from epiphany_textproj::COMPANION_VERSION" + ); +} diff --git a/spec/CONTRACT_TEXTPROJ_DOCUMENT.md b/spec/CONTRACT_TEXTPROJ_DOCUMENT.md new file mode 100644 index 0000000..cd8855d --- /dev/null +++ b/spec/CONTRACT_TEXTPROJ_DOCUMENT.md @@ -0,0 +1,155 @@ +# Contract: the Text Projection document layer + +Repo root `/home/jeans/Repos/active/epiphany`. Read this in full before writing a +line. The plan is `spec/PLAN_TEXTPROJ_DOCUMENT.md`; both its rulings are granted +and this contract states them as law. + +The two layers below you are done and are your model: `epiphany-core`'s +`textvalue*.rs` (Chapter-5 values) and `epiphany-ops`'s `textproj_*.rs` +(Chapter-6 operations). Read the latter — the document layer is the same shape. + +## Grammar-directed, and free functions + +`req:textproj:operation-vocabulary` established that the grammar's productions +govern, not the mechanical value rule. The document lines are the same: find your +production in `spec/text_projection.tex`'s Grammar chapter and implement exactly +what it says. + +**Do not implement `TextValue` for bundle types.** `epiphany-bundle` does not +depend on `epiphany-core`, so the trait is foreign and so is the type — the +orphan rule forbids it from `epiphany-textproj`. You do not need it: no +document-line production contains a `value` position; every one bottoms out in +`bytes`, `integer`, `bool`, `string`, `option`, or a closed vocabulary. Write +**free functions** — `fn project_profile(&ProfileDeclaration) -> Sexp`, +`fn parse_profile(&Sexp) -> Result`. Use +`epiphany_core::textvalue::{Sexp, read_sexp, TextError}` for the machinery. + +Do **not** add `epiphany-core` to `epiphany-bundle`'s dependencies. The bundle is +a container format and its independence from the music model is deliberate. + +## Blobs: emit none, and reject them on parse + +A blob is canonical **iff** referenced by a canonical operation or by canonical +reduced state (`req:textproj:canonical-blobs`; `core_spec` §"Canonical and +Non-Canonical Manifest Roots"). Nothing in `epiphany-core` or `epiphany-ops` +references a `BlobId` — verified by source scan — so **no blob is canonical +today** and the correct projection of every real bundle emits zero `(blob ...)` +lines. + +Three obligations, and they are not the obvious ones: + +1. **Emit side.** Implement the reachability predicate as a real function that + today provably returns the empty set. **Never project + `manifest.blob_roots`** — that is the plausible wrong answer: one line, looks + right, emits non-canonical blobs in violation of the requirement. Even + `manifest.rs`'s own doc comment says "canonical `blob_roots`" — the subset, not + the field. + +2. **Parse side: reject.** Document-level validation **MUST reject any + `(blob ...)` line** as unreferenced, with a test asserting the rejection. + A blob-bearing text at this companion version is necessarily non-canonical. + Accepting one would stage a blob into the bundle that the next projection + silently drops, which loses data *and* falsifies + `project(serialize(parse(T))) == T` for that text. Forward compatibility is + owned by header-version gating, not by leniency here. + + Line-level `project`/`parse` of the `(blob ...)` production must still be + written and unit-tested in **both** directions against synthetic data, so that + when the predicate becomes non-empty both sides are ready at once. + +3. **A trip-wire.** A test that source-scans `crates/epiphany-core/src` and + `crates/epiphany-ops/src` for the token `BlobId` and fails if it appears — same + family as the existing drift locks. Its failure message must name the + reachability predicate and tell the finder that both the emit side and the + parse-side rejection now need their real implementation. + +## The header carries exactly one version + +The header names the version of the companion the text conforms to. The parser +**accepts exactly one**: the companion version this crate implements. Anything +else is a rejection at line one. Multi-version acceptance and migrate-on-read for +text are future spec decisions — do not improvise them. + +Define the version once, as a constant, and lock it against the companion with a +test that reads `spec/text_projection.tex` and asserts the title version matches. +A constant that silently disagrees with the document it claims conformance to is +the whole failure mode this project keeps paying for. + +## Section order is normative + +`projection ::= header document lineage? profile* extension* canonical-base? +blob* envelope*` is a **sequence**, not a set. A repeated or out-of-order section +is a rejection. A parser that accepts lines in any order and sorts them is +normalizing, which `req:textproj:strict-parse` forbids. + +## What must not appear (`req:textproj:derive-or-carry`) + +* **Physical attributes** — `offset`, `compressed_length`, `compression`, + `uncompressed_length`. A serializer chooses them freely. +* **Derivable identities** — `ChunkId`, `ContentHash`, `BlobId`. Re-derived from + content. +* The **one** identity carried verbatim is `SnapshotId`, which has nothing to + derive from. +* **Non-canonical accelerators are not projected**: `operation_index_root`, + `acceleration_snapshots`, `text_projection_root`, `integrity_root`, + `operation_block_summaries`. A bundle that round-trips through text comes back + without them. That is correct; say so in a doc comment, because it reads as data + loss and is not. + +## `derived-ordering` sorts by projected form + +`req:textproj:derived-ordering` applies to exactly two sequences: the `(blob ...)` +lines and an extension's preserved chunk roots. Order **and de-duplicate** them by +the **UTF-8 bytes of their rendered form** — not by any binary key, because the +binary key reads the offset, and a chunk's file position must not decide the +text's order. Every other sequence keeps the binary order. + +## Traps + +* **The extension line has six fields** in ratified declaration order: id, + version, required, chunks, affected-kinds, barriers. `affected_object_kinds` + and `edit_barriers` are **opaque byte strings**, never structured — the + `Vec`-is-not-a-sequence trap that bit the operation layer lives here too. + Each preserved chunk root projects as kind + schema_version + **uncompressed + payload**, never as a `ChunkRef`. +* **`ProfileId::Custom`** carries a 16-byte registry id and is the one profile + that is not a bare symbol. +* **Two `WallClockDuration` types exist**, one in `epiphany-core` and one in + `epiphany-bundle`. The retention policy uses the bundle's. +* **`canonical-base` carries the root chunk's uncompressed payload inline**, as a + single byte string, plus the `SnapshotId` verbatim. +* **Envelopes are emitted in `canonical_reduction_order`** + (`epiphany_ops::canonical_reduction_order`, public and exported). + +## Style + +Match the surrounding code. Doc comments on every public item and every +non-obvious decision — especially *why* a parse rejects rather than normalizes. +`rustfmt` clean, no `clippy` warnings, no `#[allow(...)]`, no `unwrap`/`expect` on +a parse path. Touch only the file you are told to create; do not run +`cargo fmt --all`. + +## Verification + +The standing gate: `cargo fmt --all --check`; `cargo clippy --workspace +--all-targets` → 0; full workspace tests; `RUSTDOCFLAGS="-D warnings" cargo doc +--workspace --no-deps` → 0; `cargo run -q -p epiphany-testkit --example +conformance_suite`; zero golden churn. + +Plus the three this project learned the hard way: + +1. **Mutation-verify every check you write.** Assert the anchor is present, delete + or invert the check, confirm a *named* test of yours fails, restore. Report per + check, with the actual command output. A survivor is a finding — say so. +2. **Exercise every outbound normalization.** For every value you normalize on the + way out, construct a non-normalized input and prove the normalization happens. + `derived-ordering` is exactly this shape: build a document with duplicate blobs + and out-of-order chunk roots *in memory* and prove the projection collapses and + orders them. Five such normalizations shipped untested in the operation layer + because every fixture was already sorted. +3. **Assert your suite's own reach.** A round-trip suite that never exercises an + extension, a canonical base, or a multi-envelope document proves far less than + its green tick suggests. Count what you covered and assert the counts. + +Report the actual commands and their actual output. A previous agent reported +"verification passes" when errors did point into its own file. diff --git a/spec/PLAN_TEXTPROJ_DOCUMENT.md b/spec/PLAN_TEXTPROJ_DOCUMENT.md new file mode 100644 index 0000000..4873678 --- /dev/null +++ b/spec/PLAN_TEXTPROJ_DOCUMENT.md @@ -0,0 +1,217 @@ +# Text Projection — the document layer: scope and plan + +Status: **scoping complete, two rulings needed before dispatch.** +Prepared against `master` @ `fdb4a57`. Every claim was checked against the code; +where I ran a probe I say so. + +This is the last phase. Chapter-5 values (`cf81074`) and Chapter-6 operations +(`fdb4a57`) already project and parse strictly. What remains is the document +around them: the header, identity, profile, extension, canonical-base and blob +lines — and then the two round-trip equations over a whole bundle. + +--- + +## 1. Decisions needed + +### Ruling A — no blob can be canonical today. What should the projector do? + +`req:textproj:canonical-blobs` and `core_spec` §"Canonical and Non-Canonical +Manifest Roots" agree exactly: a blob is canonical **iff** it is *referenced by a +canonical operation or by canonical reduced state*. A blob referenced only by +acceleration structures is non-canonical and `MUSTNOT` be projected. Canonicality +is by **reachability**, not by membership in `blob_roots`. + +I grepped `epiphany-core` and `epiphany-ops` for `BlobId`: **no hit.** No +operation payload and no Chapter-5 value carries a blob reference. There is no +mechanism by which a blob can be reached from canonical state, so **today the +correct projection of every real bundle emits zero `(blob ...)` lines**, and the +production is future-proofing. + +The dangerous wrong answer is projecting `manifest.blob_roots` wholesale — it +looks obviously right, is one line of code, and violates the requirement by +emitting non-canonical blobs. + +**Recommendation.** Implement the reachability predicate as a real function that +today provably returns the empty set, and *prove* it rather than asserting it: a +test that fails the moment any core or ops type gains a `BlobId` field, so the +hook is implemented when it is first needed rather than silently skipped. Same +discipline as computing a bound instead of spelling it. The `(blob ...)` +projection and parse must still be written and tested against a synthetic +document, because the parser must accept texts that a future writer emits. + +### Ruling B — the header version is the companion version, and that couples every byte + +The companion says the header names *"the version of **this companion** the text +conforms to."* So a projection written today starts `(text-projection (0 6 0))`. + +Two consequences worth accepting deliberately: + +* The companion's own worked example currently reads `(text-projection (0 3 0))` + — **stale**. My conformance lock covers the envelope line of that example, not + the header above it. +* Every companion version bump changes the first line of every projection, so + `req:textproj:canonical-text` is *version-relative* and the text vector corpus + must be regenerated on each bump. + +**Recommendation.** Keep the coupling — a text that does not say which rules it +follows cannot be strictly parsed against them — fix the stale example, and make +the corpus regeneration a documented, one-command step so the cost is mechanical +rather than a surprise. If instead you want the header to track a slower-moving +*format* version independent of editorial revisions, that is a spec change and +should be decided now, not after a corpus exists. + +--- + +## 2. What I verified + +**A new crate is unavoidable, and the orphan rule does not bite.** +`epiphany-bundle` depends only on `epiphany-determinism` — **not** on +`epiphany-core`. So `TextValue` (a core trait) cannot be implemented for +`Manifest`, `BlobRef`, `ProfileDeclaration` and friends from any third crate. + +That would have been a real problem, except: **no document-line production +contains a `value` position.** Every one bottoms out in leaves — `bytes`, +`integer`, `bool`, `string`, `option` — plus the closed vocabularies +`profile-id`, `chunk-kind`, `schema`. So the document layer needs **free +functions**, not trait impls, exactly as the grammar-directed operation layer +did. No orphan problem, no new dependency edge, and `epiphany-bundle` stays +independent of the music model as designed. + +`epiphany-textproj` depends on core (for `Sexp`, `render`, `read_sexp`), ops (for +`project_envelope`/`parse_envelope` and `canonical_reduction_order`), and bundle. + +**Parse cannot produce a `Manifest`.** The text carries **payloads inline** — the +canonical base's root chunk payload, each extension's preserved chunk payloads, +each blob's payload — whereas a `Manifest` holds `ChunkRef`s with offsets and +lengths that the text deliberately erases. The parse output needs its own type +(§4), and the bundle writer synthesizes the physical layer. + +**The write path already does the synthesis.** `Bundle::create` + `commit` +assigns offsets, content-addresses and de-duplicates chunks, and derives every +`ChunkId`/`ContentHash`. That is precisely what `req:textproj:derive-or-carry` +assumes, so serialization stages payloads and lets `commit` do the rest. + +**Envelope ordering is available.** `epiphany_ops::canonical_reduction_order` is +public and exported; `Bundle::read_operation_block` yields raw envelope bytes for +`decode_envelope`. + +**Accelerators are dropped, by design.** `operation_index_root`, +`acceleration_snapshots`, `text_projection_root`, `integrity_root` and +`operation_block_summaries` are non-canonical and are not projected. A bundle +that goes text → binary comes back without them. That is correct and should be +stated in a doc comment, because it looks like data loss and is not. + +--- + +## 3. The round trip, stated precisely + +`req:textproj:roundtrip` gives two equations. **Both hold; neither says bytes +survive.** + +``` + semantics(parse(project(B))) = semantics(B) -- weaker + project(serialize(parse(T))) = T -- byte-checkable, the corpus test +``` + +Going bundle → text → bundle is **deliberately lossy in the physical layer** and +in three specific ways an implementer will otherwise read as bugs: + +1. **Duplicate blobs collapse.** `req:textproj:derived-ordering` de-duplicates + blob lines and extension chunk roots by *projected form*. Two byte-identical + blobs stored twice are one blob; that is the erasure working. +2. **Layout is regenerated.** Offsets, compression, block splitting, chunk count. +3. **Accelerators vanish**, as above. + +All three preserve `semantics(...)`, and none of them affects the second +equation, which quantifies over *texts*, not bundles. Write the corpus against +the second equation. + +--- + +## 4. Work breakdown + +New crate `crates/epiphany-textproj`. Three waves; file boundaries are +one-per-agent. + +### Wave 1 — scaffold and types (blocks everything; small) + +`Cargo.toml`, `lib.rs`, and the parse-output type. Roughly: + +```rust +pub struct TextDocument { + pub document_id: DocumentId, + pub lineage_id: Option, + pub profiles: Vec, + pub extensions: Vec, // chunk payloads INLINE + pub canonical_base: Option, // root payload INLINE + pub blobs: Vec, // payloads INLINE + pub envelopes: Vec, // canonical operation order +} +``` + +The `Text*` types exist because the corresponding bundle types carry `ChunkRef`s +the text erases. Do not try to reuse `SnapshotRef`/`ExtensionDeclaration` +verbatim. + +### Wave 2 — three parallel agents + +* **`project.rs`** — `Bundle` → `TextDocument` → text. Reads chunk and blob + payloads through the bundle, decodes envelopes, sorts by + `canonical_reduction_order`, and applies `derived-ordering`'s sort-and-dedup by + projected form to blob lines and extension chunk roots. +* **`parse.rs`** — text → `TextDocument`, strictly. Line-oriented: each line is + one complete s-expression (`req:textproj:envelope-per-line`), and the line + *order* is itself constrained by the `projection` production — header, document, + lineage?, profile*, extension*, canonical-base?, blob*, envelope*. A + out-of-order or repeated section is a rejection, not a reordering. +* **`serialize.rs`** — `TextDocument` → `Bundle`. Stages the base root chunk, + extension chunks, blobs and one operation-envelope block; builds the manifest; + commits. Block splitting is a free physical choice — one block is fine and + simplest. + +### Wave 3 — conformance + +Text vectors in the shape of `spec/vectors/decode_vectors.txt`, drift-locked and +gated as conformance step **`[7e]`**, plus the two round-trip equations over the +generated-score corpus. This is also where the stale `(0 3 0)` header in the +companion's example gets fixed and locked. + +--- + +## 5. Traps + +* **`(blob ...)` is unreachable from real data.** Its tests need a synthetic + document. See Ruling A. +* **Section order is normative.** The `projection` production is a sequence, not + a set. A parser that accepts lines in any order and sorts them is normalizing. +* **`derived-ordering` sorts by *projected form*, not by any binary key** — order + and de-duplicate on the rendered UTF-8 bytes of each line, which is the whole + point (a chunk's offset must not decide the text's order). +* **The extension line is easy to get subtly wrong.** Six fields in ratified + declaration order — id, version, required, chunks, affected-kinds, barriers — + and `affected_object_kinds`/`edit_barriers` are opaque **byte strings**, never + structured. The `Vec`-is-not-a-sequence trap that bit the ops layer lives + here too. +* **`ProfileId::Custom` carries a 16-byte registry id** and is the one profile + that is not a bare symbol. +* **Do not project `manifest.blob_roots`.** See Ruling A; it is the plausible + wrong answer. +* **Two `WallClockDuration` types exist** — one in `epiphany-core`, one in + `epiphany-bundle`. The retention policy uses the bundle's. + +--- + +## 6. Verification contract + +The standing gate, plus: + +* **Mutation-verify every check**, anchor asserted before substitution, and report + per-check with output. +* **Exercise every outbound normalization** — the contract rule added after five + such normalizations shipped untested in the ops layer. `derived-ordering`'s + sort-and-dedup is exactly this shape: build a document with duplicate blobs and + out-of-order chunk roots *in memory* and prove the projection collapses and + orders them. +* **Assert the corpus's own reach.** A round-trip suite that never exercises an + extension, a canonical base, or a multi-envelope document proves less than its + green tick suggests. State the counts.