From 244bd6bd964b623ce72e03359d401743bad3de42 Mon Sep 17 00:00:00 2001 From: Levi Neuwirth Date: Tue, 21 Jul 2026 15:05:28 -0400 Subject: [PATCH] Text Projection: the document layer, and the equation closes `epiphany-textproj` projects a bundle to its canonical text and reads it back: the header, identity, profile, extension, canonical-base and blob productions, plus the pipeline that joins them to the operation layer already in place. Free functions, not `TextValue` impls. `epiphany-bundle` does not depend on `epiphany-core`, so the trait and the manifest types are both foreign to this crate and the orphan rule forbids the impls. That looked like it would force a new dependency edge until the productions were read properly: no document line contains a `value` position, so none is needed. The bundle stays independent of the music model, as designed. Blobs are handled on both sides and neither is the obvious one. The emit side filters `blob_roots` through a real reachability predicate that today provably returns empty -- never the field wholesale, which is the plausible wrong answer. The parse side rejects any blob line outright, while still parsing the production in isolation so both halves are ready together. A trip-wire source-scans epiphany-core and epiphany-ops for `BlobId` and names both obligations when it fires; it asserts it actually scanned files, so it cannot pass vacuously. Two rejections were added on the agents' own reading of the spec rather than the brief, and both are right. Envelopes must already be in canonical reduction order (text_projection.tex:458) -- without that, two texts differing only in envelope order would denote one document. And a hand-built `TextDocument` carrying blobs is refused rather than silently dropped, since serializing it would produce a bundle whose next projection loses those bytes. Section order is enforced as a sequence, not a set: a repeated or out-of-order section is a rejection, never something the parser sorts back into place. `derived-ordering` sorts and de-duplicates blob lines and extension chunk roots by *rendered* form, proved against fixtures that are unsorted in memory rather than already canonical -- the blind spot that let five outbound normalizations ship untested in the operation layer. Serialization stages payloads and lets `commit` assign every offset, hash and chunk id, which is exactly what `req:textproj:derive-or-carry` assumes; nothing is hashed by hand. The emitted bundle deliberately carries no accelerators. And the three directions compose: `project(serialize(parse(T))) == T` holds end-to-end over the companion's own worked example. Eight checks re-mutated independently of the agents' reports, each killed by its own named test; all eight document productions diffed against the grammar. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 10 + Cargo.toml | 4 + crates/epiphany-textproj/Cargo.toml | 14 + crates/epiphany-textproj/src/lib.rs | 117 ++ crates/epiphany-textproj/src/parse.rs | 1125 ++++++++++++++++ crates/epiphany-textproj/src/project.rs | 1176 +++++++++++++++++ crates/epiphany-textproj/src/serialize.rs | 561 ++++++++ .../tests/companion_version.rs | 59 + spec/CONTRACT_TEXTPROJ_DOCUMENT.md | 155 +++ spec/PLAN_TEXTPROJ_DOCUMENT.md | 217 +++ 10 files changed, 3438 insertions(+) create mode 100644 crates/epiphany-textproj/Cargo.toml create mode 100644 crates/epiphany-textproj/src/lib.rs create mode 100644 crates/epiphany-textproj/src/parse.rs create mode 100644 crates/epiphany-textproj/src/project.rs create mode 100644 crates/epiphany-textproj/src/serialize.rs create mode 100644 crates/epiphany-textproj/tests/companion_version.rs create mode 100644 spec/CONTRACT_TEXTPROJ_DOCUMENT.md create mode 100644 spec/PLAN_TEXTPROJ_DOCUMENT.md diff --git a/Cargo.lock b/Cargo.lock index cb20006..72d0e0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1286,6 +1286,16 @@ dependencies = [ "epiphany-ops", ] +[[package]] +name = "epiphany-textproj" +version = "0.0.0" +dependencies = [ + "epiphany-bundle", + "epiphany-core", + "epiphany-determinism", + "epiphany-ops", +] + [[package]] name = "equivalent" version = "1.0.2" diff --git a/Cargo.toml b/Cargo.toml index 4c4ae81..fb854e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "crates/epiphany-core", "crates/epiphany-bundle", "crates/epiphany-ops", + "crates/epiphany-textproj", "crates/epiphany-layout-ir", "crates/epiphany-engrave", "crates/epiphany-render-svg", @@ -44,6 +45,9 @@ epiphany-bundle = { path = "crates/epiphany-bundle" } # testkit drives its convergence, reduction-determinism, and equivocation gates, # so its path is declared here alongside the other intra-workspace crates. epiphany-ops = { path = "crates/epiphany-ops" } +# The Text Projection companion is the document-level bridge between canonical +# bundles and their normative s-expression form. +epiphany-textproj = { path = "crates/epiphany-textproj" } # Agent E's epiphany-layout-ir is the layout IR + constraint-solver interface # (Chapters 7 & 9). Agent F's testkit drives its layout round-trip gate # (v0 acceptance criterion 6), so its path is declared here alongside the other diff --git a/crates/epiphany-textproj/Cargo.toml b/crates/epiphany-textproj/Cargo.toml new file mode 100644 index 0000000..2aa14af --- /dev/null +++ b/crates/epiphany-textproj/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "epiphany-textproj" +version = "0.0.0" +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +repository.workspace = true +description = "The Epiphany Text Projection companion: the canonical s-expression document representation and its strict bundle round trip." + +[dependencies] +epiphany-determinism.workspace = true +epiphany-core.workspace = true +epiphany-bundle.workspace = true +epiphany-ops.workspace = true diff --git a/crates/epiphany-textproj/src/lib.rs b/crates/epiphany-textproj/src/lib.rs new file mode 100644 index 0000000..9a5cf7f --- /dev/null +++ b/crates/epiphany-textproj/src/lib.rs @@ -0,0 +1,117 @@ +#![forbid(unsafe_code)] +//! The Epiphany Text Projection companion document layer. +//! +//! This crate bridges canonical bundle documents and the normative line-oriented +//! s-expression representation defined by `spec/text_projection.tex`. + +pub mod parse; +pub mod project; +pub mod serialize; + +use epiphany_bundle::{ + ChunkKind, DocumentId, ExtensionId, FrontierBytes, LineageId, ProfileDeclaration, ProfileId, + ReductionAlgorithmVersion, SchemaVersion, SemVer, SnapshotId, +}; +use epiphany_ops::OperationEnvelope; + +/// The one Text Projection companion version implemented by this crate. +/// +/// A parser must reject every other version rather than migrating or +/// normalizing it on read. +pub const COMPANION_VERSION: (u32, u32, u32) = (0, 7, 0); + +/// A parsed canonical Text Projection document. +/// +/// Under `req:textproj:derive-or-carry`, this representation deliberately erases +/// physical offsets, compressed and uncompressed storage lengths, and +/// compression choices because serialization is free to choose a new physical +/// layout. It also omits the derivable `ChunkId`, `ContentHash`, and `BlobId`; +/// those identities are recomputed from kind, schema, and inline payload. +/// Finally, it drops the non-canonical `operation_index_root`, +/// `acceleration_snapshots`, `text_projection_root`, `integrity_root`, and +/// `operation_block_summaries` accelerators. None contributes to canonical +/// document semantics, so a bundle serialized from this form correctly rebuilds +/// or omits them rather than carrying stale physical metadata. +#[derive(Debug, PartialEq)] +pub struct TextDocument { + /// Logical identity of the projected document. + pub document_id: DocumentId, + /// Optional shared-ancestor identity used for document genealogy. + pub lineage_id: Option, + /// Profile declarations in canonical manifest order. + pub profiles: Vec, + /// Extension declarations with every preserved chunk payload inline. + pub extensions: Vec, + /// Optional canonical base with its snapshot root payload inline. + pub canonical_base: Option, + /// Canonically reachable blobs with their payloads inline. + pub blobs: Vec, + /// Operation envelopes in canonical reduction order. + pub envelopes: Vec, +} + +/// An extension declaration in its text-document form. +/// +/// Unlike the bundle's `ExtensionDeclaration`, this type carries preserved +/// chunks as semantic kind/schema/payload triples, not physical `ChunkRef`s. +#[derive(Debug, PartialEq)] +pub struct TextExtension { + /// Opaque identity of the extension. + pub extension_id: ExtensionId, + /// Semantic version of the extension declaration. + pub version: SemVer, + /// Whether an implementation unaware of the extension must refuse editing. + pub required: bool, + /// Preserved extension chunks, inline and ordered by projected form. + pub chunks: Vec, + /// Canonical opaque encoding of affected object kinds. + pub affected_object_kinds: Vec, + /// Canonical opaque encoding of the extension's edit barriers. + pub edit_barriers: Vec, +} + +/// One preserved extension chunk with all physical reference data erased. +#[derive(Debug, PartialEq)] +pub struct TextChunk { + /// Semantic role of the chunk. + pub kind: ChunkKind, + /// Schema version governing the payload bytes. + pub schema_version: SchemaVersion, + /// Uncompressed chunk payload carried inline. + pub payload: Vec, +} + +/// A canonical base snapshot in its text-document form. +/// +/// The snapshot identity is carried because it is opaque, while the root chunk +/// identity and content hash are derived from its schema and inline payload. +#[derive(Debug, PartialEq)] +pub struct TextCanonicalBase { + /// Opaque snapshot identity, carried verbatim. + pub snapshot_id: SnapshotId, + /// Opaque causal frontier materialized by the snapshot. + pub covers_causal_frontier: FrontierBytes, + /// Reduction algorithm version used to produce the snapshot. + pub reduction_algorithm_version: ReductionAlgorithmVersion, + /// Profile under which the snapshot was produced. + pub profile_id: ProfileId, + /// Schema version of the snapshot root chunk. + pub root_schema_version: SchemaVersion, + /// Uncompressed snapshot root payload carried inline. + pub root_payload: Vec, +} + +/// A canonical blob in its text-document form. +/// +/// The payload is inline; its bundle `BlobId`, content hash, offset, lengths, +/// and compression metadata are deliberately absent and are derived or chosen +/// when serialized. +#[derive(Debug, PartialEq)] +pub struct TextBlob { + /// RFC 6838 media type. + pub media_type: String, + /// Optional declared maximum uncompressed size. + pub declared_max_uncompressed_length: Option, + /// Uncompressed blob payload carried inline. + pub payload: Vec, +} diff --git a/crates/epiphany-textproj/src/parse.rs b/crates/epiphany-textproj/src/parse.rs new file mode 100644 index 0000000..1dac604 --- /dev/null +++ b/crates/epiphany-textproj/src/parse.rs @@ -0,0 +1,1125 @@ +//! Strict parsing of canonical Text Projection documents. +//! +//! Parsing is grammar-directed over [`epiphany_core::textvalue::Sexp`] values +//! read by [`epiphany_core::textvalue::read_sexp`], reports +//! [`epiphany_core::textvalue::TextError`], and delegates envelope productions +//! to [`epiphany_ops::parse_envelope`]. +//! +//! # Free functions, not `TextValue` impls +//! +//! `TextValue` is a core-crate trait and every document-line type +//! (`Manifest`, `ProfileDeclaration`, ...) belongs to `epiphany-bundle`, which +//! does not depend on `epiphany-core`. Implementing a foreign trait for a +//! foreign type from this third crate is exactly what the orphan rule +//! forbids, and no document-line production needs it anyway: every one +//! bottoms out in `bytes`, `integer`, `bool`, `string`, `option`, or a closed +//! vocabulary, never a Chapter-5 `value`. So this module is free functions +//! throughout, one pair (or one parser, where there is no corresponding +//! `project_*` in this file) per grammar production. +//! +//! # Two rejections, and why each is a rejection rather than a normalization +//! +//! Every check below rejects rather than repairs, per `req:textproj:strict-parse`. +//! Two are load-bearing enough to call out specifically: +//! +//! * **Any `(blob ...)` line is rejected**, unconditionally, in +//! [`parse_document`]. At `crate::COMPANION_VERSION` no canonical operation +//! and no canonical reduced state can carry a `BlobId` (a source scan -- +//! the companion `epiphany-core`/`epiphany-ops` crates never mention the +//! type), so canonical state cannot reference a blob and every blob line is +//! necessarily unreferenced (`req:textproj:reject-unreferenced-blobs`). +//! Accepting one would stage a blob into the bundle that the very next +//! projection silently drops -- losing data and falsifying +//! `project(serialize(parse(T))) == T` for that text. The `(blob ...)` +//! *production* itself still parses -- `parse_blob` is exercised directly +//! against synthetic data below -- so the accept side is ready the moment +//! the reachability predicate becomes non-empty; only the document-level +//! decision to use it is a permanent "no". +//! * **Any header version other than `crate::COMPANION_VERSION` is rejected +//! at line one** (`req:textproj:header-version`). Multi-version acceptance +//! and migrate-on-read are deferred spec decisions; this parser does not +//! speculate about them. +//! +//! # Section order is a sequence, not a set +//! +//! `projection ::= header document lineage? profile* extension* +//! canonical-base? blob* envelope*` is consumed by [`parse_document`] in +//! exactly that order, greedily and without backtracking: each stage takes +//! every line that belongs to it and stops at the first line that does not. +//! Anything left over once every stage has run -- a repeated singular +//! section, a section that shows up before the one it follows, or simple +//! garbage -- is rejected as a whole. A parser that instead recognised every +//! line by its own head symbol and sorted them into place would accept +//! exactly the disordered and repeated texts this design rejects; that is +//! the normalization `req:textproj:strict-parse` forbids, so this parser +//! does not do it. + +use epiphany_bundle::{ + ChunkKind, DocumentId, ExtensionId, FrontierBytes, LineageId, ProfileConstraints, + ProfileDeclaration, ProfileId, ProfileRegistryId, ReductionAlgorithmVersion, RetentionPolicy, + SchemaVersion, SemVer, SnapshotId, WallClockDuration, +}; +use epiphany_core::textvalue::{read_sexp, Sexp, TextError, TextValue}; +use epiphany_ops::{canonical_reduction_order, parse_envelope, OperationEnvelope}; + +use crate::{ + TextBlob, TextCanonicalBase, TextChunk, TextDocument, TextExtension, COMPANION_VERSION, +}; + +// =========================================================================== +// The document-level driver. +// =========================================================================== + +/// Parses a complete canonical Text Projection into a [`TextDocument`], +/// rejecting anything that is not the canonical projection of the document it +/// denotes (`req:textproj:strict-parse`). +/// +/// `text` must be exactly what `req:textproj:envelope-per-line` describes: +/// lines separated by a single U+000A, a final U+000A, and nothing else +/// trailing. Every line is one complete s-expression, and the sequence of +/// lines must follow `projection`'s section order precisely -- see the +/// module documentation for why an out-of-order or repeated section is a +/// rejection rather than something to sort back into place. +pub fn parse_document(text: &str) -> Result { + let raw_lines = split_lines(text)?; + let mut lines = Lines { + lines: raw_lines, + pos: 0, + }; + + // header: mandatory, first (`req:textproj:header-version`). + require_line( + &mut lines, + "text-projection", + "a projection must begin with a header line naming the companion version", + parse_header, + )?; + + // document: mandatory, immediately after the header. + let document_id = require_line( + &mut lines, + "document", + "a projection must carry a document line immediately after its header", + parse_document_id_line, + )?; + + // lineage? + let lineage_id = take_line(&mut lines, "lineage", parse_lineage)?; + + // profile*, in canonical (profile_id, version) order -- no repeats. + let mut profiles = Vec::new(); + while let Some(profile) = take_line(&mut lines, "profile", parse_profile)? { + profiles.push(profile); + } + if profiles + .windows(2) + .any(|w| (w[0].profile_id, w[0].version) >= (w[1].profile_id, w[1].version)) + { + return Err(TextError::NotStrictlyIncreasing( + "profile declarations must be ordered, without repetition, by (profile_id, version)", + )); + } + + // extension*, in canonical (extension_id, version) order -- no repeats. + let mut extensions = Vec::new(); + while let Some(extension) = take_line(&mut lines, "extension", parse_extension)? { + extensions.push(extension); + } + if extensions + .windows(2) + .any(|w| (w[0].extension_id, w[0].version) >= (w[1].extension_id, w[1].version)) + { + return Err(TextError::NotStrictlyIncreasing( + "extension declarations must be ordered, without repetition, by (extension_id, version)", + )); + } + + // canonical-base? + let canonical_base = take_line(&mut lines, "canonical-base", parse_canonical_base)?; + + // blob*: collected only so the rejection below can fire; see the module + // documentation and `req:textproj:reject-unreferenced-blobs`. + let mut blobs = Vec::new(); + while let Some(blob) = take_line(&mut lines, "blob", parse_blob)? { + blobs.push(blob); + } + if !blobs.is_empty() { + return Err(TextError::NotCanonical( + "a (blob ...) line is unreferenced by canonical state: no canonical operation or \ + reduced state can carry a BlobId at this companion version, so no blob line can \ + ever be canonical here", + )); + } + + // envelope*, in canonical reduction order. + let mut envelopes = Vec::new(); + while let Some(line) = lines.peek() { + let sexp = read_sexp(line)?; + if line_head(&sexp) != Some("envelope") { + break; + } + lines.advance(); + envelopes.push(parse_envelope(line)?); + } + check_envelope_order(&envelopes)?; + + // Anything left over is a line that belongs to no remaining stage: a + // repeated singular section, a section reappearing after a later one, or + // plain garbage after the envelopes. Every one of those is a rejection, + // never a re-sort. + if lines.peek().is_some() { + return Err(TextError::Syntax( + "a line appears out of the projection's normative section order, or a section that \ + may occur at most once is repeated", + )); + } + + Ok(TextDocument { + document_id, + lineage_id, + profiles, + extensions, + canonical_base, + blobs: Vec::new(), + envelopes, + }) +} + +/// A cursor over a projection's lines, advanced only by [`take_line`] and +/// [`require_line`] so every stage of [`parse_document`] consumes lines in +/// exactly one pass. +struct Lines<'a> { + lines: Vec<&'a str>, + pos: usize, +} + +impl<'a> Lines<'a> { + fn peek(&self) -> Option<&'a str> { + self.lines.get(self.pos).copied() + } + + fn advance(&mut self) { + self.pos += 1; + } +} + +/// Splits a whole projection into its lines, enforcing the layout half of +/// `req:textproj:envelope-per-line`: a single trailing U+000A and no other +/// trailing whitespace. Without this check a projection missing its final +/// U+000A would still split into the same lines by `str::split('\n')` and +/// parse identically -- the trailing-newline requirement would be silently +/// unenforced. An embedded blank line (a stray `U+000A U+000A`, anywhere) +/// needs no special case here: it becomes an empty line, and every stage +/// below rejects an empty line when it tries to read an s-expression from it. +fn split_lines(text: &str) -> Result, TextError> { + let body = text.strip_suffix('\n').ok_or(TextError::Syntax( + "a projection must end with exactly one U+000A and no other trailing whitespace", + ))?; + Ok(body.split('\n').collect()) +} + +/// The head symbol of a line's s-expression, or `None` if it is not headed by +/// one (a bare leaf line is never a valid section head, so this only ever +/// hides a genuine mismatch). +fn line_head(sexp: &Sexp) -> Option<&str> { + sexp.as_list()?.first()?.as_symbol() +} + +/// Consumes the next line and applies `parse` to it, but only if it is +/// present and its head symbol is `head`. Otherwise the cursor is left +/// untouched and `Ok(None)` is returned -- the mechanism by which every +/// optional or repeatable section in `projection` is optional, and by which a +/// line belonging to a later or earlier stage is left for another stage (or +/// the final leftover check) to deal with, rather than being consumed out of +/// place. +fn take_line<'a, T>( + lines: &mut Lines<'a>, + head: &str, + parse: impl FnOnce(&Sexp) -> Result, +) -> Result, TextError> { + let Some(line) = lines.peek() else { + return Ok(None); + }; + let sexp = read_sexp(line)?; + if line_head(&sexp) != Some(head) { + return Ok(None); + } + lines.advance(); + Ok(Some(parse(&sexp)?)) +} + +/// As [`take_line`], but the line is mandatory: a missing or mismatched line +/// is `missing`, not `Ok(None)`. Used for the header and the document line, +/// the two productions `projection` requires unconditionally. +fn require_line<'a, T>( + lines: &mut Lines<'a>, + head: &str, + missing: &'static str, + parse: impl FnOnce(&Sexp) -> Result, +) -> Result { + let Some(line) = lines.peek() else { + return Err(TextError::Syntax(missing)); + }; + let sexp = read_sexp(line)?; + if line_head(&sexp) != Some(head) { + return Err(TextError::Syntax(missing)); + } + lines.advance(); + parse(&sexp) +} + +/// `req:textproj:derived-ordering`'s "every other sequence keeps the binary +/// order" names the envelopes' binary order as +/// [`epiphany_ops::canonical_reduction_order`] -- a deterministic function of +/// the envelopes' own causal contexts and stamps, not a free choice a writer +/// makes. So a text whose envelope lines are already in that order is the +/// only text `req:textproj:canonical-text` permits for that envelope set, and +/// accepting some other order here -- and silently keeping it -- would be +/// exactly the normalization `req:textproj:strict-parse` forbids: the next +/// projection re-sorts the same envelopes into canonical order, so +/// `project(serialize(parse(T))) == T` fails for precisely the malformed `T` +/// this rejects up front. +fn check_envelope_order(envelopes: &[OperationEnvelope]) -> Result<(), TextError> { + let given: Vec<&OperationEnvelope> = envelopes.iter().collect(); + let canonical = canonical_reduction_order(&given); + let already_canonical = given + .iter() + .zip(canonical.iter()) + .all(|(a, b)| std::ptr::eq(*a, *b)); + if !already_canonical { + return Err(TextError::NotStrictlyIncreasing( + "envelope lines must appear in canonical reduction order \ + (epiphany_ops::canonical_reduction_order)", + )); + } + Ok(()) +} + +// =========================================================================== +// Leaves shared by several productions. +// =========================================================================== + +/// The lexical class of `s`, for error messages. Mirrors the private helper +/// of the same name in `epiphany_core::textvalue` and in every `textproj_*` +/// module of `epiphany-ops`: `Sexp::class` is not public, so each grammar- +/// directed module restates this one match rather than depend on it. +fn class_of(s: &Sexp) -> &'static str { + match s { + Sexp::List(_) => "list", + Sexp::Symbol(_) => "symbol", + Sexp::Int(_) => "integer", + Sexp::Bytes(_) => "byte string", + Sexp::Str(_) => "string", + } +} + +/// Reads the grammar's opaque `bytes` terminal as a plain byte string. +/// `Vec`'s generic `TextValue` impl denotes a *sequence of integers*, a +/// different production, so it must never stand in for an opaque payload, +/// identifier, or hash -- the same trap the operation layer names in +/// `textproj_kind.rs`. +fn parse_bytes(s: &Sexp) -> Result, TextError> { + match s { + Sexp::Bytes(bytes) => Ok(bytes.clone()), + _ => Err(TextError::Expected { + expected: "byte string", + found: class_of(s), + }), + } +} + +/// Reads a fixed 16-byte opaque identifier: a `DocumentId`, `LineageId`, +/// `SnapshotId`, `ExtensionId`, or a custom `ProfileId`'s registry id. `what` +/// names the identifier for the rejection message. +fn parse_id16(s: &Sexp, what: &'static str) -> Result<[u8; 16], TextError> { + parse_bytes(s)? + .try_into() + .map_err(|_| TextError::NotCanonical(what)) +} + +/// The grammar's anonymous `version ::= "(" integer " " integer " " integer +/// ")"` production, shared verbatim by the header (against +/// `crate::COMPANION_VERSION`) and by every `SemVer` field. +fn parse_version_triple(s: &Sexp) -> Result<(u32, u32, u32), TextError> { + let items = s.as_list().ok_or(TextError::Expected { + expected: "version", + found: class_of(s), + })?; + let [major, minor, patch] = items else { + return Err(TextError::Arity { + type_name: "version", + expected: 3, + found: items.len(), + }); + }; + Ok((u32::parse(major)?, u32::parse(minor)?, u32::parse(patch)?)) +} + +fn parse_semver(s: &Sexp) -> Result { + let (major, minor, patch) = parse_version_triple(s)?; + Ok(SemVer::new(major, minor, patch)) +} + +/// The grammar's `profile-id` production: a bare symbol for each +/// closed-vocabulary profile, and `(custom )` -- the one profile +/// identity that is not a bare symbol -- for `ProfileId::Custom`, whose +/// registry id is 16 opaque bytes. +fn parse_profile_id(s: &Sexp) -> Result { + if let Some(name) = s.as_symbol() { + return match name { + "full" => Ok(ProfileId::Full), + "read-only" => Ok(ProfileId::ReadOnly), + "lite" => Ok(ProfileId::Lite), + _ => Err(TextError::UnknownConstructor { + type_name: "ProfileId", + found: name.to_owned(), + }), + }; + } + let fields = s.expect_struct("custom", 1)?; + let registry_id = parse_id16( + &fields[0], + "a custom ProfileId registry id is exactly 16 bytes", + )?; + Ok(ProfileId::Custom(ProfileRegistryId(registry_id))) +} + +// =========================================================================== +// header, document, lineage. +// =========================================================================== + +/// `header ::= "(text-projection " version ")"`. Accepts exactly +/// `crate::COMPANION_VERSION`; any other version is a rejection at line one +/// (`req:textproj:header-version`). Multi-version acceptance and +/// migrate-on-read are deferred spec decisions this parser does not +/// speculate about. +fn parse_header(s: &Sexp) -> Result<(), TextError> { + let fields = s.expect_struct("text-projection", 1)?; + let version = parse_version_triple(&fields[0])?; + if version != COMPANION_VERSION { + return Err(TextError::NotCanonical( + "the header names a companion version other than the one this crate implements", + )); + } + Ok(()) +} + +/// `document ::= "(document " bytes ")"`. +fn parse_document_id_line(s: &Sexp) -> Result { + let fields = s.expect_struct("document", 1)?; + Ok(DocumentId(parse_id16( + &fields[0], + "a DocumentId is exactly 16 bytes", + )?)) +} + +/// `lineage ::= "(lineage " bytes ")"`. +fn parse_lineage(s: &Sexp) -> Result { + let fields = s.expect_struct("lineage", 1)?; + Ok(LineageId(parse_id16( + &fields[0], + "a LineageId is exactly 16 bytes", + )?)) +} + +// =========================================================================== +// profile, constraints, retention. +// =========================================================================== + +/// `profile ::= "(profile " profile-id " " version " " constraints ")"`. +fn parse_profile(s: &Sexp) -> Result { + let fields = s.expect_struct("profile", 3)?; + Ok(ProfileDeclaration { + profile_id: parse_profile_id(&fields[0])?, + version: parse_semver(&fields[1])?, + constraints: parse_constraints(&fields[2])?, + }) +} + +/// `constraints ::= "(constraints " integer " " retention ")"`. +fn parse_constraints(s: &Sexp) -> Result { + let fields = s.expect_struct("constraints", 2)?; + Ok(ProfileConstraints { + max_uncompressed_block_size: u64::parse(&fields[0])?, + retention_policy: parse_retention(&fields[1])?, + }) +} + +/// `retention ::= "(retention " integer " " option " " bool ")"`. +/// +/// The middle field is `Option`, and `WallClockDuration` is +/// a newtype over `i64`: per `req:textproj:value-projection` clause 2 a +/// newtype projects transparently as its field alone, so the option wraps a +/// bare integer, not a `(wall-clock-duration )` struct. This is +/// **`epiphany_bundle::WallClockDuration`**, not `epiphany_core`'s +/// same-named type -- the retention policy is a bundle type through and +/// through. +fn parse_retention(s: &Sexp) -> Result { + let fields = s.expect_struct("retention", 3)?; + Ok(RetentionPolicy { + retain_previous_manifests: u32::parse(&fields[0])?, + retain_duration: Option::::parse(&fields[1])?.map(WallClockDuration), + retain_named_checkpoints: bool::parse(&fields[2])?, + }) +} + +// =========================================================================== +// extension, chunk, chunk-kind, schema. +// =========================================================================== + +/// `extension ::= "(extension " bytes " " version " " bool " (" chunk* ") " +/// bytes " " bytes ")"` -- id, version, required, chunks, affected-kinds, +/// barriers, the ratified declaration order. `affected_object_kinds` and +/// `edit_barriers` are opaque byte strings, never structured: the bundle +/// preserves them without interpreting them, and this projection interprets +/// nothing the bundle does not. +fn parse_extension(s: &Sexp) -> Result { + let fields = s.expect_struct("extension", 6)?; + Ok(TextExtension { + extension_id: ExtensionId(parse_id16( + &fields[0], + "an ExtensionId is exactly 16 bytes", + )?), + version: parse_semver(&fields[1])?, + required: bool::parse(&fields[2])?, + chunks: parse_chunks(&fields[3])?, + affected_object_kinds: parse_bytes(&fields[4])?, + edit_barriers: parse_bytes(&fields[5])?, + }) +} + +/// `chunk ::= "(chunk " chunk-kind " " schema " " bytes ")"`: a preserved +/// extension chunk root projected as kind, schema version, and uncompressed +/// payload -- never a `ChunkRef`, which the projection has no file to point +/// into (`req:textproj:derive-or-carry`). +fn parse_chunk(s: &Sexp) -> Result { + let fields = s.expect_struct("chunk", 3)?; + Ok(TextChunk { + kind: parse_chunk_kind(&fields[0])?, + schema_version: parse_schema_version(&fields[1])?, + payload: parse_bytes(&fields[2])?, + }) +} + +/// The chunk* sequence inside an `extension` line. `req:textproj:derived-ordering` +/// orders and de-duplicates an extension's preserved chunk roots by their +/// *projected form* -- the binary `ChunkRef` order breaks ties on the file +/// offset, which this projection erases, so the binary order cannot be +/// inherited here. Accepting a disordered or duplicated chunk list and +/// re-sorting it at serialize time would be normalization; this rejects it +/// instead. +fn parse_chunks(s: &Sexp) -> Result, TextError> { + let items = s.as_list().ok_or(TextError::Expected { + expected: "chunk sequence", + found: class_of(s), + })?; + let mut out = Vec::with_capacity(items.len()); + let mut previous_rendered: Option = None; + for item in items { + let rendered = item.render(); + if previous_rendered + .as_deref() + .is_some_and(|previous| rendered.as_str() <= previous) + { + return Err(TextError::NotStrictlyIncreasing( + "an extension's preserved chunk roots must be ordered, without repetition, by \ + projected form", + )); + } + out.push(parse_chunk(item)?); + previous_rendered = Some(rendered); + } + Ok(out) +} + +/// `chunk-kind`: a bare symbol per `ChunkKind` variant, in the same +/// declaration order as the discriminants `req:format:chunkkind-discriminants` +/// pins. +fn parse_chunk_kind(s: &Sexp) -> Result { + match s.as_symbol() { + Some("operation-envelope-block") => Ok(ChunkKind::OperationEnvelopeBlock), + Some("operation-index") => Ok(ChunkKind::OperationIndex), + Some("snapshot") => Ok(ChunkKind::Snapshot), + Some("blob") => Ok(ChunkKind::Blob), + Some("extension-data") => Ok(ChunkKind::ExtensionData), + Some("text-projection") => Ok(ChunkKind::TextProjection), + Some("layout-cache") => Ok(ChunkKind::LayoutCache), + Some("integrity-index") => Ok(ChunkKind::IntegrityIndex), + Some("manifest") => Ok(ChunkKind::Manifest), + Some(name) => Err(TextError::UnknownConstructor { + type_name: "ChunkKind", + found: name.to_owned(), + }), + None => Err(TextError::Expected { + expected: "chunk-kind", + found: class_of(s), + }), + } +} + +/// `schema ::= "(schema " integer " " integer ")"`. +fn parse_schema_version(s: &Sexp) -> Result { + let fields = s.expect_struct("schema", 2)?; + Ok(SchemaVersion::new( + u16::parse(&fields[0])?, + u16::parse(&fields[1])?, + )) +} + +// =========================================================================== +// canonical-base. +// =========================================================================== + +/// `canonical-base ::= "(canonical-base " bytes " " bytes " " integer " " +/// profile-id " " schema " " bytes ")"` -- snapshot id, frontier, reduction +/// version, profile, root schema, root payload. The `SnapshotId` is the one +/// identity `req:textproj:derive-or-carry` carries verbatim rather than +/// re-deriving (schema major 0 has no snapshot producer, so it has nothing to +/// derive from); the root chunk's own id and content hash are re-derived by +/// whoever serializes this back into a bundle, never read from the text. +fn parse_canonical_base(s: &Sexp) -> Result { + let fields = s.expect_struct("canonical-base", 6)?; + Ok(TextCanonicalBase { + snapshot_id: SnapshotId(parse_id16(&fields[0], "a SnapshotId is exactly 16 bytes")?), + covers_causal_frontier: FrontierBytes::from_bytes(parse_bytes(&fields[1])?), + reduction_algorithm_version: ReductionAlgorithmVersion(u32::parse(&fields[2])?), + profile_id: parse_profile_id(&fields[3])?, + root_schema_version: parse_schema_version(&fields[4])?, + root_payload: parse_bytes(&fields[5])?, + }) +} + +// =========================================================================== +// blob. +// =========================================================================== + +/// `blob ::= "(blob " string " " option " " bytes ")"` -- media type, +/// declared maximum uncompressed length, payload. +/// +/// This is the production-level parser only. It accepts a well-formed +/// `(blob ...)` line unconditionally, exactly as every other production +/// parser here does; the decision that **no** accepted blob line may reach a +/// [`TextDocument`] is made once, at the document level, in +/// [`parse_document`] (`req:textproj:reject-unreferenced-blobs`). Keeping the +/// two apart -- this function parses, `parse_document` rejects -- means the +/// accept side is already correct and already tested the day the +/// reachability predicate stops returning the empty set, rather than needing +/// to be written from scratch alongside a spec bump. +fn parse_blob(s: &Sexp) -> Result { + let fields = s.expect_struct("blob", 3)?; + Ok(TextBlob { + media_type: String::parse(&fields[0])?, + declared_max_uncompressed_length: Option::::parse(&fields[1])?, + payload: parse_bytes(&fields[2])?, + }) +} + +// =========================================================================== +// Tests. +// =========================================================================== + +#[cfg(test)] +mod tests { + use super::*; + use epiphany_core::{OperationId, ReplicaId, WallClockTime}; + use epiphany_ops::{ + project_envelope, AuthorId, CausalContext, EnvelopeHash, HybridLogicalClock, + OperationStamp, ResolveEquivocationPayload, + }; + + // ----------------------------------------------------------------- + // Test fixtures. + // ----------------------------------------------------------------- + + /// Joins `lines` with a single U+000A each, including a final one -- + /// exactly `req:textproj:envelope-per-line`'s layout. + fn projection(lines: &[&str]) -> String { + let mut out = String::new(); + for line in lines { + out.push_str(line); + out.push('\n'); + } + out + } + + const HEADER: &str = "(text-projection (0 7 0))"; + const DOCUMENT: &str = "(document #x00000000000000000000000000000001)"; + + /// A minimal but complete valid projection: just the two mandatory lines. + fn minimal_valid_document() -> String { + projection(&[HEADER, DOCUMENT]) + } + + /// A simple, independent (empty causal context) envelope, so several of + /// these can be combined without any causal-order machinery beyond their + /// HLC physical time. + fn sample_envelope(replica: u64, counter: u64, physical_time: i64) -> OperationEnvelope { + let id = OperationId::new(ReplicaId(replica), counter); + OperationEnvelope { + id, + author: AuthorId(0x1122_3344), + stamp: OperationStamp::new( + HybridLogicalClock::new(WallClockTime(physical_time), 0), + id, + ), + causal_context: CausalContext::new(), + transaction: None, + payload: epiphany_ops::OperationPayload::ResolveEquivocation( + ResolveEquivocationPayload { + target: id, + chosen: EnvelopeHash([7; 32]), + }, + ), + } + } + + // ----------------------------------------------------------------- + // The worked example: a real, spec-authored, multi-section document. + // ----------------------------------------------------------------- + + /// The Grammar chapter's worked example, read live from the companion so + /// this test cannot drift from the document it claims to parse. + fn worked_example() -> String { + const SPEC: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../spec/text_projection.tex" + )); + SPEC.split_once("\\chapter{A Worked Example}") + .expect("the specification contains the worked example") + .1 + .split_once("\\begin{lstlisting}\n") + .expect("the worked example contains a projection listing") + .1 + .split_once("\\end{lstlisting}") + .expect("the worked projection listing is closed") + .0 + .to_owned() + } + + #[test] + fn the_worked_example_parses_to_its_documented_fields() { + let text = worked_example(); + let document = parse_document(&text).expect("the worked example is a valid projection"); + + assert_eq!(document.document_id, DocumentId([0x05; 16])); + assert_eq!(document.lineage_id, None); + assert_eq!(document.profiles.len(), 1); + assert_eq!(document.profiles[0].profile_id, ProfileId::Full); + assert_eq!(document.profiles[0].version, SemVer::new(0, 1, 0)); + assert_eq!( + document.profiles[0].constraints.max_uncompressed_block_size, + 67_108_864 + ); + assert_eq!( + document.profiles[0] + .constraints + .retention_policy + .retain_previous_manifests, + 1 + ); + assert_eq!( + document.profiles[0] + .constraints + .retention_policy + .retain_duration, + None + ); + assert!( + document.profiles[0] + .constraints + .retention_policy + .retain_named_checkpoints + ); + assert!(document.extensions.is_empty()); + + let base = document + .canonical_base + .as_ref() + .expect("the worked example carries a canonical base"); + assert_eq!( + base.snapshot_id, + SnapshotId([0x1f, 0x8b, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + ); + assert_eq!(base.covers_causal_frontier, FrontierBytes::empty()); + assert_eq!( + base.reduction_algorithm_version, + ReductionAlgorithmVersion(1) + ); + assert_eq!(base.profile_id, ProfileId::Full); + assert_eq!(base.root_schema_version, SchemaVersion::V0); + assert_eq!(base.root_payload, vec![0u8, 0u8]); + + assert!(document.blobs.is_empty()); + assert_eq!(document.envelopes.len(), 1); + } + + // ----------------------------------------------------------------- + // A synthetic document exercising lineage, multiple profiles, an + // extension with preserved chunks, a canonical base, and multiple + // envelopes -- everything the worked example alone does not reach. + // ----------------------------------------------------------------- + + #[test] + fn a_rich_synthetic_document_parses_every_section() { + let lineage = "(lineage #x00000000000000000000000000000002)"; + let profile_full = "(profile full (0 1 0) (constraints 67108864 (retention 1 () true)))"; + let profile_read_only = + "(profile read-only (0 1 0) (constraints 1024 (retention 0 (some 5) false)))"; + let extension = "(extension #x00000000000000000000000000000003 (1 0 0) true \ + ((chunk operation-envelope-block (schema 0 1) #xaa) (chunk snapshot (schema 0 1) #xbb)) \ + #xaabb #xccdd)"; + let base = + "(canonical-base #x00000000000000000000000000000004 #x 1 full (schema 0 1) #x0102)"; + + let e1 = sample_envelope(1, 1, 10); + let e2 = sample_envelope(2, 1, 20); + let e1_line = project_envelope(&e1); + let e2_line = project_envelope(&e2); + + let text = projection(&[ + HEADER, + DOCUMENT, + lineage, + profile_full, + profile_read_only, + extension, + base, + &e1_line, + &e2_line, + ]); + + let document = parse_document(&text).expect("a well-formed rich document parses"); + assert_eq!( + document.lineage_id, + Some(LineageId([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) + ); + assert_eq!(document.profiles.len(), 2); + assert_eq!(document.extensions.len(), 1); + assert_eq!(document.extensions[0].chunks.len(), 2); + assert_eq!( + document.extensions[0].chunks[0].kind, + ChunkKind::OperationEnvelopeBlock + ); + assert_eq!(document.extensions[0].chunks[1].kind, ChunkKind::Snapshot); + assert_eq!( + document.extensions[0].affected_object_kinds, + vec![0xaa, 0xbb] + ); + assert_eq!(document.extensions[0].edit_barriers, vec![0xcc, 0xdd]); + assert!(document.canonical_base.is_some()); + assert_eq!(document.envelopes, vec![e1, e2]); + } + + // ----------------------------------------------------------------- + // Rejection classes. Each is mutation-verified (see the task report): + // the anchor assertion below is deleted or inverted, a NAMED test in + // this list is confirmed to fail, and the code is restored. + // ----------------------------------------------------------------- + + #[test] + fn non_canonical_header_version_is_rejected() { + let text = projection(&["(text-projection (0 6 0))", DOCUMENT]); + assert_eq!( + parse_document(&text), + Err(TextError::NotCanonical( + "the header names a companion version other than the one this crate implements" + )) + ); + } + + #[test] + fn the_companion_version_itself_is_accepted() { + // The mutation-testing counterpart of the above: the exact version + // this crate implements must still parse. + assert!(parse_document(&minimal_valid_document()).is_ok()); + } + + #[test] + fn a_blob_line_is_always_rejected() { + let blob_line = "(blob \"audio/wav\" () #x00)"; + let text = projection(&[HEADER, DOCUMENT, blob_line]); + assert!(matches!( + parse_document(&text), + Err(TextError::NotCanonical(_)) + )); + } + + #[test] + fn the_blob_production_itself_parses_synthetic_data() { + // Line-level parse of `(blob ...)`, exercised directly against + // synthetic data per the task contract: the accept side is ready + // even though `parse_document` never lets it through. + let sexp = read_sexp("(blob \"audio/wav\" (some 1024) #x0a0b)").unwrap(); + let blob = parse_blob(&sexp).expect("a well-formed blob production parses"); + assert_eq!(blob.media_type, "audio/wav"); + assert_eq!(blob.declared_max_uncompressed_length, Some(1024)); + assert_eq!(blob.payload, vec![0x0a, 0x0b]); + } + + #[test] + fn missing_trailing_lf_is_rejected() { + let text = minimal_valid_document(); + let without_final_lf = text.strip_suffix('\n').unwrap(); + assert_eq!( + parse_document(without_final_lf), + Err(TextError::Syntax( + "a projection must end with exactly one U+000A and no other trailing whitespace" + )) + ); + } + + #[test] + fn an_extra_trailing_blank_line_is_rejected() { + let mut text = minimal_valid_document(); + text.push('\n'); // a second, stray trailing U+000A + assert!(parse_document(&text).is_err()); + } + + #[test] + fn missing_header_is_rejected() { + assert!(parse_document("").is_err()); + } + + #[test] + fn missing_document_line_is_rejected() { + let text = projection(&[HEADER]); + assert_eq!( + parse_document(&text), + Err(TextError::Syntax( + "a projection must carry a document line immediately after its header" + )) + ); + } + + #[test] + fn a_repeated_header_line_is_rejected() { + let text = projection(&[HEADER, HEADER, DOCUMENT]); + assert!(parse_document(&text).is_err()); + } + + #[test] + fn a_repeated_document_line_is_rejected() { + let text = projection(&[HEADER, DOCUMENT, DOCUMENT]); + assert_eq!( + parse_document(&text), + Err(TextError::Syntax( + "a line appears out of the projection's normative section order, or a section \ + that may occur at most once is repeated" + )) + ); + } + + #[test] + fn out_of_order_sections_are_rejected_not_sorted() { + let profile = "(profile full (0 1 0) (constraints 67108864 (retention 1 () true)))"; + let extension = "(extension #x00000000000000000000000000000003 (1 0 0) false () #x #x)"; + let base = "(canonical-base #x00000000000000000000000000000004 #x 1 full (schema 0 1) #x)"; + let lineage = "(lineage #x00000000000000000000000000000002)"; + let e1_line = project_envelope(&sample_envelope(1, 1, 10)); + + let scenarios: &[&[&str]] = &[ + // A profile line before the (mandatory) document line. + &[HEADER, profile, DOCUMENT], + // Lineage after a profile, instead of before it. + &[HEADER, DOCUMENT, profile, lineage], + // An extension before the profile section. + &[HEADER, DOCUMENT, extension, profile], + // A canonical base before the extension section. + &[HEADER, DOCUMENT, base, extension], + // An envelope before the canonical base. + &[HEADER, DOCUMENT, &e1_line, base], + ]; + for lines in scenarios { + let text = projection(lines); + assert!( + parse_document(&text).is_err(), + "expected rejection for out-of-order lines: {lines:?}" + ); + } + } + + #[test] + fn profile_declarations_must_be_ordered_and_unique() { + let full = "(profile full (0 1 0) (constraints 1 (retention 0 () false)))"; + let read_only = "(profile read-only (0 1 0) (constraints 1 (retention 0 () false)))"; + + // Correct ascending order accepts. + let ok = projection(&[HEADER, DOCUMENT, full, read_only]); + assert!(parse_document(&ok).is_ok()); + + // Reversed order is rejected, not silently re-sorted. + let reversed = projection(&[HEADER, DOCUMENT, read_only, full]); + assert_eq!( + parse_document(&reversed), + Err(TextError::NotStrictlyIncreasing( + "profile declarations must be ordered, without repetition, by (profile_id, version)" + )) + ); + + // An exact repeat (same key) is rejected too. + let repeated = projection(&[HEADER, DOCUMENT, full, full]); + assert!(parse_document(&repeated).is_err()); + } + + #[test] + fn extension_declarations_must_be_ordered_and_unique() { + let ext_a = "(extension #x00000000000000000000000000000001 (1 0 0) false () #x #x)"; + let ext_b = "(extension #x00000000000000000000000000000002 (1 0 0) false () #x #x)"; + + let ok = projection(&[HEADER, DOCUMENT, ext_a, ext_b]); + assert!(parse_document(&ok).is_ok()); + + let reversed = projection(&[HEADER, DOCUMENT, ext_b, ext_a]); + assert_eq!( + parse_document(&reversed), + Err(TextError::NotStrictlyIncreasing( + "extension declarations must be ordered, without repetition, by (extension_id, version)" + )) + ); + + let repeated = projection(&[HEADER, DOCUMENT, ext_a, ext_a]); + assert!(parse_document(&repeated).is_err()); + } + + #[test] + fn extension_chunk_roots_must_be_ordered_and_deduplicated() { + let chunk_a = "(chunk operation-envelope-block (schema 0 1) #xaa)"; + let chunk_b = "(chunk snapshot (schema 0 1) #xbb)"; + + let ordered = format!( + "(extension #x00000000000000000000000000000001 (1 0 0) false ({chunk_a} {chunk_b}) #x #x)" + ); + let text = projection(&[HEADER, DOCUMENT, &ordered]); + assert!(parse_document(&text).is_ok()); + + let reversed = format!( + "(extension #x00000000000000000000000000000001 (1 0 0) false ({chunk_b} {chunk_a}) #x #x)" + ); + let text = projection(&[HEADER, DOCUMENT, &reversed]); + assert_eq!( + parse_document(&text), + Err(TextError::NotStrictlyIncreasing( + "an extension's preserved chunk roots must be ordered, without repetition, by \ + projected form" + )) + ); + + let duplicated = format!( + "(extension #x00000000000000000000000000000001 (1 0 0) false ({chunk_a} {chunk_a}) #x #x)" + ); + let text = projection(&[HEADER, DOCUMENT, &duplicated]); + assert!(parse_document(&text).is_err()); + } + + #[test] + fn envelope_lines_must_be_in_canonical_reduction_order() { + let e1 = sample_envelope(1, 1, 10); + let e2 = sample_envelope(2, 1, 20); + let e1_line = project_envelope(&e1); + let e2_line = project_envelope(&e2); + + // Ascending physical time is the canonical order: accepted. + let ok = projection(&[HEADER, DOCUMENT, &e1_line, &e2_line]); + let parsed = parse_document(&ok).expect("already-canonical envelope order is accepted"); + assert_eq!(parsed.envelopes, vec![e1, e2]); + + // Swapped: no longer canonical reduction order, rejected outright. + let swapped = projection(&[HEADER, DOCUMENT, &e2_line, &e1_line]); + assert_eq!( + parse_document(&swapped), + Err(TextError::NotStrictlyIncreasing( + "envelope lines must appear in canonical reduction order \ + (epiphany_ops::canonical_reduction_order)" + )) + ); + } + + #[test] + fn profile_id_accepts_every_closed_symbol_and_the_custom_form() { + for (text, expected) in [ + ("full", ProfileId::Full), + ("read-only", ProfileId::ReadOnly), + ("lite", ProfileId::Lite), + ( + "(custom #x00000000000000000000000000000009)", + ProfileId::Custom(ProfileRegistryId([ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, + ])), + ), + ] { + let sexp = read_sexp(text).unwrap(); + assert_eq!(parse_profile_id(&sexp).unwrap(), expected); + } + } + + #[test] + fn profile_id_rejects_an_unknown_symbol_and_a_malformed_custom_id() { + assert!(parse_profile_id(&read_sexp("bespoke").unwrap()).is_err()); + assert!(parse_profile_id(&read_sexp("(custom #x00)").unwrap()).is_err()); + } + + #[test] + fn chunk_kind_covers_exactly_the_nine_variants() { + for (name, kind) in [ + ( + "operation-envelope-block", + ChunkKind::OperationEnvelopeBlock, + ), + ("operation-index", ChunkKind::OperationIndex), + ("snapshot", ChunkKind::Snapshot), + ("blob", ChunkKind::Blob), + ("extension-data", ChunkKind::ExtensionData), + ("text-projection", ChunkKind::TextProjection), + ("layout-cache", ChunkKind::LayoutCache), + ("integrity-index", ChunkKind::IntegrityIndex), + ("manifest", ChunkKind::Manifest), + ] { + let sexp = read_sexp(name).unwrap(); + assert_eq!(parse_chunk_kind(&sexp).unwrap(), kind); + } + assert!(parse_chunk_kind(&read_sexp("unknown-kind").unwrap()).is_err()); + } + + // ----------------------------------------------------------------- + // The suite's own reach. + // ----------------------------------------------------------------- + + /// Distinct rejection classes this file's tests exercise. Each entry + /// corresponds to a check in `parse_document` or a leaf parser above that + /// rejects rather than normalizes; the list documents, and the assertion + /// pins, how many of them this file actually drives to a rejection. + const EXERCISED_REJECTION_CLASSES: &[&str] = &[ + "non-canonical header version", + "a (blob ...) line, wherever it appears", + "missing final U+000A", + "an extra trailing U+000A", + "missing header line", + "missing document line", + "repeated header line", + "repeated document line", + "a section line before the mandatory document line", + "a section reappearing after a later section (lineage after profile)", + "an extension before the profile section", + "a canonical-base before the extension section", + "an envelope before the canonical base", + "profile declarations out of (profile_id, version) order", + "duplicate profile declaration key", + "extension declarations out of (extension_id, version) order", + "duplicate extension declaration key", + "extension chunk roots out of projected-form order", + "duplicate extension chunk roots", + "envelope lines out of canonical reduction order", + "an unknown ProfileId symbol", + "a malformed custom ProfileId registry id", + "an unknown ChunkKind symbol", + ]; + + #[test] + fn the_suite_s_reach_is_counted() { + assert_eq!( + EXERCISED_REJECTION_CLASSES.len(), + 23, + "update this count, and add a test, whenever a new rejection class is exercised" + ); + } +} diff --git a/crates/epiphany-textproj/src/project.rs b/crates/epiphany-textproj/src/project.rs new file mode 100644 index 0000000..b95685a --- /dev/null +++ b/crates/epiphany-textproj/src/project.rs @@ -0,0 +1,1176 @@ +//! Projection of canonical bundle documents to their normative text form. +//! +//! This module implements the `Bundle -> TextDocument -> text` half of the +//! companion: [`document_from_bundle`] reads a bundle's manifest and its +//! reachable chunk/blob payloads into a [`TextDocument`], and the `project_*` +//! functions turn each grammar production of `spec/text_projection.tex`'s +//! Grammar chapter into its canonical [`Sexp`]. [`project_text_document`] +//! assembles a whole document's lines in the normative `projection` sequence, +//! and [`project_bundle`] composes both stages. +//! +//! # Grammar-directed, not value-directed +//! +//! `req:textproj:operation-vocabulary` established, for the operation layer, +//! that the grammar's productions govern rather than the mechanical +//! `TextValue` rule; the document layer is the same shape +//! (`CONTRACT_TEXTPROJ_DOCUMENT.md`). No document-line production contains a +//! `value` position, so every function here is a **free function**, never a +//! `TextValue` impl: `epiphany-bundle` does not depend on `epiphany-core`, and +//! implementing a foreign trait for a foreign type would be the orphan rule +//! violation the contract calls out. +//! +//! Envelope lines are the one exception: they are already a solved problem one +//! layer down. [`epiphany_ops::project_envelope`] projects an +//! [`OperationEnvelope`] line, and [`epiphany_ops::canonical_reduction_order`] +//! is the single function that orders the whole operation set. Both are used +//! here, not reimplemented. +//! +//! # What is deliberately absent +//! +//! Per `req:textproj:derive-or-carry`, nothing here carries a chunk's or +//! blob's `offset`, `compressed_length`, `compression`, or +//! `uncompressed_length`, nor a `ChunkId`, `ContentHash`, or `BlobId`: every +//! one is either a serializer's free physical choice or a value re-derived +//! from content. The non-canonical accelerators +//! (`operation_index_root`, `acceleration_snapshots`, `text_projection_root`, +//! `integrity_root`, `operation_block_summaries`) are never read here either. +//! A bundle that round-trips through text comes back without them; that is +//! correct; it is not data loss, because none of the five contributes to +//! canonical document semantics. + +use std::collections::BTreeSet; + +use epiphany_bundle::{ + BlobId, BlockStore, Bundle, BundleError, ChunkKind, DocumentId, LineageId, ProfileConstraints, + ProfileDeclaration, ProfileId, RetentionPolicy, SchemaVersion, SemVer, +}; +use epiphany_core::textvalue::Sexp; +use epiphany_ops::{ + canonical_reduction_order, decode_envelope, project_envelope, EnvelopeDecodeError, + OperationEnvelope, +}; + +use crate::{ + TextBlob, TextCanonicalBase, TextChunk, TextDocument, TextExtension, COMPANION_VERSION, +}; + +// =========================================================================== +// Errors. +// =========================================================================== + +/// A failure encountered projecting a bundle to its Text Projection text. +#[derive(Debug)] +pub enum ProjectError { + /// Reading a chunk, an operation-envelope block, or a blob through the + /// bundle failed. + Bundle(BundleError), + /// A stored operation-envelope block held bytes that do not decode to a + /// canonical [`OperationEnvelope`]. + Envelope(EnvelopeDecodeError), +} + +impl core::fmt::Display for ProjectError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + ProjectError::Bundle(e) => write!(f, "bundle read failed: {e}"), + ProjectError::Envelope(e) => write!(f, "operation envelope failed to decode: {e}"), + } + } +} + +impl std::error::Error for ProjectError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + ProjectError::Bundle(e) => Some(e), + ProjectError::Envelope(e) => Some(e), + } + } +} + +impl From for ProjectError { + fn from(e: BundleError) -> Self { + ProjectError::Bundle(e) + } +} + +impl From for ProjectError { + fn from(e: EnvelopeDecodeError) -> Self { + ProjectError::Envelope(e) + } +} + +// =========================================================================== +// Small shared helpers. +// =========================================================================== + +/// The grammar's `bool` terminal: `true` or `false`, spelled as a symbol. +fn project_bool(value: bool) -> Sexp { + Sexp::sym(if value { "true" } else { "false" }) +} + +/// The grammar's `option` terminal: `()` when absent, `(some )` when +/// present, projecting the payload with `project`. +fn project_option(value: Option, project: impl FnOnce(T) -> Sexp) -> Sexp { + match value { + None => Sexp::none(), + Some(v) => Sexp::some(project(v)), + } +} + +/// The grammar's shared `version` shape, `( )`: used +/// verbatim by the header (the companion version) and, restated as a +/// [`SemVer`], by profile/extension declarations and the canonical base. +fn project_version_triple(major: u32, minor: u32, patch: u32) -> Sexp { + Sexp::List(vec![Sexp::int(major), Sexp::int(minor), Sexp::int(patch)]) +} + +fn project_semver(version: &SemVer) -> Sexp { + project_version_triple(version.major, version.minor, version.patch) +} + +/// Orders and de-duplicates already-projected elements by the **UTF-8 bytes of +/// their rendered form**, ascending, keeping at most one per distinct +/// rendering (`req:textproj:derived-ordering`). Used for exactly the two +/// sequences the requirement names: the `(blob ...)` lines, and one +/// extension's preserved chunk lines. Every other projected sequence keeps +/// whatever order its caller already put it in (the manifest's binary order, +/// which for those sequences is itself a function only of preserved data). +/// +/// This is deliberately a *projection-time* operation rather than a +/// construction-time one: it must produce the correct text even when handed +/// an out-of-order or duplicated slice, which is exactly what +/// `derived_ordering_sorts_and_dedups_blobs_and_chunks` below constructs and +/// checks. +fn ordered_by_projected_form(items: &[T], project: impl Fn(&T) -> Sexp) -> Vec { + let mut rendered: Vec<(String, Sexp)> = items + .iter() + .map(|item| { + let sexp = project(item); + let text = sexp.render(); + (text, sexp) + }) + .collect(); + rendered.sort_by(|(a, _), (b, _)| a.cmp(b)); + rendered.dedup_by(|(a, _), (b, _)| a == b); + rendered.into_iter().map(|(_, sexp)| sexp).collect() +} + +// =========================================================================== +// header, document, lineage. +// =========================================================================== + +/// `header ::= "(text-projection " version ")" LF`. +/// +/// Always [`COMPANION_VERSION`]: `req:textproj:header-version` fixes the +/// header to name the one companion version this crate implements. +pub fn project_header() -> Sexp { + let (major, minor, patch) = COMPANION_VERSION; + Sexp::List(vec![ + Sexp::sym("text-projection"), + project_version_triple(major, minor, patch), + ]) +} + +/// `document ::= "(document " bytes ")" LF`. +pub fn project_document(document_id: &DocumentId) -> Sexp { + Sexp::List(vec![ + Sexp::sym("document"), + Sexp::Bytes(document_id.as_bytes().to_vec()), + ]) +} + +/// `lineage ::= "(lineage " bytes ")" LF`. +pub fn project_lineage(lineage_id: &LineageId) -> Sexp { + Sexp::List(vec![ + Sexp::sym("lineage"), + Sexp::Bytes(lineage_id.as_bytes().to_vec()), + ]) +} + +// =========================================================================== +// profile, profile-id, constraints, retention. +// =========================================================================== + +/// `profile-id ::= "full" | "read-only" | "lite" | "(custom " bytes ")"`. +/// +/// `Custom` is the one profile identity that is not a bare symbol: it carries +/// a 16-byte registry id (`req:textproj:profile-id`). +pub fn project_profile_id(profile_id: &ProfileId) -> Sexp { + match profile_id { + ProfileId::Full => Sexp::sym("full"), + ProfileId::ReadOnly => Sexp::sym("read-only"), + ProfileId::Lite => Sexp::sym("lite"), + ProfileId::Custom(registry_id) => Sexp::List(vec![ + Sexp::sym("custom"), + Sexp::Bytes(registry_id.as_bytes().to_vec()), + ]), + } +} + +/// `retention ::= "(retention " integer " " option " " bool ")"`. +/// +/// Fields, positionally: `retain_previous_manifests`, `retain_duration`, +/// `retain_named_checkpoints`. `WallClockDuration` is a newtype over `i64`, so +/// it projects transparently as the bare integer inside the option, per the +/// value-projection rule for newtypes. +pub fn project_retention(retention: &RetentionPolicy) -> Sexp { + Sexp::List(vec![ + Sexp::sym("retention"), + Sexp::int(retention.retain_previous_manifests), + project_option(retention.retain_duration, |duration| Sexp::int(duration.0)), + project_bool(retention.retain_named_checkpoints), + ]) +} + +/// `constraints ::= "(constraints " integer " " retention ")"`. +pub fn project_constraints(constraints: &ProfileConstraints) -> Sexp { + Sexp::List(vec![ + Sexp::sym("constraints"), + Sexp::int(constraints.max_uncompressed_block_size), + project_retention(&constraints.retention_policy), + ]) +} + +/// `profile ::= "(profile " profile-id " " version " " constraints ")" LF`. +pub fn project_profile(profile: &ProfileDeclaration) -> Sexp { + Sexp::List(vec![ + Sexp::sym("profile"), + project_profile_id(&profile.profile_id), + project_semver(&profile.version), + project_constraints(&profile.constraints), + ]) +} + +// =========================================================================== +// chunk, chunk-kind, schema, extension. +// =========================================================================== + +/// `chunk-kind ::= "operation-envelope-block" | "operation-index" | "snapshot" +/// | "blob" | "extension-data" | "text-projection" | "layout-cache" +/// | "integrity-index" | "manifest"`. +/// +/// Exhaustive over [`ChunkKind`]'s nine variants: adding a tenth to the bundle +/// crate is a compile error here until this match is extended, rather than a +/// silently-unprojectable kind. +pub fn project_chunk_kind(kind: ChunkKind) -> Sexp { + Sexp::sym(match kind { + ChunkKind::OperationEnvelopeBlock => "operation-envelope-block", + ChunkKind::OperationIndex => "operation-index", + ChunkKind::Snapshot => "snapshot", + ChunkKind::Blob => "blob", + ChunkKind::ExtensionData => "extension-data", + ChunkKind::TextProjection => "text-projection", + ChunkKind::LayoutCache => "layout-cache", + ChunkKind::IntegrityIndex => "integrity-index", + ChunkKind::Manifest => "manifest", + }) +} + +/// `schema ::= "(schema " integer " " integer ")"`. +pub fn project_schema(schema: &SchemaVersion) -> Sexp { + Sexp::List(vec![ + Sexp::sym("schema"), + Sexp::int(schema.major), + Sexp::int(schema.minor), + ]) +} + +/// `chunk ::= "(chunk " chunk-kind " " schema " " bytes ")"`. +/// +/// The bytes are the chunk's **uncompressed payload**, never a `ChunkRef`: +/// the projection has no file to point into +/// (`req:textproj:extension-declaration`). +pub fn project_chunk(chunk: &TextChunk) -> Sexp { + Sexp::List(vec![ + Sexp::sym("chunk"), + project_chunk_kind(chunk.kind), + project_schema(&chunk.schema_version), + Sexp::Bytes(chunk.payload.clone()), + ]) +} + +/// `extension ::= "(extension " bytes " " version " " bool " (" chunk* ") " +/// bytes " " bytes ")" LF`, fields in the ratified declaration order: id, +/// version, required, chunks, affected-kinds, barriers. +/// +/// `affected_object_kinds` and `edit_barriers` are opaque byte strings, never +/// structured sequences: the bundle preserves them without interpreting them, +/// and the projection interprets nothing the bundle does not +/// (`req:textproj:extension-declaration`). The preserved-chunk list is ordered +/// and de-duplicated by projected form (`req:textproj:derived-ordering`), +/// because `ChunkRef`'s binary order breaks ties on the physical offset. +pub fn project_extension(extension: &TextExtension) -> Sexp { + let chunks = ordered_by_projected_form(&extension.chunks, project_chunk); + Sexp::List(vec![ + Sexp::sym("extension"), + Sexp::Bytes(extension.extension_id.as_bytes().to_vec()), + project_semver(&extension.version), + project_bool(extension.required), + Sexp::List(chunks), + Sexp::Bytes(extension.affected_object_kinds.clone()), + Sexp::Bytes(extension.edit_barriers.clone()), + ]) +} + +// =========================================================================== +// canonical-base. +// =========================================================================== + +/// `canonical-base ::= "(canonical-base " bytes " " bytes " " integer " " +/// profile-id " " schema " " bytes ")" LF`: snapshot id, frontier, reduction +/// version, profile, root schema, root payload. +/// +/// The `SnapshotId` is the one identity carried verbatim rather than derived +/// (`req:textproj:derive-or-carry`); the root chunk's kind is `Snapshot` by +/// role and is not written, and its `ChunkId`/hash are re-derived from the +/// schema and payload carried here (`req:textproj:base-snapshot-inline`). +pub fn project_canonical_base(base: &TextCanonicalBase) -> Sexp { + Sexp::List(vec![ + Sexp::sym("canonical-base"), + Sexp::Bytes(base.snapshot_id.as_bytes().to_vec()), + Sexp::Bytes(base.covers_causal_frontier.as_bytes().to_vec()), + Sexp::int(base.reduction_algorithm_version.0), + project_profile_id(&base.profile_id), + project_schema(&base.root_schema_version), + Sexp::Bytes(base.root_payload.clone()), + ]) +} + +// =========================================================================== +// blob. +// =========================================================================== + +/// `blob ::= "(blob " string " " option " " bytes ")" LF`: media type, +/// declared maximum uncompressed length, payload. +/// +/// The `BlobId`, content hash, offset, lengths, and compression are all +/// re-derived or freely chosen by a serializer, never carried +/// (`req:textproj:derive-or-carry`, `req:textproj:canonical-blobs`). +pub fn project_blob(blob: &TextBlob) -> Sexp { + Sexp::List(vec![ + Sexp::sym("blob"), + Sexp::Str(blob.media_type.clone()), + project_option(blob.declared_max_uncompressed_length, Sexp::int), + Sexp::Bytes(blob.payload.clone()), + ]) +} + +// =========================================================================== +// Canonical-blob reachability (`req:textproj:canonical-blobs`). +// =========================================================================== + +/// The `BlobId`s canonically reachable from `envelopes`: referenced by a +/// canonical operation, or by canonical reduced state +/// (`req:textproj:canonical-blobs`; core specification Chapter 8 +/// §"Canonical and Non-Canonical Manifest Roots"). A blob referenced only by +/// an acceleration structure is **not** in this set and MUSTNOT be projected. +/// +/// This is a real predicate over the decoded canonical operations, not an +/// assertion of emptiness: it is written so that the day an operation payload +/// or a Chapter-5 value gains a field that names a blob, extending the walk +/// below is the one and only change needed to make that blob projectable. +/// +/// That day has not arrived. Every [`OperationPayload`](epiphany_ops::OperationPayload) +/// variant and every [`OperationKind`](epiphany_ops::OperationKind) variant is +/// exhaustively enumerated by `epiphany-ops`, and canonical reduced state +/// (`epiphany_ops::MaterializedState`) is built exclusively from those same +/// payloads — and the token `BlobId` does not occur anywhere in +/// `epiphany-core` or `epiphany-ops` (verified by +/// `blobid_is_absent_from_core_and_ops_source`, below, which fails the moment +/// it does). So there is no field, on any canonical value or payload this +/// crate can name, to extract a blob reference from — and this function's +/// result is provably the empty set today. +fn canonically_reachable_blob_ids(_envelopes: &[OperationEnvelope]) -> BTreeSet { + // Nothing to walk: see the doc comment above and the trip-wire test. + BTreeSet::new() +} + +/// The canonical blobs of a document, read through the bundle: the +/// [`TextBlob`]s of exactly the `BlobRef`s in `bundle.manifest().blob_roots` +/// whose id is canonically reachable from `envelopes`. +/// +/// **Never** `bundle.manifest().blob_roots` wholesale — that is the plausible +/// wrong answer named by the contract: one line, looks right, and emits every +/// non-canonical blob the manifest happens to carry, in direct violation of +/// `req:textproj:canonical-blobs`. Filtering through +/// [`canonically_reachable_blob_ids`] is what keeps this correct even though, +/// today, that filter is always the empty set — so this function always +/// returns `Ok(Vec::new())` for a real bundle, and does so *because* nothing +/// resolves as reachable, not because the manifest's field is skipped by +/// construction. +fn canonical_blobs( + bundle: &Bundle, + envelopes: &[OperationEnvelope], +) -> Result, ProjectError> { + let reachable = canonically_reachable_blob_ids(envelopes); + let mut blobs = Vec::new(); + for blob_ref in &bundle.manifest().blob_roots { + if reachable.contains(&blob_ref.blob_id) { + let payload = bundle.read_blob(blob_ref)?; + blobs.push(TextBlob { + media_type: blob_ref.media_type.clone(), + declared_max_uncompressed_length: blob_ref.declared_max_uncompressed_length, + payload, + }); + } + } + Ok(blobs) +} + +// =========================================================================== +// Bundle -> TextDocument. +// =========================================================================== + +/// Builds a [`TextDocument`] from a bundle's current manifest, reading every +/// chunk and blob payload the projection carries inline: each extension's +/// preserved chunks, the canonical base's root chunk, and (today, always +/// none) canonically reachable blobs. +/// +/// Operation envelopes are decoded from **every** operation-envelope block the +/// manifest references (`manifest.operation_roots`; block boundaries are a +/// storage artifact, not semantic structure) via +/// [`epiphany_ops::decode_envelope`], then put into +/// [`epiphany_ops::canonical_reduction_order`] — computed once over the whole +/// gathered set, because the order is causal and cannot be decided per block. +pub fn document_from_bundle( + bundle: &Bundle, +) -> Result { + let manifest = bundle.manifest(); + + let mut decoded = Vec::new(); + for root in &manifest.operation_roots { + for raw in bundle.read_operation_block(root)? { + decoded.push(decode_envelope(&raw)?); + } + } + let envelopes: Vec = { + let refs: Vec<&OperationEnvelope> = decoded.iter().collect(); + canonical_reduction_order(&refs) + .into_iter() + .cloned() + .collect() + }; + + let mut extensions = Vec::with_capacity(manifest.extension_declarations.len()); + for extension in &manifest.extension_declarations { + let mut chunks = Vec::with_capacity(extension.preserved_chunk_roots.len()); + for chunk_ref in &extension.preserved_chunk_roots { + let payload = bundle.read_chunk(chunk_ref)?; + chunks.push(TextChunk { + kind: chunk_ref.kind, + schema_version: chunk_ref.schema_version, + payload, + }); + } + extensions.push(TextExtension { + extension_id: extension.extension_id, + version: extension.version, + required: extension.required, + chunks, + affected_object_kinds: extension.affected_object_kinds.clone(), + edit_barriers: extension.edit_barriers.clone(), + }); + } + + let canonical_base = match &manifest.canonical_base { + Some(base) => { + let root_payload = bundle.read_chunk(&base.root)?; + Some(TextCanonicalBase { + snapshot_id: base.snapshot_id, + covers_causal_frontier: base.covers_causal_frontier.clone(), + reduction_algorithm_version: base.reduction_algorithm_version, + profile_id: base.profile_id, + root_schema_version: base.root.schema_version, + root_payload, + }) + } + None => None, + }; + + let blobs = canonical_blobs(bundle, &decoded)?; + + Ok(TextDocument { + document_id: manifest.document_id, + lineage_id: manifest.lineage_id, + // `Manifest::decode`'s own re-encode check (see `epiphany-bundle`) + // guarantees a bundle's `profile_declarations` are already the + // canonical (sorted, deduplicated) sequence; `canonical_profiles` + // names that guarantee rather than leaning on it silently. + profiles: manifest.canonical_profiles(), + extensions, + canonical_base, + blobs, + envelopes, + }) +} + +// =========================================================================== +// TextDocument -> text. +// =========================================================================== + +/// Projects a whole [`TextDocument`] to its canonical text: the header, then +/// every present section in the normative `projection` sequence (header, +/// document, lineage?, profile*, extension*, canonical-base?, blob*, +/// envelope*), one line per element, each terminated by a single LF +/// (`req:textproj:envelope-per-line`). Section order is normative and is +/// simply the order this function writes in; it introduces no ordering of its +/// own beyond `req:textproj:derived-ordering`'s blob sort. +pub fn project_text_document(document: &TextDocument) -> String { + let mut lines: Vec = Vec::new(); + lines.push(project_header()); + lines.push(project_document(&document.document_id)); + if let Some(lineage_id) = &document.lineage_id { + lines.push(project_lineage(lineage_id)); + } + lines.extend(document.profiles.iter().map(project_profile)); + lines.extend(document.extensions.iter().map(project_extension)); + if let Some(base) = &document.canonical_base { + lines.push(project_canonical_base(base)); + } + lines.extend(ordered_by_projected_form(&document.blobs, project_blob)); + + let mut out = String::new(); + for line in &lines { + out.push_str(&line.render()); + out.push('\n'); + } + for envelope in &document.envelopes { + out.push_str(&project_envelope(envelope)); + out.push('\n'); + } + out +} + +/// Composes both stages: reads `bundle` into a [`TextDocument`], then projects +/// it to its canonical text. +pub fn project_bundle(bundle: &Bundle) -> Result { + Ok(project_text_document(&document_from_bundle(bundle)?)) +} + +#[cfg(test)] +mod tests { + use std::fs; + use std::path::{Path, PathBuf}; + + use epiphany_bundle::{ + encode_block, BlobRef, CompressionAlgorithm, ExtensionDeclaration, ExtensionId, FileUuid, + FrontierBytes, Manifest, MemStore, ProfileRegistryId, ReductionAlgorithmVersion, + SnapshotId, SnapshotRef, StagedChunk, + }; + use epiphany_core::textvalue::read_sexp; + use epiphany_core::{OperationId, RegionId, ReplicaId, WallClockTime}; + use epiphany_determinism::CanonicalEncode; + use epiphany_ops::{ + AuthorId, CausalContext, DeleteRegionOp, HybridLogicalClock, OperationKind, + OperationPayload, OperationStamp, + }; + + use super::*; + + // ----------------------------------------------------------------- + // Pinned against the companion's worked example (Chapter "A Worked + // Example"). These are the strongest checks in this file: literal + // strings copied from the specification itself, not values this test + // module invented. + // ----------------------------------------------------------------- + + #[test] + fn header_matches_the_worked_example() { + assert_eq!(project_header().render(), "(text-projection (0 7 0))"); + } + + #[test] + fn document_id_matches_the_worked_example() { + let id = DocumentId([0x05; 16]); + assert_eq!( + project_document(&id).render(), + "(document #x05050505050505050505050505050505)" + ); + } + + #[test] + fn profile_matches_the_worked_example() { + assert_eq!( + project_profile(&ProfileDeclaration::full()).render(), + "(profile full (0 1 0) (constraints 67108864 (retention 1 () true)))" + ); + } + + #[test] + fn canonical_base_matches_the_worked_example() { + let mut snapshot_id_bytes = [0u8; 16]; + snapshot_id_bytes[0] = 0x1f; + snapshot_id_bytes[1] = 0x8b; + let base = TextCanonicalBase { + snapshot_id: SnapshotId(snapshot_id_bytes), + covers_causal_frontier: FrontierBytes::empty(), + reduction_algorithm_version: ReductionAlgorithmVersion(1), + profile_id: ProfileId::Full, + root_schema_version: SchemaVersion::V0, + root_payload: vec![0x00, 0x00], + }; + assert_eq!( + project_canonical_base(&base).render(), + "(canonical-base #x1f8b0000000000000000000000000000 #x 1 full (schema 0 1) #x0000)" + ); + } + + // ----------------------------------------------------------------- + // profile-id: the closed vocabulary, plus the one non-symbol case. + // ----------------------------------------------------------------- + + #[test] + fn profile_id_projects_every_closed_vocabulary_symbol() { + assert_eq!(project_profile_id(&ProfileId::Full).render(), "full"); + assert_eq!( + project_profile_id(&ProfileId::ReadOnly).render(), + "read-only" + ); + assert_eq!(project_profile_id(&ProfileId::Lite).render(), "lite"); + } + + #[test] + fn profile_id_custom_carries_its_sixteen_byte_registry_id() { + let id = ProfileId::Custom(ProfileRegistryId([0xAB; 16])); + assert_eq!( + project_profile_id(&id).render(), + "(custom #xabababababababababababababababab)" + ); + } + + // ----------------------------------------------------------------- + // chunk-kind: exhaustive over all nine variants. + // ----------------------------------------------------------------- + + #[test] + fn chunk_kind_projects_every_variant_to_its_grammar_symbol() { + let expected = [ + ( + ChunkKind::OperationEnvelopeBlock, + "operation-envelope-block", + ), + (ChunkKind::OperationIndex, "operation-index"), + (ChunkKind::Snapshot, "snapshot"), + (ChunkKind::Blob, "blob"), + (ChunkKind::ExtensionData, "extension-data"), + (ChunkKind::TextProjection, "text-projection"), + (ChunkKind::LayoutCache, "layout-cache"), + (ChunkKind::IntegrityIndex, "integrity-index"), + (ChunkKind::Manifest, "manifest"), + ]; + for (kind, symbol) in expected { + assert_eq!(project_chunk_kind(kind).render(), symbol); + } + } + + #[test] + fn schema_projects_major_then_minor() { + assert_eq!( + project_schema(&SchemaVersion::new(2, 9)).render(), + "(schema 2 9)" + ); + } + + // ----------------------------------------------------------------- + // extension: six fields in the ratified order, opaque byte strings for + // affected-kinds/barriers (the `Vec`-is-not-a-sequence trap). + // ----------------------------------------------------------------- + + #[test] + fn extension_projects_six_fields_in_declaration_order() { + let extension = TextExtension { + extension_id: ExtensionId([9; 16]), + version: SemVer::new(1, 2, 3), + required: true, + chunks: vec![TextChunk { + kind: ChunkKind::ExtensionData, + schema_version: SchemaVersion::V0, + payload: vec![0xAA], + }], + affected_object_kinds: vec![0x01, 0x02], + edit_barriers: vec![0x03], + }; + assert_eq!( + project_extension(&extension).render(), + "(extension #x09090909090909090909090909090909 (1 2 3) true \ + ((chunk extension-data (schema 0 1) #xaa)) #x0102 #x03)" + ); + } + + #[test] + fn extension_affected_kinds_and_barriers_are_byte_strings_not_sequences() { + let extension = TextExtension { + extension_id: ExtensionId([0; 16]), + version: SemVer::new(0, 0, 0), + required: false, + chunks: Vec::new(), + affected_object_kinds: vec![1, 2, 3], + edit_barriers: vec![4, 5], + }; + let Sexp::List(fields) = project_extension(&extension) else { + panic!("an extension projection is a list"); + }; + // Positions: [0]=`extension`, [1]=id, [2]=version, [3]=required, + // [4]=chunks, [5]=affected-kinds, [6]=barriers. + assert_eq!(fields[5], Sexp::Bytes(vec![1, 2, 3])); + assert_eq!(fields[6], Sexp::Bytes(vec![4, 5])); + // Never the `Vec`-is-a-sequence spelling (a list of integers). + assert_ne!( + fields[5], + Sexp::List(vec![Sexp::int(1), Sexp::int(2), Sexp::int(3)]) + ); + } + + // ----------------------------------------------------------------- + // blob: line-level project, tested against synthetic data (per the + // contract: the emit side must be ready even though no real bundle can + // populate it today). + // ----------------------------------------------------------------- + + #[test] + fn blob_projects_media_type_declared_max_and_payload() { + let blob = TextBlob { + media_type: "audio/wav".to_owned(), + declared_max_uncompressed_length: Some(1 << 20), + payload: vec![0xDE, 0xAD], + }; + assert_eq!( + project_blob(&blob).render(), + "(blob \"audio/wav\" (some 1048576) #xdead)" + ); + } + + #[test] + fn blob_with_no_declared_maximum_projects_an_absent_option() { + let blob = TextBlob { + media_type: "application/octet-stream".to_owned(), + declared_max_uncompressed_length: None, + payload: Vec::new(), + }; + assert_eq!( + project_blob(&blob).render(), + "(blob \"application/octet-stream\" () #x)" + ); + } + + /// The `(blob ...)` production round-trips through the s-expression + /// reader (the lexical layer parse.rs will build its semantic parser on). + /// This is not a substitute for parse.rs's own `parse_blob` and the + /// parse-side rejection the contract requires of it; it proves the emit + /// side is well-formed and ready. + #[test] + fn blob_projection_reads_back_through_the_sexp_reader() { + let blob = TextBlob { + media_type: "text/plain".to_owned(), + declared_max_uncompressed_length: Some(4), + payload: vec![1, 2, 3, 4], + }; + let rendered = project_blob(&blob).render(); + let read_back = read_sexp(&rendered).expect("projected blob line is valid s-expression"); + assert_eq!(read_back, project_blob(&blob), "reading back is idempotent"); + } + + // ----------------------------------------------------------------- + // `req:textproj:derived-ordering`: sort AND de-duplicate by rendered + // form, exercised against out-of-order, duplicated, in-memory data -- + // never against a fixture that was already sorted. + // ----------------------------------------------------------------- + + #[test] + fn derived_ordering_sorts_and_dedups_blob_lines() { + let a = TextBlob { + media_type: "a/a".to_owned(), + declared_max_uncompressed_length: None, + payload: vec![1], + }; + let b = TextBlob { + media_type: "b/b".to_owned(), + declared_max_uncompressed_length: None, + payload: vec![2], + }; + let a_duplicate = TextBlob { + media_type: "a/a".to_owned(), + declared_max_uncompressed_length: None, + payload: vec![1], + }; + // Deliberately out of order (b before a) and with a duplicate of a. + let blobs = vec![b, a_duplicate, a]; + let ordered = ordered_by_projected_form(&blobs, project_blob); + let rendered: Vec = ordered.iter().map(Sexp::render).collect(); + assert_eq!( + rendered, + vec![ + "(blob \"a/a\" () #x01)".to_owned(), + "(blob \"b/b\" () #x02)".to_owned(), + ], + "duplicate collapsed to one line, and the surviving two lines ordered ascending \ + by rendered form" + ); + } + + #[test] + fn derived_ordering_sorts_and_dedups_an_extensions_chunk_lines() { + let x = TextChunk { + kind: ChunkKind::ExtensionData, + schema_version: SchemaVersion::V0, + payload: vec![0xAA], + }; + let y = TextChunk { + kind: ChunkKind::LayoutCache, + schema_version: SchemaVersion::V0, + payload: vec![0xBB], + }; + let x_duplicate = TextChunk { + kind: ChunkKind::ExtensionData, + schema_version: SchemaVersion::V0, + payload: vec![0xAA], + }; + let extension = TextExtension { + extension_id: ExtensionId([1; 16]), + version: SemVer::new(1, 0, 0), + required: false, + // Deliberately out of order (y before x) and with a duplicate of x. + chunks: vec![y, x_duplicate, x], + affected_object_kinds: Vec::new(), + edit_barriers: Vec::new(), + }; + let Sexp::List(fields) = project_extension(&extension) else { + panic!("an extension projection is a list"); + }; + let Sexp::List(chunks) = &fields[4] else { + panic!("the chunk field is a list"); + }; + assert_eq!( + chunks.len(), + 2, + "the duplicate chunk collapsed to a single line" + ); + assert_eq!( + chunks[0].render(), + "(chunk extension-data (schema 0 1) #xaa)" + ); + assert_eq!(chunks[1].render(), "(chunk layout-cache (schema 0 1) #xbb)"); + } + + // ----------------------------------------------------------------- + // The blob trap: a bundle whose manifest carries a real, non-empty + // `blob_roots` must still project zero `(blob ...)` lines, because no + // canonical operation or reduced state can reach a `BlobId` today. + // ----------------------------------------------------------------- + + /// Builds a bundle exercising every optional section at once: a lineage, + /// one extension with one preserved chunk, a canonical base, one + /// (deliberately unreachable) blob root, and two operation-envelope + /// blocks staged so their envelopes are **not** already in canonical + /// reduction order (physical time descending), across **two** separate + /// blocks so `document_from_bundle` is proven to gather across all of + /// `operation_roots`, not just the first. + fn build_sample_bundle() -> Bundle { + let env_late = sample_envelope(1, 200); // higher physical time + let env_early = sample_envelope(2, 100); // lower physical time + + // Block 0 holds the later envelope, block 1 the earlier one: storage + // order disagrees with canonical reduction order on purpose. + let op_block_0 = + StagedChunk::operation_block(encode_block(&[env_late.to_canonical_bytes()])); + let op_block_1 = + StagedChunk::operation_block(encode_block(&[env_early.to_canonical_bytes()])); + + let extension_chunk = StagedChunk { + kind: ChunkKind::ExtensionData, + schema_version: SchemaVersion::V0, + payload: b"extension-chunk-payload".to_vec(), + }; + + let base_chunk = StagedChunk { + kind: ChunkKind::Snapshot, + schema_version: SchemaVersion::V0, + payload: b"canonical-base-payload".to_vec(), + }; + + let blob_chunk = StagedChunk { + kind: ChunkKind::Blob, + schema_version: SchemaVersion::V0, + payload: b"unreachable-blob-payload".to_vec(), + }; + + let mut initial = Manifest::empty(DocumentId([3; 16])); + initial.lineage_id = Some(LineageId([4; 16])); + let mut bundle = Bundle::create(MemStore::new(), FileUuid([1; 16]), initial) + .expect("a freshly created bundle with no canonical roots is valid"); + + bundle + .commit( + &[ + op_block_0, + op_block_1, + extension_chunk, + base_chunk, + blob_chunk, + ], + |ctx| { + let mut manifest = ctx.previous_manifest.clone(); + manifest.operation_roots = vec![ctx.new_chunks[0], ctx.new_chunks[1]]; + manifest.extension_declarations = vec![ExtensionDeclaration { + extension_id: ExtensionId([9; 16]), + version: SemVer::new(1, 0, 0), + required: false, + preserved_chunk_roots: vec![ctx.new_chunks[2]], + affected_object_kinds: vec![0xAA], + edit_barriers: vec![0xBB, 0xCC], + }]; + manifest.canonical_base = Some(SnapshotRef { + snapshot_id: SnapshotId([7; 16]), + covers_causal_frontier: FrontierBytes::from_bytes(vec![1, 2, 3]), + reduction_algorithm_version: ReductionAlgorithmVersion(1), + profile_id: ProfileId::Full, + root: ctx.new_chunks[3], + hash: ctx.new_chunks[3].hash, + }); + manifest.blob_roots = vec![BlobRef { + blob_id: BlobId(ctx.new_chunks[4].hash), + media_type: "application/octet-stream".to_owned(), + offset: ctx.new_chunks[4].offset, + compressed_length: ctx.new_chunks[4].compressed_length, + uncompressed_length: ctx.new_chunks[4].uncompressed_length, + compression: CompressionAlgorithm::None, + hash: ctx.new_chunks[4].hash, + declared_max_uncompressed_length: None, + }]; + manifest + }, + ) + .expect("commit of a well-formed, self-consistent manifest succeeds"); + + // Sanity: the trap this test exists to guard against is only live if + // the manifest really does carry a non-empty, non-canonical blob root. + assert_eq!(bundle.manifest().blob_roots.len(), 1); + bundle + } + + fn sample_envelope(counter: u64, physical_time: i64) -> OperationEnvelope { + let id = OperationId::new(ReplicaId(1), counter); + OperationEnvelope { + id, + author: AuthorId(0xAB), + stamp: OperationStamp::new( + HybridLogicalClock::new(WallClockTime(physical_time), 0), + id, + ), + causal_context: CausalContext::new(), + transaction: None, + payload: OperationPayload::Primitive(OperationKind::DeleteRegion(DeleteRegionOp { + region: RegionId::new(ReplicaId(1), counter), + })), + } + } + + #[test] + fn document_from_bundle_reads_every_section_and_orders_envelopes_canonically() { + let bundle = build_sample_bundle(); + let document = document_from_bundle(&bundle).expect("bundle reads cleanly"); + + assert_eq!(document.document_id, DocumentId([3; 16])); + assert_eq!(document.lineage_id, Some(LineageId([4; 16]))); + assert_eq!(document.profiles, vec![ProfileDeclaration::full()]); + + assert_eq!(document.extensions.len(), 1); + let extension = &document.extensions[0]; + assert_eq!(extension.extension_id, ExtensionId([9; 16])); + assert_eq!(extension.chunks.len(), 1); + assert_eq!(extension.chunks[0].kind, ChunkKind::ExtensionData); + assert_eq!(extension.chunks[0].payload, b"extension-chunk-payload"); + + let base = document.canonical_base.as_ref().expect("base was staged"); + assert_eq!(base.snapshot_id, SnapshotId([7; 16])); + assert_eq!(base.root_payload, b"canonical-base-payload"); + + // The trap: a non-empty `blob_roots` still yields zero canonical + // blobs, because nothing can reach one yet. + assert!( + document.blobs.is_empty(), + "no blob is canonically reachable today, regardless of manifest.blob_roots" + ); + + // Canonical reduction order is by ascending physical time; the raw + // blocks stored the higher-physical-time envelope first. + assert_eq!(document.envelopes.len(), 2); + assert_eq!(document.envelopes[0].id, OperationId::new(ReplicaId(1), 2)); + assert_eq!(document.envelopes[1].id, OperationId::new(ReplicaId(1), 1)); + } + + #[test] + fn projected_text_never_emits_a_blob_line_and_orders_sections_correctly() { + let bundle = build_sample_bundle(); + let document = document_from_bundle(&bundle).expect("bundle reads cleanly"); + let text = project_text_document(&document); + + assert!( + !text.lines().any(|line| line.starts_with("(blob ")), + "projected text must never carry a blob line at this companion version:\n{text}" + ); + + // Section order is normative (`projection ::= header document lineage? + // profile* extension* canonical-base? blob* envelope*`): assert the + // literal sequence of line-heads, not just a count, so a section + // written out of order fails this check rather than only a count. + let lines: Vec<&str> = text.lines().collect(); + let heads: Vec<&str> = lines + .iter() + .map(|line| line.split(' ').next().expect("every line has a head token")) + .collect(); + assert_eq!( + heads, + vec![ + "(text-projection", + "(document", + "(lineage", + "(profile", + "(extension", + "(canonical-base", + "(envelope", + "(envelope", + ], + "section order is normative; got:\n{text}" + ); + } + + #[test] + fn project_bundle_composes_both_stages() { + let bundle = build_sample_bundle(); + let via_two_stages = + project_text_document(&document_from_bundle(&bundle).expect("bundle reads cleanly")); + let via_one_call = project_bundle(&bundle).expect("bundle reads cleanly"); + assert_eq!(via_one_call, via_two_stages); + } + + #[test] + fn a_corrupt_operation_envelope_is_a_typed_error_not_a_panic() { + let mut initial = Manifest::empty(DocumentId([5; 16])); + initial.lineage_id = None; + let mut bundle = Bundle::create(MemStore::new(), FileUuid([2; 16]), initial).unwrap(); + let garbage_block = StagedChunk::operation_block(encode_block(&[vec![0xFF; 4]])); + bundle + .commit(&[garbage_block], |ctx| { + let mut manifest = ctx.previous_manifest.clone(); + manifest.operation_roots = vec![ctx.new_chunks[0]]; + manifest + }) + .expect("the block's framing is well-formed even though its one envelope is not"); + + match document_from_bundle(&bundle) { + Err(ProjectError::Envelope(_)) => {} + other => panic!("expected a decode error for garbage envelope bytes, got {other:?}"), + } + } + + #[test] + fn a_corrupt_chunk_hash_is_a_bundle_error() { + let bundle = build_sample_bundle(); + let extension_root = bundle.manifest().extension_declarations[0].preserved_chunk_roots[0]; + let mut image = bundle.image().to_vec(); + // Flip one byte inside the extension chunk's stored payload region. + let corrupt_at = extension_root.offset as usize; + image[corrupt_at] ^= 0xFF; + + let corrupted = Bundle::open(MemStore::from_bytes(image)) + .expect("corrupting a non-canonical chunk's payload does not stop the bundle opening"); + match document_from_bundle(&corrupted) { + Err(ProjectError::Bundle(_)) => {} + other => panic!("expected a bundle error for a hash-mismatched chunk, got {other:?}"), + } + } + + // ----------------------------------------------------------------- + // Suite reach: count, don't just claim, coverage of an extension, a + // canonical base, and a multi-envelope document. + // ----------------------------------------------------------------- + + #[test] + fn test_suite_reach_covers_extension_base_and_multi_envelope_documents() { + let rich = document_from_bundle(&build_sample_bundle()).expect("bundle reads cleanly"); + let minimal = TextDocument { + document_id: DocumentId([6; 16]), + lineage_id: None, + profiles: vec![ProfileDeclaration::full()], + extensions: Vec::new(), + canonical_base: None, + blobs: Vec::new(), + envelopes: vec![sample_envelope(1, 1)], + }; + let documents = [rich, minimal]; + + let with_extension = documents + .iter() + .filter(|d| !d.extensions.is_empty()) + .count(); + let with_canonical_base = documents + .iter() + .filter(|d| d.canonical_base.is_some()) + .count(); + let with_multi_envelope = documents.iter().filter(|d| d.envelopes.len() > 1).count(); + + assert_eq!( + with_extension, 1, + "expected exactly one exercised document with an extension" + ); + assert_eq!( + with_canonical_base, 1, + "expected exactly one exercised document with a canonical base" + ); + assert_eq!( + with_multi_envelope, 1, + "expected exactly one exercised document with more than one envelope" + ); + } + + // ----------------------------------------------------------------- + // The trip-wire: if `BlobId` ever appears in `epiphany-core` or + // `epiphany-ops`, `canonically_reachable_blob_ids` above is no longer + // provably empty, and both the emit side (that function) and the + // parse-side `(blob ...)` rejection (parse.rs) need real + // implementations. + // ----------------------------------------------------------------- + + fn rs_files_under(dir: &Path, out: &mut Vec) { + let entries = fs::read_dir(dir) + .unwrap_or_else(|e| panic!("failed to read directory {}: {e}", dir.display())); + for entry in entries { + let path = entry.expect("directory entry is readable").path(); + if path.is_dir() { + rs_files_under(&path, out); + } else if path.extension().is_some_and(|ext| ext == "rs") { + out.push(path); + } + } + } + + #[test] + fn blobid_is_absent_from_core_and_ops_source() { + let dirs = [ + concat!(env!("CARGO_MANIFEST_DIR"), "/../epiphany-core/src"), + concat!(env!("CARGO_MANIFEST_DIR"), "/../epiphany-ops/src"), + ]; + for dir in dirs { + let mut files = Vec::new(); + rs_files_under(Path::new(dir), &mut files); + assert!( + !files.is_empty(), + "expected to find .rs sources under {dir}" + ); + for path in files { + let source = fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display())); + assert!( + !source.contains("BlobId"), + "found `BlobId` in {}: `canonically_reachable_blob_ids` \ + (crates/epiphany-textproj/src/project.rs) could provably return only the \ + empty set because neither crate could name a BlobId -- that is no longer \ + true, so both that predicate's emit side and parse.rs's `(blob ...)` \ + rejection now need their real implementations", + path.display() + ); + } + } + } +} diff --git a/crates/epiphany-textproj/src/serialize.rs b/crates/epiphany-textproj/src/serialize.rs new file mode 100644 index 0000000..f34b391 --- /dev/null +++ b/crates/epiphany-textproj/src/serialize.rs @@ -0,0 +1,561 @@ +//! Serialization of parsed Text Projection documents into canonical bundles. +//! +//! [`serialize_document`] stages the payloads a [`TextDocument`] carries inline +//! — the canonical base's root chunk, every extension's preserved chunks, and +//! one operation-envelope block — and lets [`Bundle::create`] / +//! [`Bundle::commit`] do everything `req:textproj:derive-or-carry` says a +//! serializer must not: assign offsets, content-address every chunk, and +//! de-duplicate identical content. **Nothing here computes a `ChunkId`, a +//! `ContentHash`, or an offset** — every physical field written into the +//! manifest comes straight from the [`ChunkRef`](epiphany_bundle::ChunkRef)s +//! `commit` hands back. +//! +//! # Block splitting is a free physical choice +//! +//! `req:textproj:roundtrip` only requires the *bundle's* physical layout to +//! round-trip in the second, byte-checkable equation (`project(serialize(parse(T))) +//! == T`, quantified over texts, not bundles); how a serializer packs envelopes +//! into blocks is unconstrained. This module always emits **exactly one** +//! operation-envelope block, containing every envelope the document carries (in +//! the order the document carries them) — the simplest possible choice, and +//! sufficient because block boundaries are storage artifacts, never semantic +//! structure (Chapter 8: *"the set of envelopes is the union of all envelopes +//! across all referenced blocks"*). +//! +//! # No accelerator is (re)written +//! +//! A [`TextDocument`] carries no `operation_index_root`, `acceleration_snapshots`, +//! `text_projection_root`, `integrity_root`, or `operation_block_summaries` — +//! they are non-canonical and the companion text does not carry them +//! (`req:textproj:derive-or-carry`). The manifest this module builds leaves every +//! one of those fields at its empty/`None` default. A bundle serialized from a +//! `TextDocument` therefore comes back from this module *without* any +//! accelerator a previous generation might have had. That reads as data loss; it +//! is not — none of those fields contributes to canonical document semantics, +//! and a consumer that wants them back rebuilds them the same way any bundle +//! writer does (e.g. `epiphany_bundle::fuzz` and the testkit's operation-index +//! harness both scan-and-rebuild rather than trust a carried-forward index). +//! +//! # No blob is staged +//! +//! [`TextDocument::blobs`] is never staged into `manifest.blob_roots` here. Today +//! that vector is always empty: no operation payload or Chapter-5 value in +//! `epiphany-core`/`epiphany-ops` carries a `BlobId`, so no blob is reachable from +//! canonical state (`req:textproj:canonical-blobs`), and this companion's own +//! parser rejects every `(blob ...)` line rather than populate the vector (see +//! `parse`). [`serialize_document`] still checks: a `TextDocument` assembled +//! directly against the type (bypassing `parse`) with a populated `blobs` vector +//! is refused with [`SerializeError::NonEmptyBlobs`] rather than silently +//! dropped — wiring a blob into the manifest here would require a canonical +//! `declared_max_uncompressed_length`/compression policy this companion version +//! does not specify, and silently discarding the caller's bytes is worse. + +use std::fmt; + +use epiphany_bundle::{ + encode_block, BlockStore, Bundle, BundleError, ChunkKind, CommitContext, ExtensionDeclaration, + FileUuid, Manifest, SchemaVersion, SnapshotRef, StagedChunk, +}; +use epiphany_determinism::CanonicalEncode; +use epiphany_ops::OperationEnvelope; + +use crate::TextDocument; + +/// An error [`serialize_document`] cannot recover from. +#[derive(Debug)] +pub enum SerializeError { + /// The document's `blobs` vector is non-empty. See the module documentation: + /// no blob is canonical today, so a document built by this companion's + /// `parse` never carries one, and staging one here would either wire a + /// non-canonical root into the manifest or silently drop the caller's + /// payload bytes. Neither is acceptable, so serialization refuses instead. + NonEmptyBlobs, + /// The bundle itself rejected the manifest or a staged chunk — e.g. the + /// document declares no profile this implementation understands, or a + /// staged root fails `Bundle::commit`'s structural validation. + Bundle(BundleError), +} + +impl fmt::Display for SerializeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + SerializeError::NonEmptyBlobs => f.write_str( + "the document carries a populated blobs vector, but no blob can be canonical today", + ), + SerializeError::Bundle(error) => { + write!(f, "the bundle rejected the serialized document: {error}") + } + } + } +} + +impl std::error::Error for SerializeError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + SerializeError::Bundle(error) => Some(error), + SerializeError::NonEmptyBlobs => None, + } + } +} + +impl From for SerializeError { + fn from(error: BundleError) -> Self { + SerializeError::Bundle(error) + } +} + +/// Serializes a [`TextDocument`] into a freshly created bundle over `store`. +/// +/// Two-phase, because [`Bundle::create`] requires a manifest with no canonical +/// roots or blobs (there is nothing to reference before any chunk is written): +/// this creates an empty generation-0 bundle carrying only the document's +/// identity and profile declarations, then stages every payload the document +/// carries inline and commits once, building the real manifest from the +/// [`ChunkRef`](epiphany_bundle::ChunkRef)s that commit assigns. +/// +/// Returns [`SerializeError::NonEmptyBlobs`] if `document.blobs` is non-empty +/// (see the module documentation), or [`SerializeError::Bundle`] if the bundle +/// itself refuses the manifest or a staged root. +pub fn serialize_document( + document: &TextDocument, + store: S, + file_uuid: FileUuid, +) -> Result, SerializeError> { + if !document.blobs.is_empty() { + return Err(SerializeError::NonEmptyBlobs); + } + + let mut bundle = Bundle::create(store, file_uuid, empty_manifest(document))?; + + let mut staged = Vec::new(); + if let Some(base) = &document.canonical_base { + // The canonical base's root chunk: kind and schema version are carried + // verbatim from the document (they are not derivable from anything), + // and its payload is staged exactly as the document carries it. `commit` + // computes the chunk's id, hash, and offset. + staged.push(StagedChunk { + kind: ChunkKind::Snapshot, + schema_version: base.root_schema_version, + payload: base.root_payload.clone(), + }); + } + for extension in &document.extensions { + for chunk in &extension.chunks { + staged.push(StagedChunk { + kind: chunk.kind, + schema_version: chunk.schema_version, + payload: chunk.payload.clone(), + }); + } + } + staged.push(stage_operation_envelope_block(document)); + + bundle.commit(&staged, |ctx| build_manifest(document, ctx))?; + Ok(bundle) +} + +/// The manifest [`Bundle::create`] is given: the document's identity and +/// declared profiles, and nothing else. `create` itself rejects a manifest +/// carrying canonical roots or blobs, so every root is added by the subsequent +/// commit (see [`build_manifest`]). If `document.profiles` is empty or +/// otherwise unemittable, `Bundle::create` reports that itself — this function +/// does not invent a fallback profile the document did not declare. +fn empty_manifest(document: &TextDocument) -> Manifest { + let mut manifest = Manifest::empty(document.document_id); + manifest.lineage_id = document.lineage_id; + manifest.profile_declarations = document.profiles.clone(); + manifest +} + +/// Encodes every envelope the document carries, in the order it carries them, +/// into a single operation-envelope block payload (see the module +/// documentation on why one block is the right choice here). The block's +/// schema version is the max over its envelopes' `schema_major` — never a fixed +/// baseline — so a block that carries a higher-major payload (e.g. a v1 +/// `CreateRegion` or a v2 cross-cutting value) is never mis-stamped major 0, +/// mirroring `StagedChunk::operation_block_versioned`'s own contract. +fn stage_operation_envelope_block(document: &TextDocument) -> StagedChunk { + let payloads: Vec> = document + .envelopes + .iter() + .map(CanonicalEncode::to_canonical_bytes) + .collect(); + let major = document + .envelopes + .iter() + .map(OperationEnvelope::schema_major) + .max() + .unwrap_or(0); + StagedChunk::operation_block_versioned(encode_block(&payloads), SchemaVersion::for_major(major)) +} + +/// Builds the committed manifest from the previous (empty) manifest and the +/// [`ChunkRef`](epiphany_bundle::ChunkRef)s [`Bundle::commit`] assigned to the +/// chunks [`serialize_document`] staged, in the same order it staged them: the +/// canonical base's root (if any), then each extension's preserved chunks in +/// turn, then the single operation-envelope block. Every physical field +/// (`ChunkId`, `ContentHash`, offset) comes from `ctx.new_chunks` — nothing here +/// recomputes one (`req:textproj:derive-or-carry`). `operation_index_root`, +/// `acceleration_snapshots`, `text_projection_root`, `integrity_root`, +/// `operation_block_summaries`, and `blob_roots` are left untouched at the +/// previous manifest's empty defaults; see the module documentation for why +/// that is correct rather than lossy. +fn build_manifest(document: &TextDocument, ctx: &CommitContext) -> Manifest { + let mut manifest = ctx.previous_manifest.clone(); + let mut cursor = 0usize; + + if let Some(base) = &document.canonical_base { + let root = ctx.new_chunks[cursor]; + cursor += 1; + manifest.canonical_base = Some(SnapshotRef { + snapshot_id: base.snapshot_id, + covers_causal_frontier: base.covers_causal_frontier.clone(), + reduction_algorithm_version: base.reduction_algorithm_version, + profile_id: base.profile_id, + root, + hash: root.hash, + }); + } + + let mut extension_declarations = Vec::with_capacity(document.extensions.len()); + for extension in &document.extensions { + let mut preserved_chunk_roots = Vec::with_capacity(extension.chunks.len()); + for _ in &extension.chunks { + preserved_chunk_roots.push(ctx.new_chunks[cursor]); + cursor += 1; + } + extension_declarations.push(ExtensionDeclaration { + extension_id: extension.extension_id, + version: extension.version, + required: extension.required, + preserved_chunk_roots, + affected_object_kinds: extension.affected_object_kinds.clone(), + edit_barriers: extension.edit_barriers.clone(), + }); + } + manifest.extension_declarations = extension_declarations; + + manifest.operation_roots = vec![ctx.new_chunks[cursor]]; + cursor += 1; + debug_assert_eq!( + cursor, + ctx.new_chunks.len(), + "every chunk staged by serialize_document must be wired into the manifest exactly once" + ); + manifest +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{TextCanonicalBase, TextChunk, TextExtension}; + use epiphany_bundle::{ + DocumentId, ExtensionId, FrontierBytes, LineageId, MemStore, ProfileConstraints, + ProfileDeclaration, ProfileId, ReductionAlgorithmVersion, SemVer, SnapshotId, + }; + use epiphany_determinism::fuzz::SplitMix64; + use epiphany_ops::{decode_envelope, fuzz::gen_envelope_set}; + + /// A real, varied envelope set (every operation kind the generator reaches), + /// so the schema-major derivation and the envelope round trip both have + /// something to bite on. + fn envelopes(seed: u64, n: usize) -> Vec { + let mut rng = SplitMix64::new(seed); + gen_envelope_set(&mut rng, n) + } + + /// A profile declaration distinguishable from `ProfileDeclaration::full()`'s + /// default version, so a test that forgets to carry `document.profiles` + /// through cannot pass by accident (`full()` would still be a `Full` + /// profile, just at a different version). + fn base_profile() -> ProfileDeclaration { + ProfileDeclaration { + profile_id: ProfileId::Full, + version: SemVer::new(0, 2, 0), + constraints: ProfileConstraints::DEFAULT_FULL, + } + } + + fn minimal_document(seed: u64) -> TextDocument { + TextDocument { + document_id: DocumentId([seed as u8; 16]), + lineage_id: None, + profiles: vec![base_profile()], + extensions: Vec::new(), + canonical_base: None, + blobs: Vec::new(), + envelopes: envelopes(seed, 5), + } + } + + fn document_with_extension(seed: u64) -> TextDocument { + let mut document = minimal_document(seed); + document.extensions.push(TextExtension { + extension_id: ExtensionId([7; 16]), + version: SemVer::new(1, 0, 0), + required: false, + chunks: vec![ + TextChunk { + kind: ChunkKind::ExtensionData, + schema_version: SchemaVersion::V0, + payload: b"chunk-a".to_vec(), + }, + TextChunk { + kind: ChunkKind::ExtensionData, + schema_version: SchemaVersion::V0, + payload: b"chunk-b".to_vec(), + }, + ], + affected_object_kinds: vec![0xAA], + edit_barriers: vec![0xBB, 0xCC], + }); + document + } + + fn document_with_canonical_base(seed: u64) -> TextDocument { + let mut document = minimal_document(seed); + document.canonical_base = Some(TextCanonicalBase { + snapshot_id: SnapshotId([9; 16]), + covers_causal_frontier: FrontierBytes(vec![1, 2, 3]), + reduction_algorithm_version: ReductionAlgorithmVersion(1), + profile_id: ProfileId::Full, + root_schema_version: SchemaVersion::V0, + root_payload: b"snapshot-root".to_vec(), + }); + document + } + + /// Carries an extension, a canonical base, and many envelopes at once — the + /// document every round-trip law needs to be checked against together. + fn rich_document(seed: u64) -> TextDocument { + let mut document = document_with_extension(seed); + document.canonical_base = document_with_canonical_base(seed).canonical_base; + document.envelopes = envelopes(seed, 60); + document + } + + fn serialize_and_reopen(document: &TextDocument) -> Bundle { + let bundle = serialize_document(document, MemStore::new(), FileUuid([1; 16])) + .expect("a well-formed document serializes"); + let image = bundle.into_store().into_bytes(); + Bundle::open(MemStore::from_bytes(image)).expect("the serialized bundle reopens") + } + + #[test] + fn document_identity_round_trips() { + let mut document = minimal_document(1); + document.lineage_id = Some(LineageId([2; 16])); + let reopened = serialize_and_reopen(&document); + assert_eq!(reopened.manifest().document_id, document.document_id); + assert_eq!(reopened.manifest().lineage_id, document.lineage_id); + } + + #[test] + fn profile_declarations_round_trip() { + let document = minimal_document(2); + let reopened = serialize_and_reopen(&document); + assert_eq!(reopened.manifest().profile_declarations, document.profiles); + } + + #[test] + fn missing_profile_declaration_surfaces_the_bundles_own_rejection() { + // serialize_document must not invent a fallback profile the document + // did not declare: an empty `profiles` propagates Bundle::create's own + // "no declared profile" rejection. + let mut document = minimal_document(3); + document.profiles.clear(); + let result = serialize_document(&document, MemStore::new(), FileUuid([1; 16])); + assert!(matches!(result, Err(SerializeError::Bundle(_)))); + } + + #[test] + fn canonical_base_round_trips_snapshot_id_and_payload() { + let document = document_with_canonical_base(4); + let reopened = serialize_and_reopen(&document); + let expected = document + .canonical_base + .as_ref() + .expect("fixture carries a canonical base"); + let base = reopened + .manifest() + .canonical_base + .as_ref() + .expect("canonical base present after reopen"); + assert_eq!(base.snapshot_id, expected.snapshot_id); + assert_eq!(base.covers_causal_frontier, expected.covers_causal_frontier); + assert_eq!( + base.reduction_algorithm_version, + expected.reduction_algorithm_version + ); + assert_eq!(base.profile_id, expected.profile_id); + assert_eq!(base.root.kind, ChunkKind::Snapshot); + assert_eq!(base.root.schema_version, expected.root_schema_version); + let payload = reopened + .read_chunk(&base.root) + .expect("root chunk reads and verifies"); + assert_eq!(payload, expected.root_payload); + } + + #[test] + fn extension_round_trips_fields_and_chunk_payloads() { + let document = document_with_extension(5); + let reopened = serialize_and_reopen(&document); + let expected = &document.extensions[0]; + let declaration = reopened + .manifest() + .extension_declarations + .iter() + .find(|d| d.extension_id == expected.extension_id) + .expect("extension declaration present after reopen"); + assert_eq!(declaration.version, expected.version); + assert_eq!(declaration.required, expected.required); + assert_eq!( + declaration.affected_object_kinds, + expected.affected_object_kinds + ); + assert_eq!(declaration.edit_barriers, expected.edit_barriers); + assert_eq!( + declaration.preserved_chunk_roots.len(), + expected.chunks.len() + ); + // The manifest's own canonical encoding sorts preserved_chunk_roots by + // ChunkRef (kind, hash, offset), not by the document's chunk order, so + // compare payload sets rather than assume position survives. + let mut payloads: Vec> = declaration + .preserved_chunk_roots + .iter() + .map(|root| reopened.read_chunk(root).expect("preserved chunk reads")) + .collect(); + let mut expected_payloads: Vec> = + expected.chunks.iter().map(|c| c.payload.clone()).collect(); + payloads.sort(); + expected_payloads.sort(); + assert_eq!( + payloads, expected_payloads, + "preserved chunk payloads survive, order aside" + ); + } + + #[test] + fn envelopes_round_trip_through_a_single_operation_block() { + let document = minimal_document(6); + assert!(document.envelopes.len() > 1, "fixture reach check"); + let reopened = serialize_and_reopen(&document); + assert_eq!( + reopened.manifest().operation_roots.len(), + 1, + "one operation-envelope block, by design" + ); + let root = reopened.manifest().operation_roots[0]; + let payloads = reopened + .read_operation_block(&root) + .expect("operation block reads"); + let recovered: Vec = payloads + .iter() + .map(|bytes| decode_envelope(bytes).expect("canonical envelope decodes")) + .collect(); + assert_eq!(recovered, document.envelopes); + } + + #[test] + fn operation_block_schema_major_is_derived_not_hardcoded() { + let document = rich_document(7); + let expected_major = document + .envelopes + .iter() + .map(OperationEnvelope::schema_major) + .max() + .unwrap_or(0); + assert!( + expected_major > 0, + "fixture must include a schema-major-bearing operation to exercise the derivation" + ); + let reopened = serialize_and_reopen(&document); + let root = reopened.manifest().operation_roots[0]; + assert_eq!( + root.schema_version, + SchemaVersion::for_major(expected_major) + ); + } + + #[test] + fn a_document_with_no_envelopes_still_stages_one_empty_operation_block() { + let mut document = minimal_document(8); + document.envelopes.clear(); + let reopened = serialize_and_reopen(&document); + assert_eq!(reopened.manifest().operation_roots.len(), 1); + let root = reopened.manifest().operation_roots[0]; + assert!(reopened + .read_operation_block(&root) + .expect("empty operation block still reads") + .is_empty()); + } + + #[test] + fn nonempty_blobs_are_rejected_rather_than_dropped() { + let mut document = minimal_document(9); + document.blobs.push(crate::TextBlob { + media_type: "audio/wav".to_string(), + declared_max_uncompressed_length: None, + payload: b"nope".to_vec(), + }); + let result = serialize_document(&document, MemStore::new(), FileUuid([1; 16])); + assert!(matches!(result, Err(SerializeError::NonEmptyBlobs))); + } + + #[test] + fn a_rich_document_round_trips_every_root_at_once() { + let document = rich_document(10); + let reopened = serialize_and_reopen(&document); + assert_eq!(reopened.manifest().document_id, document.document_id); + assert!(reopened.manifest().canonical_base.is_some()); + assert_eq!(reopened.manifest().extension_declarations.len(), 1); + assert_eq!(reopened.manifest().operation_roots.len(), 1); + reopened + .verify_canonical_chunks() + .expect("every canonical chunk this module wrote is intact"); + assert!(!reopened.is_read_only()); + assert!(reopened.anomalies().is_empty()); + } + + /// `req:textproj` verification discipline item 3: a round-trip suite that + /// never exercises an extension, a canonical base, or a multi-envelope + /// document proves far less than its green tick suggests. Count what this + /// suite actually covers. + #[test] + fn the_suite_exercises_an_extension_a_canonical_base_and_multiple_envelopes() { + let documents = vec![ + minimal_document(20), + document_with_extension(21), + document_with_canonical_base(22), + rich_document(23), + ]; + let with_extension = documents + .iter() + .filter(|d| !d.extensions.is_empty()) + .count(); + let with_canonical_base = documents + .iter() + .filter(|d| d.canonical_base.is_some()) + .count(); + let with_multiple_envelopes = documents.iter().filter(|d| d.envelopes.len() > 1).count(); + + assert!( + with_extension >= 1, + "reach: no document carries an extension" + ); + assert!( + with_canonical_base >= 1, + "reach: no document carries a canonical base" + ); + assert!( + with_multiple_envelopes >= 1, + "reach: no document carries more than one envelope" + ); + + for document in &documents { + let reopened = serialize_and_reopen(document); + assert_eq!(reopened.manifest().document_id, document.document_id); + } + } +} diff --git a/crates/epiphany-textproj/tests/companion_version.rs b/crates/epiphany-textproj/tests/companion_version.rs new file mode 100644 index 0000000..4dbe14c --- /dev/null +++ b/crates/epiphany-textproj/tests/companion_version.rs @@ -0,0 +1,59 @@ +use epiphany_textproj::COMPANION_VERSION; + +fn title_page_version(source: &str) -> Result<(u32, u32, u32), String> { + let title_page = source + .split_once("\\begin{titlepage}") + .and_then(|(_, after_start)| after_start.split_once("\\end{titlepage}")) + .map(|(title_page, _)| title_page) + .ok_or_else(|| "companion must contain one complete titlepage environment".to_owned())?; + + let candidates: Vec<_> = title_page + .lines() + .filter_map(|line| line.split_once("Version ").map(|(_, suffix)| suffix)) + .collect(); + if candidates.len() != 1 { + return Err(format!( + "title page must contain exactly one `Version ..` anchor; found {}", + candidates.len() + )); + } + + let token: String = candidates[0] + .chars() + .take_while(|character| character.is_ascii_digit() || *character == '.') + .collect(); + let components: Vec<_> = token.split('.').collect(); + if components.len() != 3 || components.iter().any(|component| component.is_empty()) { + return Err(format!( + "title-page version must have exactly three numeric components; found `{token}`" + )); + } + + let parse_component = |component: &str| { + component + .parse::() + .map_err(|_| format!("invalid numeric component `{component}` in title-page version")) + }; + Ok(( + parse_component(components[0])?, + parse_component(components[1])?, + parse_component(components[2])?, + )) +} + +#[test] +fn companion_version_matches_title_page() { + let companion_path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../spec/text_projection.tex" + ); + let companion = std::fs::read_to_string(companion_path) + .unwrap_or_else(|error| panic!("failed to read {companion_path}: {error}")); + let title_version = title_page_version(&companion) + .unwrap_or_else(|error| panic!("failed to parse {companion_path}: {error}")); + + assert_eq!( + title_version, COMPANION_VERSION, + "title-page companion version differs from epiphany_textproj::COMPANION_VERSION" + ); +} diff --git a/spec/CONTRACT_TEXTPROJ_DOCUMENT.md b/spec/CONTRACT_TEXTPROJ_DOCUMENT.md new file mode 100644 index 0000000..cd8855d --- /dev/null +++ b/spec/CONTRACT_TEXTPROJ_DOCUMENT.md @@ -0,0 +1,155 @@ +# Contract: the Text Projection document layer + +Repo root `/home/jeans/Repos/active/epiphany`. Read this in full before writing a +line. The plan is `spec/PLAN_TEXTPROJ_DOCUMENT.md`; both its rulings are granted +and this contract states them as law. + +The two layers below you are done and are your model: `epiphany-core`'s +`textvalue*.rs` (Chapter-5 values) and `epiphany-ops`'s `textproj_*.rs` +(Chapter-6 operations). Read the latter — the document layer is the same shape. + +## Grammar-directed, and free functions + +`req:textproj:operation-vocabulary` established that the grammar's productions +govern, not the mechanical value rule. The document lines are the same: find your +production in `spec/text_projection.tex`'s Grammar chapter and implement exactly +what it says. + +**Do not implement `TextValue` for bundle types.** `epiphany-bundle` does not +depend on `epiphany-core`, so the trait is foreign and so is the type — the +orphan rule forbids it from `epiphany-textproj`. You do not need it: no +document-line production contains a `value` position; every one bottoms out in +`bytes`, `integer`, `bool`, `string`, `option`, or a closed vocabulary. Write +**free functions** — `fn project_profile(&ProfileDeclaration) -> Sexp`, +`fn parse_profile(&Sexp) -> Result`. Use +`epiphany_core::textvalue::{Sexp, read_sexp, TextError}` for the machinery. + +Do **not** add `epiphany-core` to `epiphany-bundle`'s dependencies. The bundle is +a container format and its independence from the music model is deliberate. + +## Blobs: emit none, and reject them on parse + +A blob is canonical **iff** referenced by a canonical operation or by canonical +reduced state (`req:textproj:canonical-blobs`; `core_spec` §"Canonical and +Non-Canonical Manifest Roots"). Nothing in `epiphany-core` or `epiphany-ops` +references a `BlobId` — verified by source scan — so **no blob is canonical +today** and the correct projection of every real bundle emits zero `(blob ...)` +lines. + +Three obligations, and they are not the obvious ones: + +1. **Emit side.** Implement the reachability predicate as a real function that + today provably returns the empty set. **Never project + `manifest.blob_roots`** — that is the plausible wrong answer: one line, looks + right, emits non-canonical blobs in violation of the requirement. Even + `manifest.rs`'s own doc comment says "canonical `blob_roots`" — the subset, not + the field. + +2. **Parse side: reject.** Document-level validation **MUST reject any + `(blob ...)` line** as unreferenced, with a test asserting the rejection. + A blob-bearing text at this companion version is necessarily non-canonical. + Accepting one would stage a blob into the bundle that the next projection + silently drops, which loses data *and* falsifies + `project(serialize(parse(T))) == T` for that text. Forward compatibility is + owned by header-version gating, not by leniency here. + + Line-level `project`/`parse` of the `(blob ...)` production must still be + written and unit-tested in **both** directions against synthetic data, so that + when the predicate becomes non-empty both sides are ready at once. + +3. **A trip-wire.** A test that source-scans `crates/epiphany-core/src` and + `crates/epiphany-ops/src` for the token `BlobId` and fails if it appears — same + family as the existing drift locks. Its failure message must name the + reachability predicate and tell the finder that both the emit side and the + parse-side rejection now need their real implementation. + +## The header carries exactly one version + +The header names the version of the companion the text conforms to. The parser +**accepts exactly one**: the companion version this crate implements. Anything +else is a rejection at line one. Multi-version acceptance and migrate-on-read for +text are future spec decisions — do not improvise them. + +Define the version once, as a constant, and lock it against the companion with a +test that reads `spec/text_projection.tex` and asserts the title version matches. +A constant that silently disagrees with the document it claims conformance to is +the whole failure mode this project keeps paying for. + +## Section order is normative + +`projection ::= header document lineage? profile* extension* canonical-base? +blob* envelope*` is a **sequence**, not a set. A repeated or out-of-order section +is a rejection. A parser that accepts lines in any order and sorts them is +normalizing, which `req:textproj:strict-parse` forbids. + +## What must not appear (`req:textproj:derive-or-carry`) + +* **Physical attributes** — `offset`, `compressed_length`, `compression`, + `uncompressed_length`. A serializer chooses them freely. +* **Derivable identities** — `ChunkId`, `ContentHash`, `BlobId`. Re-derived from + content. +* The **one** identity carried verbatim is `SnapshotId`, which has nothing to + derive from. +* **Non-canonical accelerators are not projected**: `operation_index_root`, + `acceleration_snapshots`, `text_projection_root`, `integrity_root`, + `operation_block_summaries`. A bundle that round-trips through text comes back + without them. That is correct; say so in a doc comment, because it reads as data + loss and is not. + +## `derived-ordering` sorts by projected form + +`req:textproj:derived-ordering` applies to exactly two sequences: the `(blob ...)` +lines and an extension's preserved chunk roots. Order **and de-duplicate** them by +the **UTF-8 bytes of their rendered form** — not by any binary key, because the +binary key reads the offset, and a chunk's file position must not decide the +text's order. Every other sequence keeps the binary order. + +## Traps + +* **The extension line has six fields** in ratified declaration order: id, + version, required, chunks, affected-kinds, barriers. `affected_object_kinds` + and `edit_barriers` are **opaque byte strings**, never structured — the + `Vec`-is-not-a-sequence trap that bit the operation layer lives here too. + Each preserved chunk root projects as kind + schema_version + **uncompressed + payload**, never as a `ChunkRef`. +* **`ProfileId::Custom`** carries a 16-byte registry id and is the one profile + that is not a bare symbol. +* **Two `WallClockDuration` types exist**, one in `epiphany-core` and one in + `epiphany-bundle`. The retention policy uses the bundle's. +* **`canonical-base` carries the root chunk's uncompressed payload inline**, as a + single byte string, plus the `SnapshotId` verbatim. +* **Envelopes are emitted in `canonical_reduction_order`** + (`epiphany_ops::canonical_reduction_order`, public and exported). + +## Style + +Match the surrounding code. Doc comments on every public item and every +non-obvious decision — especially *why* a parse rejects rather than normalizes. +`rustfmt` clean, no `clippy` warnings, no `#[allow(...)]`, no `unwrap`/`expect` on +a parse path. Touch only the file you are told to create; do not run +`cargo fmt --all`. + +## Verification + +The standing gate: `cargo fmt --all --check`; `cargo clippy --workspace +--all-targets` → 0; full workspace tests; `RUSTDOCFLAGS="-D warnings" cargo doc +--workspace --no-deps` → 0; `cargo run -q -p epiphany-testkit --example +conformance_suite`; zero golden churn. + +Plus the three this project learned the hard way: + +1. **Mutation-verify every check you write.** Assert the anchor is present, delete + or invert the check, confirm a *named* test of yours fails, restore. Report per + check, with the actual command output. A survivor is a finding — say so. +2. **Exercise every outbound normalization.** For every value you normalize on the + way out, construct a non-normalized input and prove the normalization happens. + `derived-ordering` is exactly this shape: build a document with duplicate blobs + and out-of-order chunk roots *in memory* and prove the projection collapses and + orders them. Five such normalizations shipped untested in the operation layer + because every fixture was already sorted. +3. **Assert your suite's own reach.** A round-trip suite that never exercises an + extension, a canonical base, or a multi-envelope document proves far less than + its green tick suggests. Count what you covered and assert the counts. + +Report the actual commands and their actual output. A previous agent reported +"verification passes" when errors did point into its own file. diff --git a/spec/PLAN_TEXTPROJ_DOCUMENT.md b/spec/PLAN_TEXTPROJ_DOCUMENT.md new file mode 100644 index 0000000..4873678 --- /dev/null +++ b/spec/PLAN_TEXTPROJ_DOCUMENT.md @@ -0,0 +1,217 @@ +# Text Projection — the document layer: scope and plan + +Status: **scoping complete, two rulings needed before dispatch.** +Prepared against `master` @ `fdb4a57`. Every claim was checked against the code; +where I ran a probe I say so. + +This is the last phase. Chapter-5 values (`cf81074`) and Chapter-6 operations +(`fdb4a57`) already project and parse strictly. What remains is the document +around them: the header, identity, profile, extension, canonical-base and blob +lines — and then the two round-trip equations over a whole bundle. + +--- + +## 1. Decisions needed + +### Ruling A — no blob can be canonical today. What should the projector do? + +`req:textproj:canonical-blobs` and `core_spec` §"Canonical and Non-Canonical +Manifest Roots" agree exactly: a blob is canonical **iff** it is *referenced by a +canonical operation or by canonical reduced state*. A blob referenced only by +acceleration structures is non-canonical and `MUSTNOT` be projected. Canonicality +is by **reachability**, not by membership in `blob_roots`. + +I grepped `epiphany-core` and `epiphany-ops` for `BlobId`: **no hit.** No +operation payload and no Chapter-5 value carries a blob reference. There is no +mechanism by which a blob can be reached from canonical state, so **today the +correct projection of every real bundle emits zero `(blob ...)` lines**, and the +production is future-proofing. + +The dangerous wrong answer is projecting `manifest.blob_roots` wholesale — it +looks obviously right, is one line of code, and violates the requirement by +emitting non-canonical blobs. + +**Recommendation.** Implement the reachability predicate as a real function that +today provably returns the empty set, and *prove* it rather than asserting it: a +test that fails the moment any core or ops type gains a `BlobId` field, so the +hook is implemented when it is first needed rather than silently skipped. Same +discipline as computing a bound instead of spelling it. The `(blob ...)` +projection and parse must still be written and tested against a synthetic +document, because the parser must accept texts that a future writer emits. + +### Ruling B — the header version is the companion version, and that couples every byte + +The companion says the header names *"the version of **this companion** the text +conforms to."* So a projection written today starts `(text-projection (0 6 0))`. + +Two consequences worth accepting deliberately: + +* The companion's own worked example currently reads `(text-projection (0 3 0))` + — **stale**. My conformance lock covers the envelope line of that example, not + the header above it. +* Every companion version bump changes the first line of every projection, so + `req:textproj:canonical-text` is *version-relative* and the text vector corpus + must be regenerated on each bump. + +**Recommendation.** Keep the coupling — a text that does not say which rules it +follows cannot be strictly parsed against them — fix the stale example, and make +the corpus regeneration a documented, one-command step so the cost is mechanical +rather than a surprise. If instead you want the header to track a slower-moving +*format* version independent of editorial revisions, that is a spec change and +should be decided now, not after a corpus exists. + +--- + +## 2. What I verified + +**A new crate is unavoidable, and the orphan rule does not bite.** +`epiphany-bundle` depends only on `epiphany-determinism` — **not** on +`epiphany-core`. So `TextValue` (a core trait) cannot be implemented for +`Manifest`, `BlobRef`, `ProfileDeclaration` and friends from any third crate. + +That would have been a real problem, except: **no document-line production +contains a `value` position.** Every one bottoms out in leaves — `bytes`, +`integer`, `bool`, `string`, `option` — plus the closed vocabularies +`profile-id`, `chunk-kind`, `schema`. So the document layer needs **free +functions**, not trait impls, exactly as the grammar-directed operation layer +did. No orphan problem, no new dependency edge, and `epiphany-bundle` stays +independent of the music model as designed. + +`epiphany-textproj` depends on core (for `Sexp`, `render`, `read_sexp`), ops (for +`project_envelope`/`parse_envelope` and `canonical_reduction_order`), and bundle. + +**Parse cannot produce a `Manifest`.** The text carries **payloads inline** — the +canonical base's root chunk payload, each extension's preserved chunk payloads, +each blob's payload — whereas a `Manifest` holds `ChunkRef`s with offsets and +lengths that the text deliberately erases. The parse output needs its own type +(§4), and the bundle writer synthesizes the physical layer. + +**The write path already does the synthesis.** `Bundle::create` + `commit` +assigns offsets, content-addresses and de-duplicates chunks, and derives every +`ChunkId`/`ContentHash`. That is precisely what `req:textproj:derive-or-carry` +assumes, so serialization stages payloads and lets `commit` do the rest. + +**Envelope ordering is available.** `epiphany_ops::canonical_reduction_order` is +public and exported; `Bundle::read_operation_block` yields raw envelope bytes for +`decode_envelope`. + +**Accelerators are dropped, by design.** `operation_index_root`, +`acceleration_snapshots`, `text_projection_root`, `integrity_root` and +`operation_block_summaries` are non-canonical and are not projected. A bundle +that goes text → binary comes back without them. That is correct and should be +stated in a doc comment, because it looks like data loss and is not. + +--- + +## 3. The round trip, stated precisely + +`req:textproj:roundtrip` gives two equations. **Both hold; neither says bytes +survive.** + +``` + semantics(parse(project(B))) = semantics(B) -- weaker + project(serialize(parse(T))) = T -- byte-checkable, the corpus test +``` + +Going bundle → text → bundle is **deliberately lossy in the physical layer** and +in three specific ways an implementer will otherwise read as bugs: + +1. **Duplicate blobs collapse.** `req:textproj:derived-ordering` de-duplicates + blob lines and extension chunk roots by *projected form*. Two byte-identical + blobs stored twice are one blob; that is the erasure working. +2. **Layout is regenerated.** Offsets, compression, block splitting, chunk count. +3. **Accelerators vanish**, as above. + +All three preserve `semantics(...)`, and none of them affects the second +equation, which quantifies over *texts*, not bundles. Write the corpus against +the second equation. + +--- + +## 4. Work breakdown + +New crate `crates/epiphany-textproj`. Three waves; file boundaries are +one-per-agent. + +### Wave 1 — scaffold and types (blocks everything; small) + +`Cargo.toml`, `lib.rs`, and the parse-output type. Roughly: + +```rust +pub struct TextDocument { + pub document_id: DocumentId, + pub lineage_id: Option, + pub profiles: Vec, + pub extensions: Vec, // chunk payloads INLINE + pub canonical_base: Option, // root payload INLINE + pub blobs: Vec, // payloads INLINE + pub envelopes: Vec, // canonical operation order +} +``` + +The `Text*` types exist because the corresponding bundle types carry `ChunkRef`s +the text erases. Do not try to reuse `SnapshotRef`/`ExtensionDeclaration` +verbatim. + +### Wave 2 — three parallel agents + +* **`project.rs`** — `Bundle` → `TextDocument` → text. Reads chunk and blob + payloads through the bundle, decodes envelopes, sorts by + `canonical_reduction_order`, and applies `derived-ordering`'s sort-and-dedup by + projected form to blob lines and extension chunk roots. +* **`parse.rs`** — text → `TextDocument`, strictly. Line-oriented: each line is + one complete s-expression (`req:textproj:envelope-per-line`), and the line + *order* is itself constrained by the `projection` production — header, document, + lineage?, profile*, extension*, canonical-base?, blob*, envelope*. A + out-of-order or repeated section is a rejection, not a reordering. +* **`serialize.rs`** — `TextDocument` → `Bundle`. Stages the base root chunk, + extension chunks, blobs and one operation-envelope block; builds the manifest; + commits. Block splitting is a free physical choice — one block is fine and + simplest. + +### Wave 3 — conformance + +Text vectors in the shape of `spec/vectors/decode_vectors.txt`, drift-locked and +gated as conformance step **`[7e]`**, plus the two round-trip equations over the +generated-score corpus. This is also where the stale `(0 3 0)` header in the +companion's example gets fixed and locked. + +--- + +## 5. Traps + +* **`(blob ...)` is unreachable from real data.** Its tests need a synthetic + document. See Ruling A. +* **Section order is normative.** The `projection` production is a sequence, not + a set. A parser that accepts lines in any order and sorts them is normalizing. +* **`derived-ordering` sorts by *projected form*, not by any binary key** — order + and de-duplicate on the rendered UTF-8 bytes of each line, which is the whole + point (a chunk's offset must not decide the text's order). +* **The extension line is easy to get subtly wrong.** Six fields in ratified + declaration order — id, version, required, chunks, affected-kinds, barriers — + and `affected_object_kinds`/`edit_barriers` are opaque **byte strings**, never + structured. The `Vec`-is-not-a-sequence trap that bit the ops layer lives + here too. +* **`ProfileId::Custom` carries a 16-byte registry id** and is the one profile + that is not a bare symbol. +* **Do not project `manifest.blob_roots`.** See Ruling A; it is the plausible + wrong answer. +* **Two `WallClockDuration` types exist** — one in `epiphany-core`, one in + `epiphany-bundle`. The retention policy uses the bundle's. + +--- + +## 6. Verification contract + +The standing gate, plus: + +* **Mutation-verify every check**, anchor asserted before substitution, and report + per-check with output. +* **Exercise every outbound normalization** — the contract rule added after five + such normalizations shipped untested in the ops layer. `derived-ordering`'s + sort-and-dedup is exactly this shape: build a document with duplicate blobs and + out-of-order chunk roots *in memory* and prove the projection collapses and + orders them. +* **Assert the corpus's own reach.** A round-trip suite that never exercises an + extension, a canonical base, or a multi-envelope document proves less than its + green tick suggests. State the counts.