1087 lines
40 KiB
Rust
1087 lines
40 KiB
Rust
//! Whole-document Text Projection conformance vectors.
|
||
//!
|
||
//! The committed corpus is generated here, inside `epiphany-textproj`, because
|
||
//! malformed documents must be constructed without going through the public
|
||
//! projection API that canonicalizes outbound data. Each row carries a complete
|
||
//! UTF-8 document and a normative accept/reject verdict.
|
||
//!
|
||
//! The accept contract is the byte-checkable, text-quantified equation
|
||
//!
|
||
//! `project(serialize(parse(T))) == T`.
|
||
//!
|
||
//! It is deliberately not bundle identity. Bundle -> text -> bundle collapses
|
||
//! duplicate blobs, regenerates physical layout (offsets, compression, and
|
||
//! chunking), and drops non-canonical accelerators. Those erasures preserve
|
||
//! semantics but make a bundle-byte comparison both false and misleading.
|
||
|
||
use std::collections::BTreeMap;
|
||
|
||
use epiphany_bundle::{
|
||
ChunkKind, DocumentId, ExtensionId, FileUuid, FrontierBytes, LineageId, MemStore,
|
||
ProfileConstraints, ProfileDeclaration, ProfileId, ProfileRegistryId,
|
||
ReductionAlgorithmVersion, SchemaVersion, SemVer, SnapshotId,
|
||
};
|
||
use epiphany_core::{
|
||
AnalysisLayerId, MeasureId, OperationId, PartDefinitionId, RegionId, ReplicaId, StaffGroupId,
|
||
StaffId, StaffInstanceId, TimeSignatureId, ViewId, WallClockTime,
|
||
};
|
||
use epiphany_ops::{
|
||
AuthorId, CausalContext, CreateAnalysisLayerOp, CreateMeasureOp, CreatePartDefinitionOp,
|
||
CreateStaffGroupOp, CreateViewOp, DeleteRegionOp, HybridLogicalClock, OperationEnvelope,
|
||
OperationKind, OperationPayload, OperationStamp, SetTuningContextOp,
|
||
};
|
||
|
||
use crate::parse::parse_document;
|
||
use crate::project::{project_bundle, project_text_document, render_text_document};
|
||
use crate::serialize::serialize_document;
|
||
use crate::{TextCanonicalBase, TextChunk, TextDocument, TextExtension};
|
||
|
||
/// The corpus surface owned by this crate.
|
||
pub const SURFACE: &str = "textproj.document";
|
||
/// The committed whole-document corpus.
|
||
pub const COMMITTED: &str = include_str!("../../../spec/vectors/textproj_document_vectors.txt");
|
||
/// The path regenerated by the vector generator, relative to the workspace root.
|
||
pub const PATH: &str = "spec/vectors/textproj_document_vectors.txt";
|
||
|
||
const HEADER: &str = "\
|
||
# Epiphany Text Projection document vectors — format version 1
|
||
#
|
||
# Generated. Regenerate with:
|
||
# cargo run -q -p epiphany-testkit --example generate_vectors
|
||
# `epiphany_textproj::vectors::the_committed_corpus_matches_the_generator` fails
|
||
# on drift, so a projection change must land here deliberately.
|
||
#
|
||
# One vector per line, space-separated:
|
||
#
|
||
# <surface> <verdict> <class> <name> <utf8-hex>
|
||
#
|
||
# NORMATIVE: Text Projection companion, `req:textproj:roundtrip` and
|
||
# `req:textproj:strict-parse`. Every `accept` text must satisfy the exact-byte
|
||
# equation project(serialize(parse(T))) == T. Every `reject` text must be
|
||
# refused, not accepted and normalized to a different canonical spelling.
|
||
#
|
||
# The equation quantifies over TEXTS, not bundle bytes. Binary layout is freely
|
||
# regenerated, duplicate blobs collapse, and non-canonical accelerators vanish;
|
||
# all three preserve semantics and none weakens this text identity check.
|
||
#
|
||
# `class` names the rejection exercised. It is informative; the verdict and
|
||
# document bytes are normative. `<utf8-hex>` is lowercase with no separators.
|
||
";
|
||
|
||
/// One generated vector: `(surface, verdict, class, name, UTF-8 bytes)`.
|
||
pub type TextVector = (
|
||
&'static str,
|
||
&'static str,
|
||
&'static str,
|
||
&'static str,
|
||
Vec<u8>,
|
||
);
|
||
|
||
/// The typed outcome of checking one whole document.
|
||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
pub enum CheckVerdict {
|
||
/// The parser refused the text.
|
||
Rejected(String),
|
||
/// The parser accepted it and the byte-checkable equation held exactly.
|
||
AcceptedCanonical,
|
||
/// The parser accepted it, but serialization/projection either failed or
|
||
/// produced different bytes. This is acceptance, never rejection.
|
||
AcceptedNonCanonical(String),
|
||
}
|
||
|
||
/// One parsed row from the committed text file.
|
||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
pub struct Row {
|
||
/// Owning conformance surface.
|
||
pub surface: String,
|
||
/// Normative `accept` or `reject` verdict.
|
||
pub verdict: String,
|
||
/// Informative rejection class, or `-` for accepts.
|
||
pub class: String,
|
||
/// Stable vector name.
|
||
pub name: String,
|
||
/// Complete UTF-8 document bytes.
|
||
pub text: Vec<u8>,
|
||
}
|
||
|
||
/// Coverage counts asserted by the corpus gate.
|
||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
pub struct ReachCounts {
|
||
/// Number of accepted documents containing at least one extension.
|
||
pub extensions: usize,
|
||
/// Number containing a canonical base.
|
||
pub canonical_bases: usize,
|
||
/// Number declaring a custom profile.
|
||
pub custom_profiles: usize,
|
||
/// Number carrying a lineage id.
|
||
pub lineages: usize,
|
||
/// Number carrying more than one envelope.
|
||
pub multi_envelope: usize,
|
||
/// Actually rejected vectors per declared rejection class.
|
||
pub reject_classes: BTreeMap<String, usize>,
|
||
}
|
||
|
||
/// The exact non-vacuity contract of this corpus: four accepted documents, two
|
||
/// reaching each optional/rich feature, and one actually rejected vector for
|
||
/// each of the ten distinct rejection classes implemented by this layer.
|
||
///
|
||
/// `canonical_bases` is pinned at **0**, not two:
|
||
/// `CONTRACT_FORMAT_EPOCH_MAJOR1.md` pin 3b refuses projecting, parsing, and
|
||
/// serializing any base-bearing document, so a canonical base is no longer
|
||
/// reachable through text at all. The two formerly base-bearing accepts
|
||
/// (`extension_base_two_envelopes`, `rich_document`) keep every other feature
|
||
/// they exercised but lose their base, and the base-bearing spelling survives
|
||
/// only as the new `canonical_base_present` reject vector (class
|
||
/// `canonical-base-unsupported`).
|
||
pub fn expected_reach() -> ReachCounts {
|
||
ReachCounts {
|
||
extensions: 2,
|
||
canonical_bases: 0,
|
||
custom_profiles: 2,
|
||
lineages: 2,
|
||
multi_envelope: 2,
|
||
reject_classes: [
|
||
("blob-line", 1),
|
||
("canonical-base-unsupported", 1),
|
||
("extension-chunk-order", 1),
|
||
("extension-declaration-order", 1),
|
||
("missing-trailing-lf", 1),
|
||
("operation-envelope-order", 1),
|
||
("out-of-order-sections", 1),
|
||
("profile-declaration-order", 1),
|
||
("repeated-singular-section", 1),
|
||
("wrong-header-version", 1),
|
||
]
|
||
.into_iter()
|
||
.map(|(class, count)| (class.to_owned(), count))
|
||
.collect(),
|
||
}
|
||
}
|
||
|
||
fn sample_envelope(counter: u64, physical_time: i64) -> OperationEnvelope {
|
||
let id = OperationId::new(ReplicaId(1), counter);
|
||
OperationEnvelope {
|
||
id,
|
||
author: AuthorId(0xAB),
|
||
stamp: OperationStamp::new(HybridLogicalClock::new(WallClockTime(physical_time), 0), id),
|
||
causal_context: CausalContext::new(),
|
||
transaction: None,
|
||
payload: OperationPayload::Primitive(OperationKind::DeleteRegion(DeleteRegionOp {
|
||
region: RegionId::new(ReplicaId(1), counter),
|
||
})),
|
||
}
|
||
}
|
||
|
||
/// A `SetTuningContext` envelope (kind 34, genesis G2b). The committed corpus
|
||
/// must contain a real `set-tuning-context` document: the typed all-kind
|
||
/// round-trip test proves the *production* parses, but only a committed vector
|
||
/// proves the emitted document text is stable across implementations, which is
|
||
/// what this corpus exists for.
|
||
fn tuning_context_envelope(counter: u64, physical_time: i64) -> OperationEnvelope {
|
||
let id = OperationId::new(ReplicaId(1), counter);
|
||
OperationEnvelope {
|
||
id,
|
||
author: AuthorId(0xAB),
|
||
stamp: OperationStamp::new(HybridLogicalClock::new(WallClockTime(physical_time), 0), id),
|
||
causal_context: CausalContext::new(),
|
||
transaction: None,
|
||
payload: OperationPayload::Primitive(OperationKind::SetTuningContext(SetTuningContextOp {
|
||
settings: epiphany_ops::valuegen::tuning_context_settings(7),
|
||
})),
|
||
}
|
||
}
|
||
|
||
/// The four genesis tranche G3a envelopes (kinds 35–38,
|
||
/// `spec/CONTRACT_GENESIS_G3A_ENTITIES.md`). Same rationale as
|
||
/// `tuning_context_envelope` above: the typed all-kind round-trip test proves
|
||
/// the *production* parses, but only a committed vector proves the emitted
|
||
/// document text is stable across implementations.
|
||
fn staff_group_envelope(counter: u64, physical_time: i64) -> OperationEnvelope {
|
||
let id = OperationId::new(ReplicaId(1), counter);
|
||
OperationEnvelope {
|
||
id,
|
||
author: AuthorId(0xAB),
|
||
stamp: OperationStamp::new(HybridLogicalClock::new(WallClockTime(physical_time), 0), id),
|
||
causal_context: CausalContext::new(),
|
||
transaction: None,
|
||
payload: OperationPayload::Primitive(OperationKind::CreateStaffGroup(CreateStaffGroupOp {
|
||
group: epiphany_ops::valuegen::staff_group(
|
||
StaffGroupId::new(ReplicaId(1), 1),
|
||
vec![StaffId::new(ReplicaId(1), 1)],
|
||
),
|
||
})),
|
||
}
|
||
}
|
||
|
||
fn part_definition_envelope(counter: u64, physical_time: i64) -> OperationEnvelope {
|
||
let id = OperationId::new(ReplicaId(1), counter);
|
||
OperationEnvelope {
|
||
id,
|
||
author: AuthorId(0xAB),
|
||
stamp: OperationStamp::new(HybridLogicalClock::new(WallClockTime(physical_time), 0), id),
|
||
causal_context: CausalContext::new(),
|
||
transaction: None,
|
||
payload: OperationPayload::Primitive(OperationKind::CreatePartDefinition(
|
||
CreatePartDefinitionOp {
|
||
part: epiphany_ops::valuegen::part_definition(
|
||
PartDefinitionId::new(ReplicaId(1), 1),
|
||
vec![StaffId::new(ReplicaId(1), 1)],
|
||
),
|
||
},
|
||
)),
|
||
}
|
||
}
|
||
|
||
fn analysis_layer_envelope(counter: u64, physical_time: i64) -> OperationEnvelope {
|
||
let id = OperationId::new(ReplicaId(1), counter);
|
||
OperationEnvelope {
|
||
id,
|
||
author: AuthorId(0xAB),
|
||
stamp: OperationStamp::new(HybridLogicalClock::new(WallClockTime(physical_time), 0), id),
|
||
causal_context: CausalContext::new(),
|
||
transaction: None,
|
||
payload: OperationPayload::Primitive(OperationKind::CreateAnalysisLayer(
|
||
CreateAnalysisLayerOp {
|
||
layer: epiphany_ops::valuegen::analysis_layer(AnalysisLayerId::new(
|
||
ReplicaId(1),
|
||
1,
|
||
)),
|
||
},
|
||
)),
|
||
}
|
||
}
|
||
|
||
fn view_envelope(counter: u64, physical_time: i64) -> OperationEnvelope {
|
||
let id = OperationId::new(ReplicaId(1), counter);
|
||
OperationEnvelope {
|
||
id,
|
||
author: AuthorId(0xAB),
|
||
stamp: OperationStamp::new(HybridLogicalClock::new(WallClockTime(physical_time), 0), id),
|
||
causal_context: CausalContext::new(),
|
||
transaction: None,
|
||
payload: OperationPayload::Primitive(OperationKind::CreateView(CreateViewOp {
|
||
view: epiphany_ops::valuegen::view(
|
||
ViewId::new(ReplicaId(1), 1),
|
||
vec![AnalysisLayerId::new(ReplicaId(1), 1)],
|
||
),
|
||
})),
|
||
}
|
||
}
|
||
|
||
/// Genesis tranche G3b (kind 39, `spec/CONTRACT_GENESIS_G3B_MEASURE.md`).
|
||
/// Same rationale as `staff_group_envelope` above.
|
||
fn measure_envelope(counter: u64, physical_time: i64) -> OperationEnvelope {
|
||
let id = OperationId::new(ReplicaId(1), counter);
|
||
OperationEnvelope {
|
||
id,
|
||
author: AuthorId(0xAB),
|
||
stamp: OperationStamp::new(HybridLogicalClock::new(WallClockTime(physical_time), 0), id),
|
||
causal_context: CausalContext::new(),
|
||
transaction: None,
|
||
payload: OperationPayload::Primitive(OperationKind::CreateMeasure(CreateMeasureOp {
|
||
instance: StaffInstanceId::new(ReplicaId(1), 1),
|
||
measure: epiphany_ops::valuegen::measure(
|
||
MeasureId::new(ReplicaId(1), 1),
|
||
TimeSignatureId::new(ReplicaId(1), 1),
|
||
1,
|
||
),
|
||
})),
|
||
}
|
||
}
|
||
|
||
fn profiles(custom: bool) -> Vec<ProfileDeclaration> {
|
||
let mut profiles = vec![ProfileDeclaration::full()];
|
||
if custom {
|
||
profiles.push(ProfileDeclaration {
|
||
profile_id: ProfileId::Custom(ProfileRegistryId([0xCC; 16])),
|
||
version: SemVer::new(1, 2, 3),
|
||
constraints: ProfileConstraints::DEFAULT_FULL,
|
||
});
|
||
}
|
||
profiles
|
||
}
|
||
|
||
fn extension(id_byte: u8, payloads: &[u8]) -> TextExtension {
|
||
TextExtension {
|
||
extension_id: ExtensionId([id_byte; 16]),
|
||
version: SemVer::new(1, 0, u32::from(id_byte)),
|
||
required: false,
|
||
chunks: payloads
|
||
.iter()
|
||
.map(|byte| TextChunk {
|
||
kind: ChunkKind::ExtensionData,
|
||
schema_version: SchemaVersion::V0,
|
||
payload: vec![*byte],
|
||
})
|
||
.collect(),
|
||
affected_object_kinds: vec![id_byte],
|
||
edit_barriers: vec![id_byte, id_byte.wrapping_add(1)],
|
||
}
|
||
}
|
||
|
||
fn base(snapshot_byte: u8) -> TextCanonicalBase {
|
||
TextCanonicalBase {
|
||
snapshot_id: SnapshotId([snapshot_byte; 16]),
|
||
covers_causal_frontier: FrontierBytes::from_bytes(vec![1, 2, snapshot_byte]),
|
||
reduction_algorithm_version: ReductionAlgorithmVersion(1),
|
||
profile_id: ProfileId::Full,
|
||
root_schema_version: SchemaVersion::V0,
|
||
root_payload: vec![0xA0, snapshot_byte],
|
||
}
|
||
}
|
||
|
||
fn accept_documents() -> Vec<(&'static str, String)> {
|
||
let minimal = TextDocument {
|
||
document_id: DocumentId([1; 16]),
|
||
manifest_schema_version: SchemaVersion::V0,
|
||
lineage_id: None,
|
||
profiles: profiles(false),
|
||
extensions: Vec::new(),
|
||
canonical_base: None,
|
||
blobs: Vec::new(),
|
||
envelopes: Vec::new(),
|
||
};
|
||
let lineage_custom = TextDocument {
|
||
document_id: DocumentId([2; 16]),
|
||
manifest_schema_version: SchemaVersion::V0,
|
||
lineage_id: Some(LineageId([0x12; 16])),
|
||
profiles: profiles(true),
|
||
extensions: Vec::new(),
|
||
canonical_base: None,
|
||
blobs: Vec::new(),
|
||
envelopes: vec![sample_envelope(1, 100)],
|
||
};
|
||
// `CONTRACT_FORMAT_EPOCH_MAJOR1.md` pin 3b: text projection can no longer
|
||
// carry a canonical base at all, so this document (still named for the
|
||
// two envelopes it exercises, not for a base it no longer carries) keeps
|
||
// its extension and non-baseline schema version but drops the base that
|
||
// used to make it `extension_base_multi`.
|
||
let extension_two_envelopes = TextDocument {
|
||
document_id: DocumentId([3; 16]),
|
||
// A non-baseline carried version, so the corpus exercises the
|
||
// `document` line's schema field at a value other than the
|
||
// ubiquitous baseline (G-minor pin 8/11).
|
||
manifest_schema_version: SchemaVersion::new(0, 8),
|
||
lineage_id: None,
|
||
profiles: profiles(false),
|
||
extensions: vec![extension(1, &[1, 2])],
|
||
canonical_base: None,
|
||
blobs: Vec::new(),
|
||
envelopes: vec![sample_envelope(2, 200), sample_envelope(3, 300)],
|
||
};
|
||
// Base removed (pin 3b); lineage, custom profile, two extensions, and
|
||
// multiple envelopes are all retained.
|
||
let rich = TextDocument {
|
||
document_id: DocumentId([4; 16]),
|
||
manifest_schema_version: SchemaVersion::V0,
|
||
lineage_id: Some(LineageId([0x14; 16])),
|
||
profiles: profiles(true),
|
||
extensions: vec![extension(1, &[1, 2]), extension(2, &[3])],
|
||
canonical_base: None,
|
||
blobs: Vec::new(),
|
||
envelopes: vec![
|
||
sample_envelope(4, 400),
|
||
sample_envelope(5, 500),
|
||
sample_envelope(6, 600),
|
||
],
|
||
};
|
||
|
||
// Genesis G2b: kind 34 in the committed corpus.
|
||
//
|
||
// The manifest version stays **baseline `V0`**, and that is the point. The
|
||
// `document` line carries the *manifest's* aggregate `SchemaVersion`
|
||
// (`req:textproj:manifest-schema-carried`), and this document declares no
|
||
// edit barriers, so an aware producer leaves it at baseline. The
|
||
// *operation block* serialized underneath independently stamps `{3, 10}` —
|
||
// a separate version domain that projection discards by design. Stamping
|
||
// the manifest at `{0, 10}` here would lock an over-stamped manifest into
|
||
// the corpus while appearing to prove the operation epoch, which is
|
||
// exactly the inference `text_projection.tex` §changelog forbids. The
|
||
// block's own stamp is proven where it lives, by
|
||
// `roundtrip::tests::a_set_tuning_context_block_stamps_3_10_and_reopens_read_write`.
|
||
let tuning_context = TextDocument {
|
||
document_id: DocumentId([5; 16]),
|
||
manifest_schema_version: SchemaVersion::V0,
|
||
lineage_id: None,
|
||
profiles: profiles(false),
|
||
extensions: Vec::new(),
|
||
canonical_base: None,
|
||
blobs: Vec::new(),
|
||
envelopes: vec![tuning_context_envelope(7, 700)],
|
||
};
|
||
|
||
// Genesis G3a: kinds 35–38 in the committed corpus. Same manifest-version
|
||
// discipline as `tuning_context` above — baseline `V0`, since none of
|
||
// these documents declares an edit barrier either.
|
||
let staff_group = TextDocument {
|
||
document_id: DocumentId([6; 16]),
|
||
manifest_schema_version: SchemaVersion::V0,
|
||
lineage_id: None,
|
||
profiles: profiles(false),
|
||
extensions: Vec::new(),
|
||
canonical_base: None,
|
||
blobs: Vec::new(),
|
||
envelopes: vec![staff_group_envelope(8, 800)],
|
||
};
|
||
let part_definition = TextDocument {
|
||
document_id: DocumentId([7; 16]),
|
||
manifest_schema_version: SchemaVersion::V0,
|
||
lineage_id: None,
|
||
profiles: profiles(false),
|
||
extensions: Vec::new(),
|
||
canonical_base: None,
|
||
blobs: Vec::new(),
|
||
envelopes: vec![part_definition_envelope(9, 900)],
|
||
};
|
||
let analysis_layer = TextDocument {
|
||
document_id: DocumentId([8; 16]),
|
||
manifest_schema_version: SchemaVersion::V0,
|
||
lineage_id: None,
|
||
profiles: profiles(false),
|
||
extensions: Vec::new(),
|
||
canonical_base: None,
|
||
blobs: Vec::new(),
|
||
envelopes: vec![analysis_layer_envelope(10, 1000)],
|
||
};
|
||
let view = TextDocument {
|
||
document_id: DocumentId([9; 16]),
|
||
manifest_schema_version: SchemaVersion::V0,
|
||
lineage_id: None,
|
||
profiles: profiles(false),
|
||
extensions: Vec::new(),
|
||
canonical_base: None,
|
||
blobs: Vec::new(),
|
||
envelopes: vec![view_envelope(11, 1100)],
|
||
};
|
||
|
||
// Genesis G3b: kind 39 in the committed corpus. Same manifest-version
|
||
// discipline as `staff_group` etc above.
|
||
let measure = TextDocument {
|
||
document_id: DocumentId([10; 16]),
|
||
manifest_schema_version: SchemaVersion::V0,
|
||
lineage_id: None,
|
||
profiles: profiles(false),
|
||
extensions: Vec::new(),
|
||
canonical_base: None,
|
||
blobs: Vec::new(),
|
||
envelopes: vec![measure_envelope(12, 1200)],
|
||
};
|
||
|
||
vec![
|
||
(
|
||
"minimal",
|
||
project_text_document(&minimal).expect("an accept document carries no canonical base"),
|
||
),
|
||
(
|
||
"set_tuning_context",
|
||
project_text_document(&tuning_context)
|
||
.expect("an accept document carries no canonical base"),
|
||
),
|
||
(
|
||
"lineage_custom_profile",
|
||
project_text_document(&lineage_custom)
|
||
.expect("an accept document carries no canonical base"),
|
||
),
|
||
(
|
||
"extension_base_two_envelopes",
|
||
project_text_document(&extension_two_envelopes)
|
||
.expect("an accept document carries no canonical base"),
|
||
),
|
||
(
|
||
"rich_document",
|
||
project_text_document(&rich).expect("an accept document carries no canonical base"),
|
||
),
|
||
(
|
||
"create_staff_group",
|
||
project_text_document(&staff_group)
|
||
.expect("an accept document carries no canonical base"),
|
||
),
|
||
(
|
||
"create_part_definition",
|
||
project_text_document(&part_definition)
|
||
.expect("an accept document carries no canonical base"),
|
||
),
|
||
(
|
||
"create_analysis_layer",
|
||
project_text_document(&analysis_layer)
|
||
.expect("an accept document carries no canonical base"),
|
||
),
|
||
(
|
||
"create_view",
|
||
project_text_document(&view).expect("an accept document carries no canonical base"),
|
||
),
|
||
(
|
||
"create_measure",
|
||
project_text_document(&measure).expect("an accept document carries no canonical base"),
|
||
),
|
||
]
|
||
}
|
||
|
||
fn replace_once(text: &str, anchor: &str, replacement: &str) -> String {
|
||
assert!(
|
||
text.contains(anchor),
|
||
"vector-construction anchor is absent: {anchor}"
|
||
);
|
||
text.replacen(anchor, replacement, 1)
|
||
}
|
||
|
||
fn lines_with_final_lf(lines: &[String]) -> String {
|
||
let mut text = lines.join("\n");
|
||
text.push('\n');
|
||
text
|
||
}
|
||
|
||
fn swap_first_lines(text: &str, first_head: &str, second_head: &str) -> String {
|
||
let mut lines: Vec<String> = text.lines().map(str::to_owned).collect();
|
||
let first = lines
|
||
.iter()
|
||
.position(|line| line.starts_with(first_head))
|
||
.unwrap_or_else(|| panic!("vector-construction line is absent: {first_head}"));
|
||
let second = lines
|
||
.iter()
|
||
.position(|line| line.starts_with(second_head))
|
||
.unwrap_or_else(|| panic!("vector-construction line is absent: {second_head}"));
|
||
lines.swap(first, second);
|
||
lines_with_final_lf(&lines)
|
||
}
|
||
|
||
fn swap_first_two_lines(text: &str, head: &str) -> String {
|
||
let mut lines: Vec<String> = text.lines().map(str::to_owned).collect();
|
||
let matches: Vec<usize> = lines
|
||
.iter()
|
||
.enumerate()
|
||
.filter_map(|(index, line)| line.starts_with(head).then_some(index))
|
||
.collect();
|
||
assert!(
|
||
matches.len() >= 2,
|
||
"vector construction needs two `{head}` lines"
|
||
);
|
||
lines.swap(matches[0], matches[1]);
|
||
lines_with_final_lf(&lines)
|
||
}
|
||
|
||
fn duplicate_first_line(text: &str, head: &str) -> String {
|
||
let mut lines: Vec<String> = text.lines().map(str::to_owned).collect();
|
||
let at = lines
|
||
.iter()
|
||
.position(|line| line.starts_with(head))
|
||
.unwrap_or_else(|| panic!("vector-construction line is absent: {head}"));
|
||
let duplicate = lines[at].clone();
|
||
lines.insert(at + 1, duplicate);
|
||
lines_with_final_lf(&lines)
|
||
}
|
||
|
||
/// Every whole-document vector in stable committed order.
|
||
pub fn document_vectors() -> Vec<TextVector> {
|
||
let accepts = accept_documents();
|
||
// Bound **by name, not by index**. These were positional (`accepts[0]`..
|
||
// `accepts[3]`), which silently coupled every negative vector to the
|
||
// insertion order of the accept list: adding the G2b
|
||
// `set_tuning_context` document shifted all four and made the reject
|
||
// builders operate on the wrong source text. By-name binding makes the
|
||
// accept list reorderable and extendable without touching this block.
|
||
let by_name = |name: &str| -> &String {
|
||
&accepts
|
||
.iter()
|
||
.find(|(n, _)| *n == name)
|
||
.unwrap_or_else(|| panic!("accept document is absent: {name}"))
|
||
.1
|
||
};
|
||
let minimal = by_name("minimal");
|
||
let lineage_custom = by_name("lineage_custom_profile");
|
||
let extension_two_envelopes = by_name("extension_base_two_envelopes");
|
||
let rich = by_name("rich_document");
|
||
|
||
let mut vectors: Vec<TextVector> = accepts
|
||
.iter()
|
||
.map(|(name, text)| (SURFACE, "accept", "-", *name, text.as_bytes().to_vec()))
|
||
.collect();
|
||
|
||
// The rejected version must be one this crate does NOT implement.
|
||
// `CONTRACT_FORMAT_EPOCH_MAJOR1.md` pin 3b moved `COMPANION_VERSION` to
|
||
// 0.14.0; this vector now names 0.13.0, the immediately superseded
|
||
// companion (previously 0.12.0, when the committed version was 0.13.0) —
|
||
// rejecting the version right behind you is exactly the deferred
|
||
// migrate-on-read posture (`req:textproj:header-version`).
|
||
let wrong_version = replace_once(
|
||
minimal,
|
||
"(text-projection (0 14 0))",
|
||
"(text-projection (0 13 0))",
|
||
);
|
||
vectors.push((
|
||
SURFACE,
|
||
"reject",
|
||
"wrong-header-version",
|
||
"superseded_companion_version",
|
||
wrong_version.into_bytes(),
|
||
));
|
||
|
||
let blob = replace_once(
|
||
minimal,
|
||
"(profile full (0 1 0) (constraints 67108864 (retention 1 () true)))\n",
|
||
"(profile full (0 1 0) (constraints 67108864 (retention 1 () true)))\n\
|
||
(blob \"application/octet-stream\" () #x01)\n",
|
||
);
|
||
vectors.push((
|
||
SURFACE,
|
||
"reject",
|
||
"blob-line",
|
||
"unreferenced_blob",
|
||
blob.into_bytes(),
|
||
));
|
||
|
||
// Re-expressed on a non-base section pair (pin 3b: neither accept carries
|
||
// a canonical base to invert against any more). `projection`'s order is
|
||
// `header document lineage? profile* extension* canonical-base? blob*
|
||
// envelope*`, so a profile/extension inversion reaches
|
||
// `out-of-order-sections` exactly as the old canonical-base/extension
|
||
// inversion did, without a base.
|
||
vectors.push((
|
||
SURFACE,
|
||
"reject",
|
||
"out-of-order-sections",
|
||
"canonical_base_before_extension",
|
||
swap_first_lines(extension_two_envelopes, "(profile ", "(extension ").into_bytes(),
|
||
));
|
||
|
||
vectors.push((
|
||
SURFACE,
|
||
"reject",
|
||
"repeated-singular-section",
|
||
"lineage_repeated",
|
||
duplicate_first_line(lineage_custom, "(lineage ").into_bytes(),
|
||
));
|
||
|
||
vectors.push((
|
||
SURFACE,
|
||
"reject",
|
||
"operation-envelope-order",
|
||
"envelopes_reversed",
|
||
swap_first_two_lines(extension_two_envelopes, "(envelope ").into_bytes(),
|
||
));
|
||
|
||
vectors.push((
|
||
SURFACE,
|
||
"reject",
|
||
"profile-declaration-order",
|
||
"profiles_reversed",
|
||
swap_first_two_lines(lineage_custom, "(profile ").into_bytes(),
|
||
));
|
||
|
||
vectors.push((
|
||
SURFACE,
|
||
"reject",
|
||
"extension-declaration-order",
|
||
"extensions_reversed",
|
||
swap_first_two_lines(rich, "(extension ").into_bytes(),
|
||
));
|
||
|
||
let chunks_reversed = replace_once(
|
||
extension_two_envelopes,
|
||
"((chunk extension-data (schema 0 1) #x01) (chunk extension-data (schema 0 1) #x02))",
|
||
"((chunk extension-data (schema 0 1) #x02) (chunk extension-data (schema 0 1) #x01))",
|
||
);
|
||
vectors.push((
|
||
SURFACE,
|
||
"reject",
|
||
"extension-chunk-order",
|
||
"extension_chunks_reversed",
|
||
chunks_reversed.into_bytes(),
|
||
));
|
||
|
||
let mut missing_lf = minimal.as_bytes().to_vec();
|
||
assert_eq!(missing_lf.pop(), Some(b'\n'), "accept text ends in LF");
|
||
vectors.push((
|
||
SURFACE,
|
||
"reject",
|
||
"missing-trailing-lf",
|
||
"final_lf_missing",
|
||
missing_lf,
|
||
));
|
||
|
||
// NEW (pin 3b): a base-bearing text, refused by the parse-side check —
|
||
// built from the pre-change base-bearing spelling, so the corpus keeps a
|
||
// base-bearing text as a negative rather than losing the spelling
|
||
// entirely.
|
||
//
|
||
// This is the one legitimate caller of `render_text_document`, the
|
||
// crate-private formatter that skips pin 3b's projection refusal:
|
||
// `project_text_document` now refuses a base-bearing document, and a
|
||
// negative vector still has to *contain* the spelling it asserts is
|
||
// refused. Producing the bytes is not the same as permitting them — every
|
||
// other vector above goes through the checked projector.
|
||
let canonical_base_present = TextDocument {
|
||
document_id: DocumentId([11; 16]),
|
||
manifest_schema_version: SchemaVersion::V0,
|
||
lineage_id: None,
|
||
profiles: profiles(false),
|
||
extensions: Vec::new(),
|
||
canonical_base: Some(base(11)),
|
||
blobs: Vec::new(),
|
||
envelopes: Vec::new(),
|
||
};
|
||
vectors.push((
|
||
SURFACE,
|
||
"reject",
|
||
"canonical-base-unsupported",
|
||
"canonical_base_present",
|
||
render_text_document(&canonical_base_present).into_bytes(),
|
||
));
|
||
|
||
vectors
|
||
}
|
||
|
||
fn to_hex(bytes: &[u8]) -> String {
|
||
bytes.iter().map(|byte| format!("{byte:02x}")).collect()
|
||
}
|
||
|
||
fn from_hex(hex: &str) -> Option<Vec<u8>> {
|
||
if hex.len() % 2 != 0 {
|
||
return None;
|
||
}
|
||
(0..hex.len())
|
||
.step_by(2)
|
||
.map(|at| u8::from_str_radix(&hex[at..at + 2], 16).ok())
|
||
.collect()
|
||
}
|
||
|
||
/// Renders the drift-locked corpus file.
|
||
pub fn render() -> String {
|
||
let mut out = String::from(HEADER);
|
||
out.push_str("\n# textproj.document\n");
|
||
for (surface, verdict, class, name, text) in document_vectors() {
|
||
out.push_str(&format!(
|
||
"{surface} {verdict} {class} {name} {}\n",
|
||
to_hex(&text)
|
||
));
|
||
}
|
||
out
|
||
}
|
||
|
||
/// Parses a corpus file, skipping comments and blank lines.
|
||
pub fn parse(text: &str) -> Result<Vec<Row>, String> {
|
||
let mut rows = Vec::new();
|
||
for (index, line) in text.lines().enumerate() {
|
||
let line = line.trim();
|
||
if line.is_empty() || line.starts_with('#') {
|
||
continue;
|
||
}
|
||
let fields: Vec<&str> = line.split_whitespace().collect();
|
||
if fields.len() != 5 {
|
||
return Err(format!(
|
||
"line {}: expected 5 columns, got {}",
|
||
index + 1,
|
||
fields.len()
|
||
));
|
||
}
|
||
rows.push(Row {
|
||
surface: fields[0].to_owned(),
|
||
verdict: fields[1].to_owned(),
|
||
class: fields[2].to_owned(),
|
||
name: fields[3].to_owned(),
|
||
text: from_hex(fields[4]).ok_or_else(|| format!("line {}: bad hex", index + 1))?,
|
||
});
|
||
}
|
||
Ok(rows)
|
||
}
|
||
|
||
/// Checks one corpus surface without collapsing parser rejection with accepted
|
||
/// normalization. `None` means this crate does not own `surface`.
|
||
pub fn check(surface: &str, text: &[u8]) -> Option<CheckVerdict> {
|
||
if surface != SURFACE {
|
||
return None;
|
||
}
|
||
let text = match std::str::from_utf8(text) {
|
||
Ok(text) => text,
|
||
Err(error) => return Some(CheckVerdict::Rejected(format!("non-UTF-8 text: {error}"))),
|
||
};
|
||
let document = match parse_document(text) {
|
||
Ok(document) => document,
|
||
Err(error) => return Some(CheckVerdict::Rejected(error.to_string())),
|
||
};
|
||
let bundle = match serialize_document(&document, MemStore::new(), FileUuid([0x7E; 16])) {
|
||
Ok(bundle) => bundle,
|
||
Err(error) => {
|
||
return Some(CheckVerdict::AcceptedNonCanonical(format!(
|
||
"parse accepted, but serialization failed: {error}"
|
||
)))
|
||
}
|
||
};
|
||
let projected = match project_bundle(&bundle) {
|
||
Ok(projected) => projected,
|
||
Err(error) => {
|
||
return Some(CheckVerdict::AcceptedNonCanonical(format!(
|
||
"parse accepted, but re-projection failed: {error}"
|
||
)))
|
||
}
|
||
};
|
||
if projected.as_bytes() == text.as_bytes() {
|
||
Some(CheckVerdict::AcceptedCanonical)
|
||
} else {
|
||
Some(CheckVerdict::AcceptedNonCanonical(
|
||
"parse accepted text that re-projected to different bytes".to_owned(),
|
||
))
|
||
}
|
||
}
|
||
|
||
fn verify_rows(rows: &[Row]) -> Vec<String> {
|
||
let mut failures = Vec::new();
|
||
for row in rows {
|
||
let Some(actual) = check(&row.surface, &row.text) else {
|
||
failures.push(format!("{}: no checker owns this surface", row.surface));
|
||
continue;
|
||
};
|
||
match (row.verdict.as_str(), actual) {
|
||
("accept", CheckVerdict::AcceptedCanonical) | ("reject", CheckVerdict::Rejected(_)) => {
|
||
}
|
||
("accept", CheckVerdict::AcceptedNonCanonical(detail)) => failures.push(format!(
|
||
"{}/{}: accepted, but not canonical: {detail}",
|
||
row.surface, row.name
|
||
)),
|
||
("reject", CheckVerdict::AcceptedNonCanonical(detail)) => failures.push(format!(
|
||
"{}/{} ({}): declared reject, but was ACCEPTED and normalized: {detail}",
|
||
row.surface, row.name, row.class
|
||
)),
|
||
("reject", CheckVerdict::AcceptedCanonical) => failures.push(format!(
|
||
"{}/{} ({}): declared reject, but was ACCEPTED as canonical",
|
||
row.surface, row.name, row.class
|
||
)),
|
||
("accept", CheckVerdict::Rejected(error)) => failures.push(format!(
|
||
"{}/{}: declared accept, but was rejected: {error}",
|
||
row.surface, row.name
|
||
)),
|
||
(other, actual) => failures.push(format!(
|
||
"{}/{}: unknown verdict {other}, got {actual:?}",
|
||
row.surface, row.name
|
||
)),
|
||
}
|
||
}
|
||
failures
|
||
}
|
||
|
||
/// Measures the corpus's actual reach. Reject counts include only vectors the
|
||
/// parser really refused; an accidentally canonicalized reject therefore drops
|
||
/// its class to zero instead of receiving credit from its label.
|
||
pub fn reach(text: &str) -> Result<ReachCounts, Vec<String>> {
|
||
let rows = parse(text).map_err(|error| vec![error])?;
|
||
let mut reach = ReachCounts {
|
||
extensions: 0,
|
||
canonical_bases: 0,
|
||
custom_profiles: 0,
|
||
lineages: 0,
|
||
multi_envelope: 0,
|
||
reject_classes: BTreeMap::new(),
|
||
};
|
||
let mut failures = Vec::new();
|
||
|
||
for row in &rows {
|
||
match row.verdict.as_str() {
|
||
"accept" => {
|
||
let text = match std::str::from_utf8(&row.text) {
|
||
Ok(text) => text,
|
||
Err(error) => {
|
||
failures.push(format!("{}/{}: {error}", row.surface, row.name));
|
||
continue;
|
||
}
|
||
};
|
||
let document = match parse_document(text) {
|
||
Ok(document) => document,
|
||
Err(error) => {
|
||
failures.push(format!("{}/{}: {error}", row.surface, row.name));
|
||
continue;
|
||
}
|
||
};
|
||
reach.extensions += usize::from(!document.extensions.is_empty());
|
||
reach.canonical_bases += usize::from(document.canonical_base.is_some());
|
||
reach.custom_profiles += usize::from(
|
||
document
|
||
.profiles
|
||
.iter()
|
||
.any(|profile| matches!(profile.profile_id, ProfileId::Custom(_))),
|
||
);
|
||
reach.lineages += usize::from(document.lineage_id.is_some());
|
||
reach.multi_envelope += usize::from(document.envelopes.len() > 1);
|
||
}
|
||
"reject" => {
|
||
if matches!(
|
||
check(&row.surface, &row.text),
|
||
Some(CheckVerdict::Rejected(_))
|
||
) {
|
||
*reach.reject_classes.entry(row.class.clone()).or_default() += 1;
|
||
}
|
||
}
|
||
other => failures.push(format!(
|
||
"{}/{}: unknown verdict {other}",
|
||
row.surface, row.name
|
||
)),
|
||
}
|
||
}
|
||
|
||
if failures.is_empty() {
|
||
Ok(reach)
|
||
} else {
|
||
Err(failures)
|
||
}
|
||
}
|
||
|
||
/// Verifies every verdict and the exact non-vacuity reach contract, returning
|
||
/// the number of checked whole documents or all disagreements.
|
||
pub fn verify(text: &str) -> Result<usize, Vec<String>> {
|
||
let rows = parse(text).map_err(|error| vec![error])?;
|
||
let mut failures = verify_rows(&rows);
|
||
match reach(text) {
|
||
Ok(actual) if actual == expected_reach() => {}
|
||
Ok(actual) => failures.push(format!(
|
||
"corpus reach mismatch: expected {:?}, got {actual:?}",
|
||
expected_reach()
|
||
)),
|
||
Err(mut errors) => failures.append(&mut errors),
|
||
}
|
||
if failures.is_empty() {
|
||
Ok(rows.len())
|
||
} else {
|
||
Err(failures)
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn the_committed_corpus_matches_the_generator() {
|
||
assert_eq!(
|
||
COMMITTED,
|
||
render(),
|
||
"\n{PATH} is stale. Regenerate:\n \
|
||
cargo run -q -p epiphany-testkit --example generate_vectors\n"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn the_reference_implementation_agrees_with_every_vector() {
|
||
match verify(COMMITTED) {
|
||
Ok(count) => assert_eq!(count, 20, "the corpus has unexpectedly thinned"),
|
||
Err(failures) => panic!(
|
||
"{} disagreement(s):\n{}",
|
||
failures.len(),
|
||
failures.join("\n")
|
||
),
|
||
}
|
||
}
|
||
|
||
/// This test intentionally names TEXT identity. Comparing either bundle's
|
||
/// bytes would be wrong: duplicate blobs collapse, layout is regenerated,
|
||
/// and non-canonical accelerators vanish while semantics remain unchanged.
|
||
#[test]
|
||
fn every_accept_vector_satisfies_the_text_quantified_byte_equation() {
|
||
for row in parse(COMMITTED)
|
||
.expect("the committed corpus parses")
|
||
.into_iter()
|
||
.filter(|row| row.verdict == "accept")
|
||
{
|
||
assert_eq!(
|
||
check(&row.surface, &row.text),
|
||
Some(CheckVerdict::AcceptedCanonical),
|
||
"{}/{} does not satisfy project(serialize(parse(T))) == T",
|
||
row.surface,
|
||
row.name
|
||
);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn the_corpus_reach_is_exact_and_non_vacuous() {
|
||
assert_eq!(
|
||
reach(COMMITTED).expect("the committed corpus has measurable reach"),
|
||
expected_reach(),
|
||
"the corpus must reach 2 extensions, 2 canonical bases, 2 custom profiles, \
|
||
2 lineages, 2 multi-envelope documents, and one real rejection in each class"
|
||
);
|
||
}
|
||
|
||
/// (t11) Genesis tranche G3a: text projection round-trips all four
|
||
/// kinds (`create-staff-group`, `create-part-definition`,
|
||
/// `create-analysis-layer`, `create-view`) it added.
|
||
///
|
||
/// **Mutation:** drop one parse arm (e.g.
|
||
/// `OperationKindTag::CreateStaffGroup` from `OperationKind::parse` in
|
||
/// `textproj_kind.rs`); must fail.
|
||
#[test]
|
||
fn t11_g3a_kinds_round_trip() {
|
||
for name in [
|
||
"create_staff_group",
|
||
"create_part_definition",
|
||
"create_analysis_layer",
|
||
"create_view",
|
||
] {
|
||
let text = accept_documents()
|
||
.into_iter()
|
||
.find(|(n, _)| *n == name)
|
||
.unwrap_or_else(|| panic!("accept document is absent: {name}"))
|
||
.1;
|
||
let document =
|
||
parse_document(&text).unwrap_or_else(|e| panic!("{name} must parse: {e}"));
|
||
assert_eq!(document.envelopes.len(), 1, "{name} carries one envelope");
|
||
let reprojected = project_text_document(&document)
|
||
.expect("an accept document carries no canonical base");
|
||
assert_eq!(
|
||
reprojected, text,
|
||
"{name}: project(serialize(parse(T))) == T must hold"
|
||
);
|
||
}
|
||
}
|
||
|
||
/// (t12) Genesis tranche G3b: text projection round-trips the new
|
||
/// `create-measure` kind, and the companion version is **0.14.0**
|
||
/// (`CONTRACT_FORMAT_EPOCH_MAJOR1.md` pin 3b bumped it from 0.13.0), with
|
||
/// the negative vector rejecting **0.13.0** (the immediately superseded
|
||
/// companion).
|
||
///
|
||
/// **Mutation:** drop `OperationKindTag::CreateMeasure` from
|
||
/// `OperationKind::parse` in `textproj_kind.rs`; must fail. Separately,
|
||
/// leave `COMPANION_VERSION` at `(0, 13, 0)`; the negative vector must
|
||
/// fail.
|
||
#[test]
|
||
fn t12_g3b_kinds_round_trip_and_companion_is_0_14_0_rejecting_0_13_0() {
|
||
assert_eq!(
|
||
crate::COMPANION_VERSION,
|
||
(0, 14, 0),
|
||
"the companion version must be 0.14.0"
|
||
);
|
||
|
||
let name = "create_measure";
|
||
let text = accept_documents()
|
||
.into_iter()
|
||
.find(|(n, _)| *n == name)
|
||
.unwrap_or_else(|| panic!("accept document is absent: {name}"))
|
||
.1;
|
||
let document = parse_document(&text).unwrap_or_else(|e| panic!("{name} must parse: {e}"));
|
||
assert_eq!(document.envelopes.len(), 1, "{name} carries one envelope");
|
||
let reprojected =
|
||
project_text_document(&document).expect("an accept document carries no canonical base");
|
||
assert_eq!(
|
||
reprojected, text,
|
||
"{name}: project(serialize(parse(T))) == T must hold"
|
||
);
|
||
|
||
// The negative vector must reject exactly the immediately superseded
|
||
// companion, 0.13.0.
|
||
let rows = parse(COMMITTED).expect("the committed corpus parses");
|
||
let superseded = rows
|
||
.iter()
|
||
.find(|r| r.name == "superseded_companion_version")
|
||
.expect("the superseded_companion_version vector is present");
|
||
assert_eq!(superseded.verdict, "reject");
|
||
let text = String::from_utf8(superseded.text.clone()).expect("utf8");
|
||
assert!(
|
||
text.contains("(text-projection (0 13 0))"),
|
||
"the negative vector must name the immediately superseded companion 0.13.0, got: {text}"
|
||
);
|
||
assert!(
|
||
parse_document(&text).is_err(),
|
||
"the superseded companion version must be rejected"
|
||
);
|
||
}
|
||
}
|