From 5111d655da7689c73b3ad3189d9ecb3cd207f888 Mon Sep 17 00:00:00 2001 From: Levi Neuwirth Date: Mon, 27 Jul 2026 22:31:32 -0400 Subject: [PATCH] Implement D0-B storage publication interfaces --- Cargo.lock | 43 + Cargo.toml | 1 + crates/levcs-store/Cargo.toml | 1 + crates/levcs-store/src/checkpoint.rs | 262 +- crates/levcs-store/src/completion.rs | 444 ++++ crates/levcs-store/src/drive.rs | 598 ++--- crates/levcs-store/src/index.rs | 18 +- crates/levcs-store/src/journal.rs | 86 +- crates/levcs-store/src/lib.rs | 14 + crates/levcs-store/src/options.rs | 164 ++ crates/levcs-store/src/recovery.rs | 2109 ++++++++++++++++- crates/levcs-store/src/roots.rs | 1345 +++++++++++ crates/levcs-store/src/segment.rs | 253 +- crates/levcs-store/src/staging.rs | 521 +++- crates/levcs-store/src/transaction.rs | 109 +- crates/levcs-store/src/types.rs | 60 +- .../levcs-store/tests/recovery_checkpoint.rs | 1 + .../tests/recovery_checkpoint_faults.rs | 1 + crates/levcs-store/tests/recovery_manifest.rs | 141 +- crates/levcs-store/tests/recovery_receipts.rs | 1 + crates/levcs-store/tests/recovery_step8.rs | 333 +++ doc/instance-throughput-rewrite-plan.md | 83 + doc/phase1-storage-spine-scope.md | 99 +- 23 files changed, 6071 insertions(+), 616 deletions(-) create mode 100644 crates/levcs-store/src/completion.rs create mode 100644 crates/levcs-store/src/roots.rs create mode 100644 crates/levcs-store/tests/recovery_step8.rs diff --git a/Cargo.lock b/Cargo.lock index 3725f18..cd280ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -266,6 +266,15 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +[[package]] +name = "bitmaps" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" +dependencies = [ + "typenum", +] + [[package]] name = "blake2" version = "0.10.6" @@ -1359,6 +1368,20 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "im" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" +dependencies = [ + "bitmaps", + "rand_core 0.6.4", + "rand_xoshiro", + "sized-chunks", + "typenum", + "version_check", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -1669,6 +1692,7 @@ dependencies = [ "ed25519-dalek", "hdrhistogram", "hex", + "im", "levcs-core", "levcs-protocol", "libc", @@ -2199,6 +2223,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "ratatui" version = "0.28.1" @@ -2613,6 +2646,16 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "sized-chunks" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" +dependencies = [ + "bitmaps", + "typenum", +] + [[package]] name = "slab" version = "0.4.12" diff --git a/Cargo.toml b/Cargo.toml index 3ddfc8c..bfd78a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,6 +77,7 @@ anyhow = "1.0" # for why each of these is taken and what was rejected in its place. arc-swap = "1.7" crossbeam-channel = "0.5" +im = "15.1" rustix = { version = "1.1", features = ["fs"] } libc = "0.2" memmap2 = "0.9" diff --git a/crates/levcs-store/Cargo.toml b/crates/levcs-store/Cargo.toml index f7c184e..25a7e3f 100644 --- a/crates/levcs-store/Cargo.toml +++ b/crates/levcs-store/Cargo.toml @@ -34,6 +34,7 @@ thiserror = { workspace = true } hex = { workspace = true } arc-swap = { workspace = true } crossbeam-channel = { workspace = true } +im = { workspace = true } rustix = { workspace = true } libc = { workspace = true } memmap2 = { workspace = true } diff --git a/crates/levcs-store/src/checkpoint.rs b/crates/levcs-store/src/checkpoint.rs index 6425268..f9942e5 100644 --- a/crates/levcs-store/src/checkpoint.rs +++ b/crates/levcs-store/src/checkpoint.rs @@ -30,10 +30,11 @@ use std::io::IoSlice; use std::path::{Path, PathBuf}; use levcs_core::ObjectId; +use levcs_protocol::v2::{RefTarget, MAX_REF_UPDATES}; use crate::format::{CHECKPOINT_DIGEST_DOMAIN, CHECKPOINT_MAGIC, STORAGE_VERSION}; use crate::index::{NamespaceCatalog, NamespaceLifecycle, NamespaceRecord, NamespaceStorageMode}; -use crate::types::{DurabilityCounters, NamespaceId, OperationId, StoreError}; +use crate::types::{AppliedRef, DurabilityCounters, NamespaceId, OperationId, StoreError}; /// 88 bytes of named fields, zero padding to 96, then a 32-byte header digest. /// Every byte that is not a named field must be zero, for the same reason @@ -51,6 +52,18 @@ const MAX_CHECKPOINT_REFS: u32 = 64_000_000; const MAX_CHECKPOINT_RECEIPTS: u32 = 64_000_000; const MAX_REF_NAME_LEN: u16 = 1024; +/// Checkpoint-body capability: every retained receipt carries its complete +/// applied-ref result. +/// +/// Checkpoints are derived and independently versioned by their flags within +/// storage version 1. A version-1 checkpoint with flags zero predates this +/// capability. It remains readable only when it has no retained receipts; +/// otherwise accepting it would publish a receipt with an invented empty ref +/// result. Such generations are rejected and recovery enters its existing +/// explicit offline-rebuild path when no newer complete generation exists. +const CHECKPOINT_FLAG_RECEIPT_REFS: u32 = 1 << 0; +const CHECKPOINT_KNOWN_FLAGS: u32 = CHECKPOINT_FLAG_RECEIPT_REFS; + // --------------------------------------------------------------------------- // Errors // --------------------------------------------------------------------------- @@ -68,8 +81,10 @@ pub enum CheckpointError { Magic, #[error("checkpoint storage version {0} is not readable")] StorageVersion(u16), - #[error("checkpoint flags and reserved fields must be zero")] + #[error("checkpoint has unknown flags or non-zero reserved fields")] Flags, + #[error("checkpoint predates complete applied-ref receipt retention")] + ReceiptRefsUnavailable, #[error("checkpoint header digest does not recompute")] HeaderDigest, #[error("checkpoint trailer magic or repeated total_len does not match")] @@ -125,6 +140,12 @@ pub struct ReceiptRecord { pub repo_sequence: u64, pub shard_sequence: u64, pub current_authority: ObjectId, + /// The exact per-transaction result, including deletions and force. + /// + /// This cannot be reconstructed from the checkpoint's current ref table: + /// that table has neither the old value nor transaction membership, and a + /// deleted ref is absent from it entirely. + pub refs: Vec, pub objects_new: u64, pub retry_until_micros: i64, /// `None` when the crash preceded a durable capture of first visibility. @@ -286,8 +307,148 @@ fn read_optional_i64( } } +fn put_optional_object_id(out: &mut Vec, value: Option) { + match value { + Some(value) => { + out.push(1); + out.extend_from_slice(value.as_bytes()); + } + None => out.push(0), + } +} + +fn read_optional_object_id( + r: &mut Reader<'_>, + what: &'static str, +) -> Result, CheckpointError> { + match r.u8(what)? { + 0 => Ok(None), + 1 => Ok(Some(ObjectId(r.bytes32(what)?))), + _ => Err(CheckpointError::Body( + "optional object-id presence byte must be 0 or 1", + )), + } +} + +fn ref_target_parts(target: &RefTarget) -> (u8, &str) { + match target { + RefTarget::Branch(name) => (1, name), + RefTarget::Release(name) => (2, name), + } +} + +fn validate_ref_target(name: &str) -> Result<(), StoreError> { + if name.len() > MAX_REF_NAME_LEN as usize { + return Err(StoreError::LimitExceeded { + limit: "checkpoint_receipt_ref_name_len", + observed: name.len() as u64, + allowed: MAX_REF_NAME_LEN as u64, + }); + } + if name.is_empty() + || name.as_bytes().contains(&0) + || levcs_core::refs::validate_ref_name(name).is_err() + { + return Err(StoreError::Corruption( + "checkpoint receipt contains an invalid applied-ref target".into(), + )); + } + Ok(()) +} + +/// Checkpoint-local physical encoding of the frozen `AppliedRefV1` value. +/// +/// The protocol codec intentionally keeps its element codec private. Plan +/// §13 keeps these checkpoint bytes internal, so this codec delegates name +/// validation to the same `levcs-core` function and pins the complete logical +/// value rather than reaching through a private protocol implementation. +fn put_applied_refs(out: &mut Vec, refs: &[AppliedRef]) -> Result<(), StoreError> { + if refs.len() > MAX_REF_UPDATES { + return Err(StoreError::LimitExceeded { + limit: "checkpoint_receipt_applied_refs", + observed: refs.len() as u64, + allowed: MAX_REF_UPDATES as u64, + }); + } + let mut targets = std::collections::BTreeSet::new(); + put_u32(out, refs.len() as u32); + for applied in refs { + if !targets.insert(applied.target.clone()) { + return Err(StoreError::Corruption( + "checkpoint receipt contains duplicate applied-ref targets".into(), + )); + } + let (kind, name) = ref_target_parts(&applied.target); + validate_ref_target(name)?; + out.push(kind); + put_u16(out, name.len() as u16); + out.extend_from_slice(name.as_bytes()); + put_optional_object_id(out, applied.old); + put_optional_object_id(out, applied.new); + out.push(u8::from(applied.force)); + } + Ok(()) +} + +fn read_applied_refs(r: &mut Reader<'_>) -> Result, CheckpointError> { + let count = r.u32("receipt applied-ref count")?; + if count as usize > MAX_REF_UPDATES { + return Err(CheckpointError::CountCeiling("receipt applied refs")); + } + // Seven bytes is the smallest valid record: kind, name length, one-byte + // name, two absent object IDs, and force. + let mut refs = Vec::with_capacity((count as usize).min(r.bytes.len() / 7 + 1)); + let mut targets = std::collections::BTreeSet::new(); + for _ in 0..count { + let kind = r.u8("receipt ref target kind")?; + let name_len = r.u16("receipt ref target name length")?; + if name_len > MAX_REF_NAME_LEN { + return Err(CheckpointError::CountCeiling( + "receipt ref target name length", + )); + } + if name_len == 0 { + return Err(CheckpointError::Body("receipt ref target name length")); + } + let name = std::str::from_utf8(r.take(name_len as usize, "receipt ref target name")?) + .map_err(|_| CheckpointError::Body("receipt ref target name utf-8"))? + .to_owned(); + if name.as_bytes().contains(&0) || levcs_core::refs::validate_ref_name(&name).is_err() { + return Err(CheckpointError::Body("receipt ref target name")); + } + let target = match kind { + 1 => RefTarget::Branch(name), + 2 => RefTarget::Release(name), + _ => return Err(CheckpointError::Body("receipt ref target kind")), + }; + if !targets.insert(target.clone()) { + return Err(CheckpointError::Body( + "duplicate receipt applied-ref target", + )); + } + let old = read_optional_object_id(r, "receipt old ref object")?; + let new = read_optional_object_id(r, "receipt new ref object")?; + let force = match r.u8("receipt applied-ref force")? { + 0 => false, + 1 => true, + _ => return Err(CheckpointError::Body("receipt applied-ref force")), + }; + refs.push(AppliedRef { + target, + old, + new, + force, + }); + } + Ok(refs) +} + impl Checkpoint { pub fn encode(&self) -> Result, StoreError> { + self.encode_with_receipt_refs(true) + } + + fn encode_with_receipt_refs(&self, include_receipt_refs: bool) -> Result, StoreError> { if self.catalog.len() as u64 > MAX_CHECKPOINT_NAMESPACES as u64 { return Err(StoreError::LimitExceeded { limit: "checkpoint_namespaces", @@ -349,6 +510,9 @@ impl Checkpoint { put_u64(&mut body, rec.repo_sequence); put_u64(&mut body, rec.shard_sequence); body.extend_from_slice(&rec.current_authority.0); + if include_receipt_refs { + put_applied_refs(&mut body, &rec.refs)?; + } put_u64(&mut body, rec.objects_new); put_i64(&mut body, rec.retry_until_micros); put_optional_i64(&mut body, rec.first_receipt_visibility_micros); @@ -361,7 +525,14 @@ impl Checkpoint { header.extend_from_slice(&CHECKPOINT_MAGIC); put_u16(&mut header, STORAGE_VERSION); put_u16(&mut header, CHECKPOINT_HEADER_LEN as u16); - put_u32(&mut header, 0); // flags: must be zero + put_u32( + &mut header, + if include_receipt_refs { + CHECKPOINT_FLAG_RECEIPT_REFS + } else { + 0 + }, + ); header.extend_from_slice(&self.root_uuid); put_u16(&mut header, self.shard_index); put_u16(&mut header, 0); // reserved @@ -408,9 +579,11 @@ impl Checkpoint { if head.u16("header_len")? as usize != CHECKPOINT_HEADER_LEN { return Err(CheckpointError::Body("header_len")); } - if head.u32("flags")? != 0 { + let checkpoint_flags = head.u32("flags")?; + if checkpoint_flags & !CHECKPOINT_KNOWN_FLAGS != 0 { return Err(CheckpointError::Flags); } + let has_receipt_refs = checkpoint_flags & CHECKPOINT_FLAG_RECEIPT_REFS != 0; let file_root_uuid = head.bytes16("root_uuid")?; let file_shard = head.u16("shard_index")?; if head.u16("reserved")? != 0 || head.u32("reserved")? != 0 { @@ -543,7 +716,9 @@ impl Checkpoint { if receipt_count > MAX_CHECKPOINT_RECEIPTS { return Err(CheckpointError::CountCeiling("receipts")); } - let mut receipts = Vec::with_capacity((receipt_count as usize).min(body.len() / 161 + 1)); + let minimum_receipt_len = if has_receipt_refs { 165 } else { 161 }; + let mut receipts = + Vec::with_capacity((receipt_count as usize).min(body.len() / minimum_receipt_len + 1)); for _ in 0..receipt_count { receipts.push(ReceiptRecord { namespace: NamespaceId(r.bytes32("receipt namespace")?), @@ -552,6 +727,11 @@ impl Checkpoint { repo_sequence: r.u64("repo_sequence")?, shard_sequence: r.u64("shard_sequence")?, current_authority: ObjectId(r.bytes32("current_authority")?), + refs: if has_receipt_refs { + read_applied_refs(&mut r)? + } else { + Vec::new() + }, objects_new: r.u64("objects_new")?, retry_until_micros: r.i64("retry_until_micros")?, first_receipt_visibility_micros: read_optional_i64(&mut r, "first_visibility")?, @@ -562,6 +742,9 @@ impl Checkpoint { if !r.finished() { return Err(CheckpointError::TrailingBytes); } + if !has_receipt_refs && !receipts.is_empty() { + return Err(CheckpointError::ReceiptRefsUnavailable); + } Ok(Self { root_uuid: file_root_uuid, @@ -719,13 +902,13 @@ pub fn install( .open(&tmp_path)?; let end = crate::sys::write_vectored_all(&mut file, &[IoSlice::new(&bytes)], counters) .map_err(|e| { - StoreError::Io(std::io::Error::new( + StoreError::from(std::io::Error::new( e.kind(), format!("checkpoint body write failed: {e}"), )) })?; if end != bytes.len() as u64 { - return Err(StoreError::Io(std::io::Error::new( + return Err(StoreError::from(std::io::Error::new( std::io::ErrorKind::WriteZero, format!( "short write installing checkpoint: wrote {end} of {} bytes; \ @@ -739,7 +922,7 @@ pub fn install( crate::sys::rename_noreplace(&tmp_path, &final_path).map_err(|e| { // Leave the temporary behind for forensics rather than unlinking it // on a path that already surprised us. - StoreError::Io(e) + StoreError::from(e) })?; crate::sys::fsync_dir(dir, counters)?; Ok(final_path) @@ -827,6 +1010,20 @@ mod tests { repo_sequence: 7, shard_sequence: 4241, current_authority: oid(0x21), + refs: vec![ + AppliedRef { + target: RefTarget::Branch("refs/branches/main".into()), + old: Some(oid(0x60)), + new: Some(oid(0x61)), + force: false, + }, + AppliedRef { + target: RefTarget::Release("refs/releases/old".into()), + old: Some(oid(0x62)), + new: None, + force: true, + }, + ], objects_new: 3, retry_until_micros: 1_700_000_900_000_000, first_receipt_visibility_micros: Some(1_700_000_000_500_000), @@ -839,6 +1036,7 @@ mod tests { repo_sequence: 14, shard_sequence: 4242, current_authority: oid(0x22), + refs: Vec::new(), objects_new: 0, retry_until_micros: 1_700_000_900_000_000, first_receipt_visibility_micros: None, @@ -854,6 +1052,52 @@ mod tests { let bytes = checkpoint.encode().expect("encode"); let decoded = Checkpoint::decode(&bytes, &ROOT, SHARD).expect("decode"); assert_eq!(decoded, checkpoint); + assert_eq!( + decoded.receipts[0].refs, checkpoint.receipts[0].refs, + "old/new values, deletion, force, target kind, and order are receipt data" + ); + } + + #[test] + fn a_legacy_checkpoint_with_receipts_requires_offline_rebuild() { + let legacy = sample() + .encode_with_receipt_refs(false) + .expect("legacy fixture"); + assert_eq!( + Checkpoint::decode(&legacy, &ROOT, SHARD).unwrap_err(), + CheckpointError::ReceiptRefsUnavailable, + "inventing an empty applied-ref result would turn a committed receipt into a lie" + ); + } + + #[test] + fn a_legacy_checkpoint_without_receipts_remains_readable() { + let mut checkpoint = sample(); + checkpoint.receipts.clear(); + let legacy = checkpoint + .encode_with_receipt_refs(false) + .expect("legacy fixture"); + assert_eq!( + Checkpoint::decode(&legacy, &ROOT, SHARD).expect("no receipt information is missing"), + checkpoint + ); + } + + #[test] + fn a_directory_of_legacy_receipt_checkpoints_enters_offline_rebuild() { + let dir = tempfile::tempdir().expect("tempdir"); + let checkpoint = sample(); + let legacy = checkpoint + .encode_with_receipt_refs(false) + .expect("legacy fixture"); + std::fs::write(dir.path().join(checkpoint.file_name()), legacy).expect("write fixture"); + match load_newest_valid(dir.path(), &ROOT, SHARD, 8).expect("load") { + CheckpointLoad::OfflineRebuildRequired { rejected } => { + assert_eq!(rejected.len(), 1); + assert_eq!(rejected[0].1, CheckpointError::ReceiptRefsUnavailable); + } + other => panic!("missing receipt data must require offline rebuild, got {other:?}"), + } } #[test] @@ -899,7 +1143,7 @@ mod tests { ); let mut flags = good.clone(); - flags[12] = 1; + flags[12] |= 2; assert_eq!( Checkpoint::decode(&flags, &ROOT, SHARD).unwrap_err(), CheckpointError::Flags diff --git a/crates/levcs-store/src/completion.rs b/crates/levcs-store/src/completion.rs new file mode 100644 index 0000000..ca877d1 --- /dev/null +++ b/crates/levcs-store/src/completion.rs @@ -0,0 +1,444 @@ +//! Runtime-agnostic, one-to-many completion for submitted transactions. +//! +//! One in-flight transaction owns a [`SharedCompletion`]. Every caller that +//! coalesces onto that transaction obtains its own [`CompletionWaiter`], while +//! the shard writer publishes exactly one [`CompletionOutcome`]. +//! +//! The outcome and waiter registrations deliberately share one mutex. A +//! waiter therefore cannot observe "not complete", lose the mutex, and +//! register after the only wake has already happened. Completion takes the +//! registered wakers while holding that mutex, then wakes them after releasing +//! it so arbitrary executor code never runs in the critical section. + +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::task::{Context, Poll, Waker}; + +use crate::types::{CommitReceipt, StoreError}; + +/// The one durable or definitive outcome shared by all attached callers. +pub type CompletionOutcome = Result; + +/// One transaction's shared completion state. +/// +/// Cloning this value clones only the `Arc`; it does not create a second +/// execution or a second outcome slot. [`SharedCompletion::complete`] accepts +/// the first outcome and treats any later attempt as a harmless no-op. +#[derive(Clone)] +pub struct SharedCompletion { + shared: Arc, +} + +struct Shared { + state: Mutex, +} + +struct State { + outcome: Option, + wakers: Vec, +} + +struct WaiterWaker { + identity: Arc<()>, + waker: Waker, +} + +/// The exactly-once future belonging to one attached caller. +/// +/// Dropping this value removes its registered waker without affecting the +/// transaction or any other waiter. Polling it after it has returned `Ready` +/// is a caller bug and panics with a stable diagnostic instead of attempting +/// to manufacture a second value. +#[must_use = "a completion waiter does nothing unless polled or awaited"] +pub struct CompletionWaiter { + shared: Arc, + identity: Arc<()>, + delivered: bool, +} + +impl SharedCompletion { + /// Creates an empty completion to be owned by one in-flight transaction. + pub fn new() -> Self { + Self { + shared: Arc::new(Shared { + state: Mutex::new(State { + outcome: None, + wakers: Vec::new(), + }), + }), + } + } + + /// Attaches one caller to this transaction's outcome. + /// + /// Attachment remains valid after completion: its first poll observes the + /// stored result immediately and requires no wake. + pub fn subscribe(&self) -> CompletionWaiter { + CompletionWaiter { + shared: Arc::clone(&self.shared), + identity: Arc::new(()), + delivered: false, + } + } + + /// Publishes the outcome and wakes every currently registered waiter. + /// + /// Returns `true` when this call published the first outcome and `false` + /// when the completion had already been signalled. No receiver is needed: + /// completing after some or all waiters have dropped is successful and + /// harmless. + /// + /// Wakers are invoked outside the state mutex. A panicking executor waker + /// is isolated so it cannot unwind the shard thread or prevent another + /// waiter from being woken. + pub fn complete(&self, outcome: CompletionOutcome) -> bool { + let wakers = { + let mut state = self.shared.lock_state(); + if state.outcome.is_some() { + return false; + } + + state.outcome = Some(outcome); + std::mem::take(&mut state.wakers) + }; + + for waiter in wakers { + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + waiter.waker.wake(); + })); + } + + true + } + + #[cfg(test)] + fn registered_waiters(&self) -> usize { + self.shared.lock_state().wakers.len() + } +} + +impl Default for SharedCompletion { + fn default() -> Self { + Self::new() + } +} + +impl Shared { + /// A poisoned completion mutex must not turn a committed transaction into + /// a shard-thread panic. The protected state remains structurally valid + /// because every mutation is an individual move or collection operation. + fn lock_state(&self) -> MutexGuard<'_, State> { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +impl Future for CompletionWaiter { + type Output = CompletionOutcome; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + if self.delivered { + panic!("CompletionWaiter polled after returning Ready"); + } + + let outcome = { + let mut state = self.shared.lock_state(); + + if let Some(outcome) = state.outcome.as_ref() { + Some(outcome.clone()) + } else { + match state + .wakers + .iter_mut() + .find(|registered| Arc::ptr_eq(®istered.identity, &self.identity)) + { + Some(registered) => { + if !registered.waker.will_wake(cx.waker()) { + registered.waker = cx.waker().clone(); + } + } + None => state.wakers.push(WaiterWaker { + identity: Arc::clone(&self.identity), + waker: cx.waker().clone(), + }), + } + None + } + }; + + match outcome { + Some(outcome) => { + self.delivered = true; + Poll::Ready(outcome) + } + None => Poll::Pending, + } + } +} + +impl Drop for CompletionWaiter { + fn drop(&mut self) { + if self.delivered { + return; + } + + let mut state = self.shared.lock_state(); + if let Some(index) = state + .wakers + .iter() + .position(|registered| Arc::ptr_eq(®istered.identity, &self.identity)) + { + state.wakers.swap_remove(index); + } + } +} + +#[cfg(test)] +mod tests { + use std::future::Future; + use std::pin::Pin; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + use std::task::{Context, Poll, Wake, Waker}; + + use super::{CompletionWaiter, SharedCompletion}; + use crate::types::{CommitReceipt, OperationId, StoreError}; + + #[derive(Default)] + struct WakeCount { + wakes: AtomicUsize, + } + + impl Wake for WakeCount { + fn wake(self: Arc) { + self.wakes.fetch_add(1, Ordering::SeqCst); + } + + fn wake_by_ref(self: &Arc) { + self.wakes.fetch_add(1, Ordering::SeqCst); + } + } + + struct PanickingWake; + + impl Wake for PanickingWake { + fn wake(self: Arc) { + panic!("executor waker panic"); + } + } + + fn receipt(sequence: u64) -> CommitReceipt { + CommitReceipt { + operation_id: OperationId::from_bytes([7; 16]), + repo_sequence: sequence, + current_authority: levcs_core::ObjectId([11; 32]), + refs: Vec::new(), + objects_new: 3, + } + } + + fn poll_with( + waiter: &mut CompletionWaiter, + wake: &Arc, + ) -> Poll> { + let waker = Waker::from(Arc::clone(wake)); + let mut context = Context::from_waker(&waker); + Pin::new(waiter).poll(&mut context) + } + + fn assert_ready_ok(outcome: Poll>, expected: &CommitReceipt) { + match outcome { + Poll::Ready(Ok(actual)) => assert_eq!(&actual, expected), + other => panic!("expected a successful ready outcome, got {other:?}"), + } + } + + #[test] + fn completion_after_first_poll_wakes_and_delivers() { + let completion = SharedCompletion::new(); + let mut waiter = completion.subscribe(); + let wake = Arc::new(WakeCount::default()); + + assert!(poll_with(&mut waiter, &wake).is_pending()); + assert_eq!(completion.registered_waiters(), 1); + + let expected = receipt(13); + assert!(completion.complete(Ok(expected.clone()))); + assert_eq!(wake.wakes.load(Ordering::SeqCst), 1); + assert_eq!(completion.registered_waiters(), 0); + assert_ready_ok(poll_with(&mut waiter, &wake), &expected); + } + + #[test] + fn completion_before_first_poll_is_immediately_ready() { + let completion = SharedCompletion::new(); + let expected = receipt(21); + assert!(completion.complete(Ok(expected.clone()))); + + let mut waiter = completion.subscribe(); + let wake = Arc::new(WakeCount::default()); + assert_ready_ok(poll_with(&mut waiter, &wake), &expected); + assert_eq!(wake.wakes.load(Ordering::SeqCst), 0); + } + + #[test] + fn a_waiter_delivers_exactly_once() { + let completion = SharedCompletion::new(); + let mut waiter = completion.subscribe(); + let wake = Arc::new(WakeCount::default()); + assert!(completion.complete(Ok(receipt(34)))); + assert!(poll_with(&mut waiter, &wake).is_ready()); + + let second_poll = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _ = poll_with(&mut waiter, &wake); + })); + assert!(second_poll.is_err()); + } + + #[test] + fn multiple_waiters_receive_the_same_outcome() { + let completion = SharedCompletion::new(); + let mut first = completion.subscribe(); + let mut second = completion.subscribe(); + let first_wake = Arc::new(WakeCount::default()); + let second_wake = Arc::new(WakeCount::default()); + + assert!(poll_with(&mut first, &first_wake).is_pending()); + assert!(poll_with(&mut second, &second_wake).is_pending()); + + let expected = receipt(55); + assert!(completion.complete(Ok(expected.clone()))); + assert_eq!(first_wake.wakes.load(Ordering::SeqCst), 1); + assert_eq!(second_wake.wakes.load(Ordering::SeqCst), 1); + assert_ready_ok(poll_with(&mut first, &first_wake), &expected); + assert_ready_ok(poll_with(&mut second, &second_wake), &expected); + } + + #[test] + fn dropping_one_waiter_removes_only_its_waker() { + let completion = SharedCompletion::new(); + let mut dropped = completion.subscribe(); + let mut retained = completion.subscribe(); + let dropped_wake = Arc::new(WakeCount::default()); + let retained_wake = Arc::new(WakeCount::default()); + + assert!(poll_with(&mut dropped, &dropped_wake).is_pending()); + assert!(poll_with(&mut retained, &retained_wake).is_pending()); + assert_eq!(completion.registered_waiters(), 2); + drop(dropped); + assert_eq!(completion.registered_waiters(), 1); + + let expected = receipt(89); + assert!(completion.complete(Ok(expected.clone()))); + assert_eq!(dropped_wake.wakes.load(Ordering::SeqCst), 0); + assert_eq!(retained_wake.wakes.load(Ordering::SeqCst), 1); + assert_ready_ok(poll_with(&mut retained, &retained_wake), &expected); + } + + #[test] + fn completing_after_all_waiters_drop_is_harmless() { + let completion = SharedCompletion::new(); + let mut first = completion.subscribe(); + let mut second = completion.subscribe(); + let first_wake = Arc::new(WakeCount::default()); + let second_wake = Arc::new(WakeCount::default()); + + assert!(poll_with(&mut first, &first_wake).is_pending()); + assert!(poll_with(&mut second, &second_wake).is_pending()); + drop(first); + drop(second); + assert_eq!(completion.registered_waiters(), 0); + + assert!(completion.complete(Ok(receipt(144)))); + assert_eq!(first_wake.wakes.load(Ordering::SeqCst), 0); + assert_eq!(second_wake.wakes.load(Ordering::SeqCst), 0); + assert!(!completion.complete(Ok(receipt(145)))); + } + + #[test] + fn io_error_is_shared_losslessly_between_waiters() { + let completion = SharedCompletion::new(); + let mut first = completion.subscribe(); + let mut second = completion.subscribe(); + let wake = Arc::new(WakeCount::default()); + let original = Arc::new(std::io::Error::other("one failure")); + + assert!(completion.complete(Err(StoreError::Io(Arc::clone(&original))))); + let first_error = match poll_with(&mut first, &wake) { + Poll::Ready(Err(StoreError::Io(error))) => error, + other => panic!("unexpected first outcome: {other:?}"), + }; + let second_error = match poll_with(&mut second, &wake) { + Poll::Ready(Err(StoreError::Io(error))) => error, + other => panic!("unexpected second outcome: {other:?}"), + }; + + assert!(Arc::ptr_eq(&original, &first_error)); + assert!(Arc::ptr_eq(&first_error, &second_error)); + } + + #[test] + fn panicking_waker_cannot_unwind_completion_or_skip_other_waiters() { + let completion = SharedCompletion::new(); + let mut panicking = completion.subscribe(); + let mut retained = completion.subscribe(); + let panic_waker = Waker::from(Arc::new(PanickingWake)); + let mut panic_context = Context::from_waker(&panic_waker); + let retained_wake = Arc::new(WakeCount::default()); + + assert!(Pin::new(&mut panicking) + .poll(&mut panic_context) + .is_pending()); + assert!(poll_with(&mut retained, &retained_wake).is_pending()); + + let completed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + completion.complete(Ok(receipt(233))) + })); + assert!(matches!(completed, Ok(true))); + assert_eq!(retained_wake.wakes.load(Ordering::SeqCst), 1); + } + + #[test] + fn poisoned_state_mutex_does_not_panic_completion() { + let completion = SharedCompletion::new(); + let shared = Arc::clone(&completion.shared); + + let poisoned = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _guard = shared.state.lock().expect("initial lock"); + panic!("poison completion state"); + })); + assert!(poisoned.is_err()); + + assert!(completion.complete(Ok(receipt(377)))); + let mut waiter = completion.subscribe(); + let wake = Arc::new(WakeCount::default()); + assert!(poll_with(&mut waiter, &wake).is_ready()); + } + + #[test] + fn repeated_pending_polls_replace_instead_of_accumulating_wakers() { + let completion = SharedCompletion::new(); + let mut waiter = completion.subscribe(); + let first_wake = Arc::new(WakeCount::default()); + let second_wake = Arc::new(WakeCount::default()); + + assert!(poll_with(&mut waiter, &first_wake).is_pending()); + assert!(poll_with(&mut waiter, &second_wake).is_pending()); + assert_eq!(completion.registered_waiters(), 1); + + assert!(completion.complete(Ok(receipt(610)))); + assert_eq!(first_wake.wakes.load(Ordering::SeqCst), 0); + assert_eq!(second_wake.wakes.load(Ordering::SeqCst), 1); + } + + #[test] + fn completion_is_send_sync_and_waiter_is_send() { + fn assert_send_sync() {} + fn assert_send() {} + + assert_send_sync::(); + assert_send::(); + } +} diff --git a/crates/levcs-store/src/drive.rs b/crates/levcs-store/src/drive.rs index e32a6dc..20d1fa5 100644 --- a/crates/levcs-store/src/drive.rs +++ b/crates/levcs-store/src/drive.rs @@ -36,14 +36,11 @@ use std::sync::Arc; use levcs_core::ObjectId; -use crate::format::{ - Frame, FrameHeader, JournalHeader, TransactionFramePayloadV1, JOURNAL_HEADER_LEN, -}; +use crate::format::{Frame, FrameHeader, TransactionFramePayloadV1}; use crate::index::{NamespaceCatalog, NamespaceLifecycle, NamespaceRecord, NamespaceStorageMode}; -use crate::journal::{Journal, TailStop}; -use crate::recovery::{self, ActiveJournalDisposition, FrameFacts, PayloadFacts}; +use crate::journal::Journal; +use crate::recovery::{self, FrameFacts, PayloadFacts}; use crate::segment::{self, RootLayout, ShardPaths}; -use crate::sys; use crate::types::{DurabilityCounterSnapshot, DurabilityCounters, NamespaceId, StoreError}; /// A single shard's journal, opened directly. @@ -90,8 +87,14 @@ const DRIVE_MANIFEST_CANDIDATES: usize = 8; const DRIVE_TERMINAL_STATUS_GRACE_MICROS: i64 = 900_000_000; /// What recovery concluded about a driven shard after reopening. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug)] pub struct DriveRecovery { + /// The full recovered state retained by the production entry point. + /// + /// The summary fields below are projections only. Keeping this value is + /// what lets the Wave B equivalence test compare the engine and drive + /// roots rather than merely comparing a sequence list. + pub recovered: recovery::RecoveredShard, /// Frames recovery adopted, in `shard_sequence` order. Always a contiguous /// prefix of what was appended — recovery stops at the first incomplete /// frame and discards everything after it, even a syntactically complete @@ -106,7 +109,7 @@ pub struct DriveRecovery { /// The full report recovery produced, verbatim. /// /// The four fields above are a convenience projection of it and nothing - /// more; [`DriveRecovery::from_report`] is the only thing that builds + /// more; [`DriveRecovery::from_recovered`] is the only thing that builds /// them, so they cannot disagree with this. /// /// Added after the Wave A review (record 2026-07-24-D). The seam @@ -121,6 +124,21 @@ pub struct DriveRecovery { pub report: recovery::ShardRecoveryReport, } +// Preserve the Wave A comparison contract for the diagnostic projection. The +// newly retained `RecoveredShard` contains mmap/file ownership whose identity +// is deliberately not value-comparable. +impl PartialEq for DriveRecovery { + fn eq(&self, other: &Self) -> bool { + self.adopted_shard_sequences == other.adopted_shard_sequences + && self.tail_stop_offset == other.tail_stop_offset + && self.quarantined_bytes == other.quarantined_bytes + && self.used_manifest_fallback == other.used_manifest_fallback + && self.report == other.report + } +} + +impl Eq for DriveRecovery {} + impl DriveRecovery { /// Project a recovery report into the seam's shape. /// @@ -133,16 +151,18 @@ impl DriveRecovery { /// records *why* the scan stopped; whether that stop discarded anything is /// a separate judgment (a clean journal always stops at its first unwritten /// byte), and only the scan site holds both halves. - fn from_report(report: recovery::ShardRecoveryReport, tail_stop_offset: Option) -> Self { + fn from_recovered(recovered: recovery::RecoveredShard) -> Self { + let report = &recovered.report; Self { adopted_shard_sequences: report.adopted_shard_sequences.clone(), - tail_stop_offset, + tail_stop_offset: recovered.tail_stop_offset, quarantined_bytes: report.quarantined_bytes, used_manifest_fallback: matches!( report.manifest_source, Some(recovery::ManifestSource::Fallback { .. }) ), - report, + report: report.clone(), + recovered, } } @@ -423,11 +443,16 @@ impl ShardDrive { /// supplies the real ref state. pub fn checkpoint(&mut self) -> Result { let facts = self.durable_facts()?; + if facts.is_empty() { + return Err(StoreError::Conflict( + "an empty Phase 1 shard has no manifest and cannot install a checkpoint".into(), + )); + } let created_at_micros = now_micros(); let mut catalog = NamespaceCatalog::new(); let mut receipts = Vec::with_capacity(facts.len()); - for fact in &facts { + for (fact, recovered) in &facts { if catalog.get(&fact.namespace).is_none() { catalog.bind(NamespaceRecord { namespace: fact.namespace, @@ -446,13 +471,16 @@ impl ShardDrive { fact.event_digest, )?; } - receipts.push(receipt_for(fact, created_at_micros)?); + receipts.push(receipt_for(fact, recovered, created_at_micros)?); } let checkpoint = crate::checkpoint::Checkpoint { root_uuid: self.root_uuid, shard_index: self.shard, - shard_committed_sequence: facts.last().map(|f| f.shard_sequence).unwrap_or(0), + shard_committed_sequence: facts + .last() + .map(|(facts, _)| facts.shard_sequence) + .unwrap_or(0), // The resume point of scope 3.6: `journal_id` accompanies the // offset because an offset alone is meaningless once the journal // has rotated, and recovery checks the identity before it trusts @@ -467,8 +495,92 @@ impl ShardDrive { let path = crate::checkpoint::install(&self.paths.checkpoints(), &checkpoint, &self.counters)?; - // Only after a successful install, so the store is never below its - // retention floor at an instant a crash could observe. + + // A checkpoint file is derived state, not authority by directory + // presence. Seal the active frames and publish both the new tail range + // and this checkpoint row in one manifest generation. Installing an + // intermediate seal-only generation here would let manifest_retain=2 + // evict the predecessor checkpoint manifest between two checkpoints. + let previous = segment::load_manifest_with_fallback(&self.paths, &self.root_uuid)? + .map(|(manifest, _)| manifest); + let generation = previous + .as_ref() + .map(|manifest| manifest.generation.saturating_add(1)) + .unwrap_or(1); + let mut retained_tail_ranges = previous + .as_ref() + .map(|manifest| manifest.retained_tail_ranges.clone()) + .unwrap_or_default(); + let active_path = self.journal.path().to_path_buf(); + let sealed_active = if let Some(last) = self.journal.last_appended_shard_sequence() { + let first = self.journal.header().first_shard_sequence; + let destination = + segment::seal_journal(&mut self.journal, &self.paths, generation, &self.counters)?; + let filename = destination + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| StoreError::Corruption("segment name is not UTF-8".into()))? + .to_string(); + retained_tail_ranges.push(crate::format::TailRange { + generation, + first_shard_sequence: first, + last_shard_sequence: last, + filename, + }); + true + } else { + false + }; + + let filename = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| StoreError::Corruption("checkpoint name is not UTF-8".into()))? + .to_string(); + let mut checkpoint_rows = previous + .as_ref() + .map(|manifest| manifest.checkpoints.clone()) + .unwrap_or_default(); + checkpoint_rows.retain(|(sequence, _)| *sequence != checkpoint.shard_committed_sequence); + checkpoint_rows.push((checkpoint.shard_committed_sequence, filename)); + let retain = self.checkpoint_retain.max(2) as usize; + if checkpoint_rows.len() > retain { + let drop_count = checkpoint_rows.len() - retain; + checkpoint_rows.drain(..drop_count); + } + let manifest = crate::format::Manifest { + root_uuid: self.root_uuid, + generation, + base_generation: previous + .as_ref() + .map(|manifest| manifest.base_generation) + .unwrap_or(0), + retained_tail_ranges, + index_runs: previous + .as_ref() + .map(|manifest| manifest.index_runs.clone()) + .unwrap_or_default(), + checkpoints: checkpoint_rows, + committed_shard_sequence: checkpoint.shard_committed_sequence, + }; + segment::install_manifest(&self.paths, &manifest, self.manifest_retain, &self.counters)?; + + if sealed_active { + segment::unlink_sealed_journal(&active_path, &self.paths, &self.counters)?; + self.journal = Journal::create( + &self.paths.active(), + fresh_id(b"journal"), + checkpoint.shard_committed_sequence.saturating_add(1), + self.shard, + self.root_uuid, + self.preallocate_bytes, + now_micros(), + Arc::clone(&self.counters), + )?; + } + + // Only after both the checkpoint and its authoritative manifest are + // fenced, so every retained manifest keeps a retained referent. crate::checkpoint::prune( &self.paths.checkpoints(), self.checkpoint_retain, @@ -480,7 +592,9 @@ impl ShardDrive { /// Every frame durably reachable in this shard right now, in /// `shard_sequence` order, each proved complete by the same readers /// recovery uses. - fn durable_facts(&self) -> Result, StoreError> { + fn durable_facts( + &self, + ) -> Result, StoreError> { let mut facts = Vec::new(); if let Some((manifest, _)) = segment::load_manifest_with_fallback(&self.paths, &self.root_uuid)? @@ -572,331 +686,31 @@ impl ShardDrive { /// prevent. A crash between the manifest install and the active-name /// unlink produced `0,1,2,3,0,1,2,3`. /// - /// The rule this function now follows is structural: every production - /// entry point in `recovery.rs` that belongs at this layer is called, and - /// the two that do not are named below with the reason. - /// - /// | `recovery.rs` entry point | here | - /// |---|---| - /// | `resolve_manifest` | step 2, with `SegmentFooterValidator` | - /// | `load_checkpoint` / `checkpoint_for_recovery` | step 3 | - /// | `validate_journal_binding` | every segment and the active journal | - /// | `classify_active_journal` / `complete_interrupted_seal` | scope 3.4 | - /// | `recover_journal_tail` | steps 5 to 7 (`journal::scan_journal` + `journal::quarantine_tail`) | - /// | `verify_shard_sequence` / `verify_repo_chain` | step 9 | - /// | `promote_receipt_visibility` | step 10 | - /// | `tail_outcome` | **not applicable.** It maps a scan to the frozen `RecoveryOutcome`; this seam returns the raw adoption set and A3's matrix does the oracle comparison itself. Calling it here would compute a verdict nothing reads. | - /// | `index_adopted_frames` | **not applicable.** The drive publishes no index — that is B1's `engine.rs`. Over scripted payloads `FrameFacts::objects` is empty, so the call would insert nothing and assert nothing. | - /// - /// What this function adds is the ordering between them and the - /// `DriveRecovery` projection, and nothing else. + /// The rule is now stronger and simpler: this function makes exactly one + /// call to [`recovery::recover_shard`], the same non-feature-gated entry + /// point the engine uses. It adds only the scripted-payload extractor and + /// the `DriveRecovery` summary projection. Manifest/checkpoint selection, + /// journal classification, replay, sequence verification, receipts, + /// index layering, and retained generation ownership have no drive-local + /// implementation to drift. pub fn reopen_through_recovery(root: &Path, shard: u16) -> Result { - let layout = RootLayout::new(root); - // Step 1: FORMAT, then the lock. Read first so an unopenable root is - // refused before a lock file is created inside it. - let marker = segment::read_format(&layout)?; - if shard >= marker.shard_count { - return Err(StoreError::FormatMismatch(format!( - "shard {shard} is outside the root's frozen topology of {} shards", - marker.shard_count - ))); - } - let root_uuid = marker.root_uuid; - let _lock = segment::lock_root(&layout)?; - let paths = layout.shard(shard); - let counters = Arc::new(DurabilityCounters::default()); - let mut report = recovery::ShardRecoveryReport::new(shard); - - // --- step 2: CURRENT, then the manifest fallback -------------------- - // - // Through `recovery::resolve_manifest` rather than - // `segment::load_manifest_with_fallback`: the two answer the same - // selection question, but only the former validates a referenced - // segment *beyond existence*, and only the former distinguishes "the - // pointer does not validate" from "the pointer validates and its - // referent does not". `recovery_manifest.rs` asserts the two agree. - let selection = recovery::resolve_manifest( - &paths, - &root_uuid, - &recovery::SegmentFooterValidator::new(root_uuid), - DRIVE_MANIFEST_CANDIDATES, - )?; - let manifest = match selection { - Some(selection) => { - report.manifest_generation = Some(selection.generation); - report.manifest_source = Some(selection.source.clone()); - Some(selection) - } - // No generation validated. Emptiness and total corruption are - // different facts and are never collapsed: a shard that has never - // sealed has no manifests and is healthy, while one whose every - // generation fails validation must refuse to open rather than - // silently behave like a fresh shard. - None if segment::list_manifest_generations(&paths)?.is_empty() => None, - None => { - return Err(StoreError::Corruption(format!( - "shard {shard} has manifest generations but none of the newest \ - {DRIVE_MANIFEST_CANDIDATES} validate; refusing to open rather \ - than treating a corrupt shard as an empty one" - ))) - } + let payload_facts = DrivePayloadFacts; + let config = recovery::RecoveryConfig { + max_manifest_candidates: DRIVE_MANIFEST_CANDIDATES, + manifest_retain: 2, + checkpoint_retain: DRIVE_CHECKPOINT_RETAIN, + journal_preallocate_bytes: DRIVE_PREALLOCATE_BYTES, + terminal_status_grace_micros: DRIVE_TERMINAL_STATUS_GRACE_MICROS, + max_active_index_entries: 4_000_000, + max_active_index_bytes: 512 * 1024 * 1024, + max_index_runs: 64, + max_open_index_runs: 32, + max_replay_frames: u64::MAX, + max_replay_bytes: u64::MAX, + payload_facts: &payload_facts, + projection_recovery_resolver: None, }; - let used_manifest_fallback = matches!( - report.manifest_source, - Some(recovery::ManifestSource::Fallback { .. }) - ); - - // --- step 3: the checkpoint ----------------------------------------- - // - // Three outcomes, kept distinct by the type: no generations at all is - // a shard that has not checkpointed yet and replays from the start; - // at least one validates and bounds the replay; every generation - // failing is an explicit offline-rebuild refusal and never an - // unbounded scan. - let load = recovery::load_checkpoint( - &paths.checkpoints(), - &root_uuid, - shard, - DRIVE_CHECKPOINT_RETAIN, - )?; - let checkpoint = - match recovery::checkpoint_for_recovery(load, &root_uuid, shard, &mut report) { - Some(checkpoint) => checkpoint, - None => { - debug_assert!(report.offline_rebuild_required); - return Err(StoreError::RecoveryRequired); - } - }; - // `Checkpoint::empty` also reports sequence 0, and sequence 0 is a - // legitimate committed sequence, so "was a checkpoint loaded" is read - // off the report — which only the `Loaded` branch sets — and never - // off the sequence number. - let checkpointed = report.checkpoint_sequence.is_some(); - let committed = checkpoint.shard_committed_sequence; - - let mut adopted_shard_sequences: Vec = Vec::new(); - // Only the frames actually replayed are re-verified. What a durable - // checkpoint already accounts for was verified when it was adopted; - // re-deriving it is what makes replay unbounded, which plan section - // 5.3 forbids. - let mut replayed: Vec = Vec::new(); - if checkpointed { - let mut accounted: Vec = checkpoint - .receipts - .iter() - .map(|receipt| receipt.shard_sequence) - .filter(|sequence| *sequence <= committed) - .collect(); - accounted.sort_unstable(); - adopted_shard_sequences.extend(accounted); - } - - // --- step 4: sealed segments named by the manifest, in order -------- - if let Some(selection) = &manifest { - for range in &selection.manifest.retained_tail_ranges { - let path = paths.segments().join(&range.filename); - let reader = segment::SegmentReader::open(&path, &root_uuid)?; - // The footer proves the *offset table* is intact. It says - // nothing about the frames, and nothing at all about which - // shard the file belongs to — so both are established here, - // from the journal header the sealed file still carries. - recovery::validate_journal_binding( - &reader.journal_header()?, - &root_uuid, - shard, - Some(&reader.footer().journal_id), - None, - ) - .map_err(StoreError::from)?; - - // A segment wholly at or below the checkpoint is already - // accounted for by it; step 4 replays segments *above* the - // checkpoint. - if checkpointed && range.last_shard_sequence <= committed { - continue; - } - - let sequences: Vec = reader - .footer() - .offsets - .iter() - .map(|(sequence, _, _)| *sequence) - .collect(); - for sequence in sequences { - if checkpointed && sequence <= committed { - continue; - } - // `SegmentReader::read_frame` re-reads the bytes and - // decodes them through `format::verify_complete` against - // this file's `journal_id`. Presence in the offset table - // is not evidence of anything. - let frame = reader.read_frame(sequence)?; - adopted_shard_sequences.push(sequence); - replayed.push(drive_frame_facts(&frame)?); - } - } - } - - // --- steps 5 to 7: the active journal ------------------------------- - // - // A shard with no active journal is not an error — a crash between - // sealing and opening the next journal leaves exactly that. - let mut tail_stop_offset = None; - let mut quarantined_bytes = 0; - if let Some(path) = active_journal_path(&paths)? { - let file = File::open(&path)?; - let mut header_bytes = [0u8; JOURNAL_HEADER_LEN]; - sys::pread_exact(&file, 0, &mut header_bytes)?; - let header = JournalHeader::decode(&header_bytes)?; - // `root_uuid` catches a file copied in from another instance; - // `shard_index` catches one moved between shards of this root, - // which no digest in the file can detect on its own. The sequence - // bound is what is covered *so far* — the checkpoint or the - // segments above it — not the checkpoint alone, because a journal - // opened after a rotation legitimately starts above the - // checkpoint's committed sequence. - let covered_through = adopted_shard_sequences - .last() - .copied() - .or(checkpointed.then_some(committed)); - recovery::validate_journal_binding(&header, &root_uuid, shard, None, covered_through) - .map_err(StoreError::from)?; - - // Scope 3.4: is this a live journal, or the residue of a seal that - // completed through the manifest install and lost only its final - // unlink? The two are opposite errors — replaying a sealed journal - // duplicates every frame in it, and treating a live one as sealed - // drops every frame the manifest does not yet name — so the - // decision is A2's `classify_active_journal` and never an - // ordering assumption here. - let disposition = match &manifest { - Some(selection) => recovery::classify_active_journal( - &paths, - &selection.manifest, - &header.journal_id, - &root_uuid, - )?, - // No manifest generation validated, so nothing can prove a - // seal completed. The journal is the only authority there is. - None => ActiveJournalDisposition::Replay, - }; - report.active_journal = Some(disposition.clone()); - - match disposition { - ActiveJournalDisposition::AlreadySealed { .. } => { - // The frames are already in the adopted set, taken from - // the manifest above. Finish the seal and do not scan. - drop(file); - recovery::complete_interrupted_seal(&path, &paths, &counters)?; - } - ActiveJournalDisposition::Replay => { - // Step 5 resumes at the checkpoint offset, and only when - // the checkpoint names *this* journal: an offset into a - // journal that has since rotated is meaningless, which is - // why the checkpoint records the identity alongside it. - let from = if checkpointed && checkpoint.active_journal_id == header.journal_id - { - checkpoint - .active_journal_offset - .max(JOURNAL_HEADER_LEN as u64) - } else { - JOURNAL_HEADER_LEN as u64 - }; - let (scan, record) = recovery::recover_journal_tail( - &layout.quarantine_dir(), - &file, - &header, - from, - &counters, - )?; - // Both halves of the record, from the one call that wrote - // it. The byte count without the path is evidence nobody - // can find, which is what `report.quarantined` being - // unsettable amounted to. - quarantined_bytes = record.as_ref().map(|r| r.bytes).unwrap_or(0); - report.quarantined = record.map(|r| r.path); - for scanned in &scan.frames { - let mut frame_bytes = vec![ - 0u8; - usize::try_from(scanned.len).map_err(|_| { - StoreError::from(crate::format::FrameError::Length) - })? - ]; - sys::pread_exact(&file, scanned.offset, &mut frame_bytes)?; - let frame = Frame::decode(&frame_bytes, &header.journal_id)?; - adopted_shard_sequences.push(scanned.shard_sequence); - replayed.push(drive_frame_facts(&frame)?); - } - // A stop is only a *tail* when something was actually - // discarded: a clean journal always stops at the first - // unwritten byte, and reporting that as a torn tail would - // make every ordinary crash look like a torn one. Each - // stop reason is matched explicitly — no catch-all, per - // the scope 5 charter. - tail_stop_offset = match scan.stop { - TailStop::EndOfPreallocation => None, - TailStop::NotAFrame => (quarantined_bytes > 0).then_some(scan.stop_offset), - TailStop::Incomplete(_) => Some(scan.stop_offset), - TailStop::ReadError => Some(scan.stop_offset), - }; - report.tail_stop = Some(scan.stop); - } - } - } - - // --- step 9: both halves, each with its own error type -------------- - // - // The shard domain is anchored on the checkpoint when there is one and - // on the first replayed frame otherwise; the repository domain is - // anchored on the checkpoint's catalog, extended by the first replayed - // frame of any repository the catalog does not already know. - verify_replayed_sequences( - &replayed, - checkpointed.then(|| committed.saturating_add(1)), - &checkpoint.catalog, - )?; - - // --- step 10: receipt visibility ------------------------------------ - // - // Over the checkpointed receipts and over one synthesized per replayed - // frame, so the promotion runs against both halves: an operation whose - // first visibility was durably captured must not move, and one that - // was only replayed must be promoted to recovery publication time. - let recovery_publication_micros = now_micros(); - let mut receipts = checkpoint.receipts.clone(); - for facts in &replayed { - let mut record = receipt_for(facts, recovery_publication_micros)?; - // Not durably captured: this frame was replayed, not restored - // from a checkpoint. `None` is what makes step 10 promote it. - record.first_receipt_visibility_micros = None; - receipts.push(record); - } - report.promotions = recovery::promote_receipt_visibility( - &mut receipts, - recovery_publication_micros, - DRIVE_TERMINAL_STATUS_GRACE_MICROS, - )?; - - report.adopted_shard_sequences = adopted_shard_sequences; - report.quarantined_bytes = quarantined_bytes; - debug_assert_eq!( - report.quarantined.is_some(), - quarantined_bytes > 0, - "a quarantine record and a non-zero byte count are the same fact \ - and are produced by the same call; they can never disagree" - ); - // Scope 3.8 step 12: computed, never a default. - report.ready = !report.offline_rebuild_required; - - debug_assert_eq!( - used_manifest_fallback, - matches!( - report.manifest_source, - Some(recovery::ManifestSource::Fallback { .. }) - ), - "the fallback decision and the reported manifest source must agree" - ); - Ok(DriveRecovery::from_report(report, tail_stop_offset)) + recovery::recover_shard(root, shard, &config).map(DriveRecovery::from_recovered) } } @@ -909,6 +723,7 @@ impl ShardDrive { /// transaction. fn receipt_for( facts: &FrameFacts, + recovered: &recovery::RecoveredPayloadFacts, first_visible_micros: i64, ) -> Result { Ok(crate::checkpoint::ReceiptRecord { @@ -918,7 +733,8 @@ fn receipt_for( repo_sequence: facts.repo_sequence, shard_sequence: facts.shard_sequence, current_authority: facts.current_authority, - objects_new: facts.objects.len() as u64, + refs: recovered.applied_refs.clone(), + objects_new: recovered.objects_new, retry_until_micros: facts.retry_until_micros, first_receipt_visibility_micros: Some(first_visible_micros), // Recomputed by `recovery::promote_receipt_visibility`, which may only @@ -980,33 +796,7 @@ impl PayloadFacts for DrivePayloadFacts { fn extend(&self, facts: &mut FrameFacts, payload: &[u8]) -> Result<(), StoreError> { let namespace = *facts.namespace.as_bytes(); match TransactionFramePayloadV1::decode_canonical(payload) { - Ok(decoded) => { - let extracted = decoded.facts()?; - if extracted.repo_id.0 != namespace { - return Err(StoreError::Corruption(format!( - "frame header names namespace {} but its payload names repository {}", - facts.namespace.to_hex(), - hex::encode(extracted.repo_id.0) - ))); - } - if extracted.repo_sequence != facts.repo_sequence { - return Err(StoreError::Corruption(format!( - "frame header carries repo_sequence {} but its payload carries {}", - facts.repo_sequence, extracted.repo_sequence - ))); - } - facts.event_digest = extracted.event_digest; - facts.previous_event_digest = extracted.previous_event_digest; - facts.creates_namespace = decoded.repository_create.is_some(); - facts.genesis_authority = decoded - .repository_create - .as_ref() - .map(|create| create.genesis_authority) - .unwrap_or(extracted.old_authority); - facts.current_authority = extracted.new_authority; - facts.retry_until_micros = extracted.retry_until_micros; - Ok(()) - } + Ok(_) => recovery::CanonicalPayloadFacts.extend(facts, payload), Err(_) => { facts.event_digest = synthetic_event_digest(&namespace, Some(facts.repo_sequence)); facts.previous_event_digest = @@ -1019,68 +809,28 @@ impl PayloadFacts for DrivePayloadFacts { } } } -} -fn drive_frame_facts(frame: &Frame) -> Result { - let mut facts = FrameFacts::from_header(&frame.header); - DrivePayloadFacts.extend(&mut facts, &frame.payload)?; - Ok(facts) -} + fn permits_implicit_namespace_anchor(&self) -> bool { + true + } -/// Recovery step 9 for the driven shard, through A2's two verifiers. -/// -/// They return distinct, non-interchangeable fault types by construction, so a -/// shard-domain fault and a repository-domain fault can never arrive as the -/// same error — which is what scope 4-A2 deliverable 4 requires and what a -/// single hand-rolled contiguity loop here would have destroyed. -/// -/// Both domains need an anchor, and the anchor is stated rather than implied: -/// -/// - `first_expected` is the checkpoint's committed sequence plus one when a -/// checkpoint was loaded. Without one there is nothing above which the shard -/// sequence is known to be contiguous, so the first replayed frame anchors -/// itself; a verifier started at zero would report a gap for every store -/// that has ever rotated, and one started at the first frame *with* a -/// checkpoint present would miss a hole between the two. -/// - `catalog` is the checkpoint's namespace catalog. A repository the catalog -/// does not know is anchored on its own first replayed frame, because the -/// drive seam has no earlier durable state to chain onto. -fn verify_replayed_sequences( - replayed: &[FrameFacts], - first_expected: Option, - catalog: &NamespaceCatalog, -) -> Result<(), StoreError> { - let anchor = match first_expected.or_else(|| replayed.first().map(|f| f.shard_sequence)) { - Some(anchor) => anchor, - None => return Ok(()), - }; - recovery::verify_shard_sequence(replayed, anchor).map_err(StoreError::from)?; - - let mut catalog = catalog.clone(); - let mut chained: Vec = Vec::with_capacity(replayed.len()); - for facts in replayed { - if catalog.get(&facts.namespace).is_none() { - catalog.bind(NamespaceRecord { - namespace: facts.namespace, - genesis_authority: facts.genesis_authority, - current_authority: facts.current_authority, - lifecycle: NamespaceLifecycle::Active, - storage_mode: NamespaceStorageMode::Full, - repo_sequence: facts.repo_sequence, - previous_event_digest: facts.event_digest, - })?; - } else { - chained.push(facts.clone()); + fn recovered(&self, payload: &[u8]) -> Result { + match TransactionFramePayloadV1::decode_canonical(payload) { + Ok(_) => recovery::CanonicalPayloadFacts.recovered(payload), + Err(_) => Ok(recovery::RecoveredPayloadFacts::default()), } } - recovery::verify_repo_chain(&chained, &catalog).map_err(StoreError::from)?; - Ok(()) } -/// The single `*.journal` in a shard's `active/`, if there is one. -/// -/// More than one is a corruption, not a choice to make: only the shard owner -/// names anything under its shard, and it opens exactly one journal at a time. +fn drive_frame_facts( + frame: &Frame, +) -> Result<(FrameFacts, recovery::RecoveredPayloadFacts), StoreError> { + let mut facts = FrameFacts::from_header(&frame.header); + DrivePayloadFacts.extend(&mut facts, &frame.payload)?; + let recovered = DrivePayloadFacts.recovered(&frame.payload)?; + Ok((facts, recovered)) +} + fn active_journal_path(paths: &ShardPaths) -> Result, StoreError> { let dir = match std::fs::read_dir(paths.active()) { Ok(dir) => dir, diff --git a/crates/levcs-store/src/index.rs b/crates/levcs-store/src/index.rs index d252b7a..a52537f 100644 --- a/crates/levcs-store/src/index.rs +++ b/crates/levcs-store/src/index.rs @@ -149,13 +149,19 @@ impl IndexKey { } } -/// Where the frame carrying an object lives. +/// Where the certified storage record carrying an object lives. /// -/// The location names the *frame*, not the object bytes inside it: the frame -/// is the unit the format certifies — its trailer certifies the whole — so a -/// reader validates the frame and then takes the object out of the validated -/// payload. Pointing straight at object bytes would let a reader return bytes -/// from a frame it never proved complete. +/// For inline transactions the record is a complete journal frame. For an +/// adopted projection it is a canonical, digest-bound staged chunk retained +/// by the same committed generation. In both cases the location names the +/// whole certified record, never the object bytes inside it: a reader +/// validates the frame or chunk first and only then extracts the object. +/// Pointing straight at object bytes would let a reader return bytes from a +/// record it never proved complete. +/// +/// The physical field names remain `frame_offset`/`frame_len` in storage +/// version 1. Source kind is resolved from the generation pin in the captured +/// `CommittedRoot`, so the packed index bytes do not need a new discriminant. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub struct IndexLocation { pub segment_generation: u64, diff --git a/crates/levcs-store/src/journal.rs b/crates/levcs-store/src/journal.rs index 054feba..8e402c8 100644 --- a/crates/levcs-store/src/journal.rs +++ b/crates/levcs-store/src/journal.rs @@ -243,13 +243,30 @@ impl Journal { ))); } let path = active_dir.join(format!("{first_shard_sequence}.journal")); - let mut file = File::options() - .read(true) - .write(true) - .create_new(true) - .open(&path)?; + if path.exists() { + let (journal, scan) = Self::open(&path, &root_uuid, Arc::clone(&counters))?; + let header = journal.header(); + if header.shard_index != shard_index + || header.first_shard_sequence != first_shard_sequence + || header.preallocated_len != preallocated_len + || !scan.frames.is_empty() + || scan.stop_offset != JOURNAL_HEADER_LEN as u64 + { + return Err(StoreError::Corruption(format!( + "occupied fresh-journal target {} is not the requested empty journal", + path.display() + ))); + } + sys::fsync_dir(active_dir, &counters)?; + return Ok(journal); + } - let header = JournalHeader { + // Target-scoped and deterministic: one interrupted creation can leave + // at most one invisible construction artifact. Re-entry either reuses + // a fully fenced empty journal or reconstructs a partial pre-publish + // file in place; it never allocates another temp name. + let temporary = active_dir.join(format!(".{first_shard_sequence}.journal.tmp")); + let requested_header = JournalHeader { shard_index, root_uuid, journal_id, @@ -257,11 +274,58 @@ impl Journal { preallocated_len, created_at_micros, }; - sys::preallocate(&file, preallocated_len)?; - let encoded = header.encode()?; - sys::seek_to(&mut file, 0)?; - sys::write_vectored_all(&mut file, &[IoSlice::new(&encoded)], &counters)?; - sys::fdatasync(&file, &counters)?; + let existed = temporary.exists(); + let mut file = File::options() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&temporary)?; + let decoded_header = if existed { + let mut bytes = [0u8; JOURNAL_HEADER_LEN]; + sys::pread_exact(&file, 0, &mut bytes) + .ok() + .and_then(|()| JournalHeader::decode(&bytes).ok()) + } else { + None + }; + let header = if let Some(header) = decoded_header { + if header.root_uuid != root_uuid + || header.shard_index != shard_index + || header.first_shard_sequence != first_shard_sequence + || header.preallocated_len != preallocated_len + { + return Err(StoreError::Corruption(format!( + "fresh-journal temp {} belongs to a different target", + temporary.display() + ))); + } + let scan = scan_journal(&file, &header, JOURNAL_HEADER_LEN as u64); + if !scan.frames.is_empty() || scan.stop_offset != JOURNAL_HEADER_LEN as u64 { + return Err(StoreError::Corruption(format!( + "fresh-journal temp {} is not empty", + temporary.display() + ))); + } + // Re-fence before publication. Exact bytes in page cache do not + // prove the previous process reached its fdatasync. + sys::fdatasync(&file, &counters)?; + header + } else { + // The deterministic temp is unpublished and target-scoped. A bad + // or partial header can only be an interrupted construction for + // this missing final name, so finish that construction in place. + sys::truncate(&file, 0, &counters)?; + sys::preallocate(&file, preallocated_len)?; + let encoded = requested_header.encode()?; + sys::seek_to(&mut file, 0)?; + sys::write_vectored_all(&mut file, &[IoSlice::new(&encoded)], &counters)?; + sys::fdatasync(&file, &counters)?; + requested_header + }; + if let Err(error) = sys::rename_noreplace(&temporary, &path) { + return Err(error.into()); + } sys::fsync_dir(active_dir, &counters)?; Ok(Self { diff --git a/crates/levcs-store/src/lib.rs b/crates/levcs-store/src/lib.rs index 64eb113..c421065 100644 --- a/crates/levcs-store/src/lib.rs +++ b/crates/levcs-store/src/lib.rs @@ -33,6 +33,7 @@ //! snapshots and events, never to these bytes. pub mod checkpoint; +pub mod completion; pub mod engine; pub mod failpoints; pub mod format; @@ -40,6 +41,7 @@ pub mod index; pub mod journal; pub mod options; pub mod recovery; +pub mod roots; pub mod segment; pub mod snapshot; pub mod staging; @@ -57,9 +59,21 @@ mod sys; #[cfg(feature = "store-internals")] pub mod drive; +pub use completion::{CompletionWaiter, SharedCompletion}; pub use engine::{CheckpointLease, StoreEngine}; pub use options::{StoreDirectoryAttributes, StoreOptions}; +pub use roots::{ + CommittedRoot, GenerationId, IndexDeltaLayer, LayeredObjectIndex, OperationKey, + OperationStatusMetricSnapshot, OperationStatusMetrics, OperationStatusRoot, PinnedFile, + ProjectionArtifactFormat, ReceiptTombstone, RepoState, RetainedGeneration, RetainedIndexRun, + RetainedObjectSource, RetainedProjectionArtifact, RetainedReceipt, RetainedSegment, + RetainedTail, ShardSubtree, StatusEntry, StatusPhase, StatusReservation, TerminalStatusEntry, +}; pub use snapshot::RepoSnapshot; +pub use staging::{ + ProjectionAdoption, ProjectionAdoptionOutcome, ProjectionAdoptionResolution, + ProjectionArtifact, RecoveredProjectionOutcome, RecoveredProjectionResolution, +}; pub use transaction::{StagedObject, ValidatedTransaction, ValidatedTransactionBuilder}; pub use types::{ AppliedRef, CommitEvidenceSigner, CommitReceipt, DurabilityCounterSnapshot, DurabilityCounters, diff --git a/crates/levcs-store/src/options.rs b/crates/levcs-store/src/options.rs index 24f1df5..a57ac09 100644 --- a/crates/levcs-store/src/options.rs +++ b/crates/levcs-store/src/options.rs @@ -74,6 +74,24 @@ pub struct StoreOptions { pub max_refs_per_transaction: u32, pub max_projection_objects: u64, pub max_projection_bytes: u64, + pub max_projection_chunks: u32, + /// Maximum number of transient entries in `OperationStatusRoot`. + pub max_status_entries: u64, + + // --- projection staging --------------------------------------------- + pub staging_max_sessions_per_principal: u32, + pub staging_max_sessions_global: u32, + pub staging_max_objects_per_principal: u64, + pub staging_max_objects_global: u64, + pub staging_max_bytes_per_principal: u64, + pub staging_max_bytes_global: u64, + pub staging_max_files_per_session: u64, + pub staging_max_files_per_principal: u64, + pub staging_max_files_global: u64, + pub staging_session_max_age_micros: i64, + pub staging_finalize_margin_micros: i64, + pub minimum_projection_transfer_bytes_per_second: u64, + pub staging_max_compaction_debt_bytes: u64, // --- retention and time --------------------------------------------- pub max_retry_window_micros: i64, @@ -116,6 +134,24 @@ impl Default for StoreOptions { max_refs_per_transaction: 4_096, max_projection_objects: 100_000_000, max_projection_bytes: 1024 * 1024 * 1024 * 1024, + max_projection_chunks: 1_000_000, + max_status_entries: 1_000_000, + staging_max_sessions_per_principal: 2, + staging_max_sessions_global: 16, + staging_max_objects_per_principal: 200_000_000, + staging_max_objects_global: 800_000_000, + staging_max_bytes_per_principal: 2 * 1024 * 1024 * 1024 * 1024, + staging_max_bytes_global: 8 * 1024 * 1024 * 1024 * 1024, + staging_max_files_per_session: 1_000_002, + staging_max_files_per_principal: 2_000_000, + staging_max_files_global: 8_000_000, + // A maximal 1 TiB projection at the supported 16 MiB/s floor + // takes a little over 18 hours. The 24-hour session horizon + // leaves room for the five-minute finalize margin. + staging_session_max_age_micros: 24 * 60 * 60 * 1_000_000, + staging_finalize_margin_micros: 5 * 60 * 1_000_000, + minimum_projection_transfer_bytes_per_second: 16 * 1024 * 1024, + staging_max_compaction_debt_bytes: 8 * 1024 * 1024 * 1024 * 1024, // Plan §7 stage 3 initial hosted defaults: +/-60 s skew, 1 s timer // resolution, at least 121 s replay retention. max_retry_window_micros: 900_000_000, @@ -191,12 +227,100 @@ impl StoreOptions { self.max_objects_per_transaction >= 1, "max_objects_per_transaction must be nonzero" ); + require!( + self.max_projection_objects >= 1, + "max_projection_objects must be nonzero" + ); + require!( + self.max_projection_bytes >= 1, + "max_projection_bytes must be nonzero" + ); + require!( + self.max_projection_chunks >= 1, + "max_projection_chunks must be nonzero" + ); + require!( + self.max_status_entries >= 1, + "max_status_entries must be nonzero" + ); require!( self.max_refs_per_transaction >= 1 && (self.max_refs_per_transaction as usize) <= levcs_protocol::v2::MAX_REF_UPDATES, "max_refs_per_transaction must be in 1..={}", levcs_protocol::v2::MAX_REF_UPDATES ); + require!( + self.staging_max_sessions_per_principal >= 1 + && self.staging_max_sessions_per_principal <= self.staging_max_sessions_global, + "staging session limits must be nonzero and per-principal <= global" + ); + require!( + self.staging_max_objects_per_principal >= self.max_projection_objects + && self.staging_max_objects_per_principal <= self.staging_max_objects_global, + "staging object limits must admit one maximal projection and \ + per-principal must be <= global" + ); + require!( + self.staging_max_bytes_per_principal >= self.max_projection_bytes + && self.staging_max_bytes_per_principal <= self.staging_max_bytes_global, + "staging byte limits must admit one maximal projection and \ + per-principal must be <= global" + ); + require!( + self.staging_max_files_per_session >= u64::from(self.max_projection_chunks) + 2 + && self.staging_max_files_per_session <= self.staging_max_files_per_principal + && self.staging_max_files_per_principal <= self.staging_max_files_global, + "staging file limits must admit one maximal projection's chunks, \ + session record, and manifest, and \ + per-session <= per-principal <= global" + ); + require!( + self.staging_session_max_age_micros > 0, + "staging_session_max_age_micros must be positive" + ); + require!( + self.staging_finalize_margin_micros >= 0, + "staging_finalize_margin_micros must not be negative" + ); + require!( + self.minimum_projection_transfer_bytes_per_second >= 1, + "minimum_projection_transfer_bytes_per_second must be nonzero" + ); + require!( + self.staging_max_compaction_debt_bytes >= self.max_projection_bytes, + "staging_max_compaction_debt_bytes must admit one maximal projection" + ); + + let transfer_seconds = self + .max_projection_bytes + .checked_add(self.minimum_projection_transfer_bytes_per_second - 1) + .ok_or_else(|| { + StoreError::InvalidConfiguration( + "projection transfer ceiling division overflows u64".into(), + ) + })? + / self.minimum_projection_transfer_bytes_per_second; + let transfer_micros = transfer_seconds.checked_mul(1_000_000).ok_or_else(|| { + StoreError::InvalidConfiguration("projection transfer duration overflows u64".into()) + })?; + let transfer_micros = i64::try_from(transfer_micros).map_err(|_| { + StoreError::InvalidConfiguration( + "projection transfer duration does not fit signed microseconds".into(), + ) + })?; + let required_session_age = transfer_micros + .checked_add(self.staging_finalize_margin_micros) + .ok_or_else(|| { + StoreError::InvalidConfiguration( + "projection transfer duration plus finalize margin overflows i64".into(), + ) + })?; + require!( + required_session_age <= self.staging_session_max_age_micros, + "max projection requires {required_session_age}us at the supported transfer floor, \ + exceeding staging_session_max_age_micros ({})", + self.staging_session_max_age_micros + ); require!( self.clock_skew_micros > 0, @@ -323,6 +447,46 @@ mod tests { assert!(o.validate().is_err()); } + #[test] + fn zero_status_or_staging_bounds_are_refused() { + let mut o = valid(); + o.max_status_entries = 0; + assert!(o.validate().is_err()); + + let mut o = valid(); + o.staging_max_sessions_global = 0; + assert!(o.validate().is_err()); + + let mut o = valid(); + o.minimum_projection_transfer_bytes_per_second = 0; + assert!(o.validate().is_err()); + } + + #[test] + fn staging_feasibility_is_checked_before_startup() { + let mut o = valid(); + o.staging_session_max_age_micros = o.staging_finalize_margin_micros; + assert!( + o.validate().is_err(), + "a session too short for one maximal transfer must be refused" + ); + } + + #[test] + fn staging_nested_bounds_must_be_monotonic() { + let mut o = valid(); + o.staging_max_bytes_per_principal = o.max_projection_bytes - 1; + assert!(o.validate().is_err()); + + let mut o = valid(); + o.staging_max_files_per_principal = o.staging_max_files_per_session - 1; + assert!(o.validate().is_err()); + + let mut o = valid(); + o.staging_max_files_per_session = u64::from(o.max_projection_chunks) + 1; + assert!(o.validate().is_err()); + } + #[test] fn shard_routing_is_stable_and_in_range() { for count in [1u16, 2, 4, 7, 256, 1024] { diff --git a/crates/levcs-store/src/recovery.rs b/crates/levcs-store/src/recovery.rs index a2c49f2..07a95bc 100644 --- a/crates/levcs-store/src/recovery.rs +++ b/crates/levcs-store/src/recovery.rs @@ -75,17 +75,35 @@ //! device actually retained, which is the input A3's external ACK //! reconciliation needs to catch a device that lied about a flush. +use std::collections::{BTreeMap, BTreeSet}; use std::fs::File; use std::path::{Path, PathBuf}; +use std::sync::Arc; -use levcs_core::ObjectId; +use im::Vector; +use levcs_core::{ObjectId, ObjectType}; use levcs_protocol::oracle::{self, RecoveredTailFact, RecoveryOutcome}; +use levcs_protocol::v2::{RefMutation, RefTarget, StagedProjectionInstallV1, TypedRefCas}; -use crate::checkpoint::{Checkpoint, CheckpointLoad, ReceiptRecord}; -use crate::format::{CurrentPointer, FrameError, FrameHeader, JournalHeader, Manifest}; -use crate::index::{IndexDelta, IndexKey, IndexLocation, NamespaceCatalog}; -use crate::journal::{QuarantinedTail, ScannedFrame, TailScan}; -use crate::segment::{self, ShardPaths}; +use crate::checkpoint::{Checkpoint, CheckpointError, CheckpointLoad, ReceiptRecord, RefRecord}; +use crate::format::{ + CurrentPointer, Frame, FrameError, FrameHeader, FrameObjectsV1, JournalHeader, Manifest, + TailRange, TransactionFramePayloadV1, JOURNAL_HEADER_LEN, +}; +use crate::index::{IndexDelta, IndexKey, IndexLocation, IndexRun, NamespaceCatalog}; +use crate::journal::{Journal, QuarantinedTail, ScannedFrame, TailScan, TailStop}; +use crate::options::StoreOptions; +use crate::roots::{ + GenerationId, IndexDeltaLayer, LayeredObjectIndex, PinnedFile, RetainedGeneration, + RetainedIndexRun, RetainedProjectionArtifact, RetainedSegment as RootRetainedSegment, + RetainedTail, +}; +use crate::segment::{self, RootLayout, SegmentReader, ShardPaths}; +use crate::staging::{ + ProjectionRecoveryResolver, RecoveredProjectionArtifacts, RecoveredProjectionOutcome, + RecoveredProjectionResolution, +}; +use crate::sys; use crate::types::{DurabilityCounters, NamespaceId, OperationId, StoreError}; // =========================================================================== @@ -115,6 +133,30 @@ pub enum ReferencedFileFault { /// worth the open on every startup. pub trait ReferencedFileValidator { fn validate(&self, dir: &Path, filename: &str) -> Result<(), ReferencedFileFault>; + + fn validate_segment(&self, dir: &Path, range: &TailRange) -> Result<(), ReferencedFileFault> { + self.validate(dir, &range.filename) + } + + fn validate_index( + &self, + dir: &Path, + generation: u64, + filename: &str, + ) -> Result<(), ReferencedFileFault> { + let _ = generation; + self.validate(dir, filename) + } + + fn validate_checkpoint( + &self, + dir: &Path, + shard_sequence: u64, + filename: &str, + ) -> Result<(), ReferencedFileFault> { + let _ = shard_sequence; + self.validate(dir, filename) + } } pub struct PresenceAndLengthValidator { @@ -188,6 +230,107 @@ impl ReferencedFileValidator for SegmentFooterValidator { } } +/// Full validator used by the production entry point. +/// +/// Manifest fallback is decided only after every kind of referent has passed +/// its real reader. Validating index/checkpoint files later would turn a bad +/// `CURRENT` generation into a hard open failure instead of falling back to a +/// valid predecessor, contrary to recovery step 2. +struct ProductionReferentValidator { + root_uuid: [u8; 16], + shard_index: u16, +} + +impl ReferencedFileValidator for ProductionReferentValidator { + fn validate(&self, dir: &Path, filename: &str) -> Result<(), ReferencedFileFault> { + PresenceAndLengthValidator { minimum_len: 1 }.validate(dir, filename)?; + let path = dir.join(filename); + match path.extension().and_then(|extension| extension.to_str()) { + Some("seg") => { + let reader = SegmentReader::open(&path, &self.root_uuid) + .map_err(|error| ReferencedFileFault::Invalid(error.to_string()))?; + let header = reader + .journal_header() + .map_err(|error| ReferencedFileFault::Invalid(error.to_string()))?; + validate_journal_binding( + &header, + &self.root_uuid, + self.shard_index, + Some(&reader.footer().journal_id), + None, + ) + .map_err(|error| ReferencedFileFault::Invalid(error.to_string())) + } + Some("idx") => IndexRun::open(&path, &self.root_uuid) + .map(|_| ()) + .map_err(|error| ReferencedFileFault::Invalid(error.to_string())), + Some(crate::checkpoint::CHECKPOINT_EXTENSION) => { + let bytes = std::fs::read(&path) + .map_err(|error| ReferencedFileFault::Unreadable(error.to_string()))?; + Checkpoint::decode(&bytes, &self.root_uuid, self.shard_index) + .map(|_| ()) + .map_err(|error| ReferencedFileFault::Invalid(error.to_string())) + } + _ => Err(ReferencedFileFault::Invalid( + "manifest referent has an unknown file extension".into(), + )), + } + } + + fn validate_segment(&self, dir: &Path, range: &TailRange) -> Result<(), ReferencedFileFault> { + self.validate(dir, &range.filename)?; + let reader = SegmentReader::open(&dir.join(&range.filename), &self.root_uuid) + .map_err(|error| ReferencedFileFault::Invalid(error.to_string()))?; + let footer = reader.footer(); + if footer.generation != range.generation + || footer.first_shard_sequence != range.first_shard_sequence + || footer.last_shard_sequence != range.last_shard_sequence + { + return Err(ReferencedFileFault::Invalid(format!( + "segment footer tuple ({}, {}, {}) disagrees with manifest row ({}, {}, {})", + footer.generation, + footer.first_shard_sequence, + footer.last_shard_sequence, + range.generation, + range.first_shard_sequence, + range.last_shard_sequence + ))); + } + Ok(()) + } + + fn validate_index( + &self, + dir: &Path, + generation: u64, + filename: &str, + ) -> Result<(), ReferencedFileFault> { + self.validate(dir, filename)?; + let run = IndexRun::open(&dir.join(filename), &self.root_uuid) + .map_err(|error| ReferencedFileFault::Invalid(error.to_string()))?; + if run.generation() != generation { + return Err(ReferencedFileFault::Invalid(format!( + "index run declares generation {}, manifest row declares {generation}", + run.generation() + ))); + } + Ok(()) + } + + fn validate_checkpoint( + &self, + _dir: &Path, + _shard_sequence: u64, + _filename: &str, + ) -> Result<(), ReferencedFileFault> { + // Checkpoints are derived state selected in normative recovery step 3. + // Missing, corrupt, or tuple-mismatched rows fall back within this + // manifest's ordered checkpoint list; they do not invalidate the + // transaction-authority tail closure selected in step 2. + Ok(()) + } +} + /// Why recovery did not use `CURRENT`'s referent. /// /// The three acceptance branches of scope 4-A2 are distinct variants and are @@ -292,28 +435,30 @@ fn load_generation( cause: e.to_string(), } })?; + validate_manifest_sequence_coverage(&manifest) + .map_err(|cause| ManifestFallbackReason::ReferentCorrupt { generation, cause })?; for range in &manifest.retained_tail_ranges { files - .validate(&paths.segments(), &range.filename) + .validate_segment(&paths.segments(), range) .map_err(|cause| ManifestFallbackReason::ReferentFileInvalid { generation, filename: range.filename.clone(), cause, })?; } - for (_, filename) in &manifest.index_runs { + for (run_generation, filename) in &manifest.index_runs { files - .validate(&paths.indexes(), filename) + .validate_index(&paths.indexes(), *run_generation, filename) .map_err(|cause| ManifestFallbackReason::ReferentFileInvalid { generation, filename: filename.clone(), cause, })?; } - for (_, filename) in &manifest.checkpoints { + for (shard_sequence, filename) in &manifest.checkpoints { files - .validate(&paths.checkpoints(), filename) + .validate_checkpoint(&paths.checkpoints(), *shard_sequence, filename) .map_err(|cause| ManifestFallbackReason::ReferentFileInvalid { generation, filename: filename.clone(), @@ -323,6 +468,60 @@ fn load_generation( Ok(manifest) } +fn validate_manifest_sequence_coverage(manifest: &Manifest) -> Result<(), String> { + if manifest.base_generation != 0 { + return Err(format!( + "Phase 1 recovery cannot interpret nonzero base_generation {}", + manifest.base_generation + )); + } + + if manifest + .checkpoints + .iter() + .any(|(sequence, _)| *sequence > manifest.committed_shard_sequence) + { + return Err("checkpoint row is newer than the manifest's committed sequence".into()); + } + + let ranges = &manifest.retained_tail_ranges; + if ranges.is_empty() { + return Err( + "Phase 1 manifests may not have an empty retained tail; an empty shard has no manifest" + .into(), + ); + } + if ranges[0].first_shard_sequence != 0 { + return Err(format!( + "Phase 1 retained tail begins at {}, expected 0", + ranges[0].first_shard_sequence + )); + } + for pair in ranges.windows(2) { + let expected = pair[0] + .last_shard_sequence + .checked_add(1) + .ok_or_else(|| "retained tail sequence overflow".to_string())?; + if pair[1].first_shard_sequence != expected { + return Err(format!( + "retained tail gap or overlap: generation {} ends at {}, generation {} begins at {}", + pair[0].generation, + pair[0].last_shard_sequence, + pair[1].generation, + pair[1].first_shard_sequence + )); + } + } + let last = ranges.last().expect("non-empty").last_shard_sequence; + if last != manifest.committed_shard_sequence { + return Err(format!( + "retained tail ends at {last}, manifest commits through {}", + manifest.committed_shard_sequence + )); + } + Ok(()) +} + /// Recovery step 2, with the fault taxonomy scope 4-A2 requires. /// /// The byte reads are `segment::read_current` and `segment::read_manifest`; @@ -342,9 +541,9 @@ pub fn resolve_manifest( max_candidates: usize, ) -> Result, StoreError> { let mut rejected: Vec<(u64, ManifestFallbackReason)> = Vec::new(); - let fallback_reason: Option; + let current = segment::read_current(paths, root_uuid)?; - match segment::read_current(paths, root_uuid)? { + match current { Some(pointer) => match load_generation(paths, pointer.generation, root_uuid, files) { Ok(manifest) => { return Ok(Some(ManifestSelection { @@ -357,35 +556,69 @@ pub fn resolve_manifest( } Err(reason) => { rejected.push((pointer.generation, reason.clone())); - fallback_reason = Some(reason); + // A valid pointer naming a missing or corrupt manifest gives + // recovery no closure to compare with an older generation. + // Guessing would silently roll back acknowledged transactions. + let refused_manifest = + match segment::read_manifest(paths, pointer.generation, root_uuid) { + Ok(manifest) if validate_manifest_sequence_coverage(&manifest).is_ok() => { + manifest + } + Ok(_) | Err(_) => return Ok(None), + }; + + let mut generations = segment::list_manifest_generations(paths)?; + generations.sort_unstable_by(|a, b| b.cmp(a)); + for generation in generations.into_iter().take(max_candidates.max(1)) { + if generation == pointer.generation { + continue; + } + match load_generation(paths, generation, root_uuid, files) { + Ok(manifest) + if manifest.committed_shard_sequence + == refused_manifest.committed_shard_sequence + && manifest.retained_tail_ranges + == refused_manifest.retained_tail_ranges => + { + return Ok(Some(ManifestSelection { + manifest, + generation, + path: paths.manifest(generation), + source: ManifestSource::Fallback { + reason: reason.clone(), + }, + rejected, + })); + } + Ok(_) => return Ok(None), + Err(cause) => rejected.push((generation, cause)), + } + } + return Ok(None); } }, - None => fallback_reason = Some(classify_current_failure(paths, root_uuid)), - } - - let reason = fallback_reason.expect("a fallback is only reached after a recorded failure"); - let mut generations = segment::list_manifest_generations(paths)?; - generations.sort_unstable_by(|a, b| b.cmp(a)); - for generation in generations.into_iter().take(max_candidates.max(1)) { - if rejected.iter().any(|(g, _)| *g == generation) { - continue; - } - match load_generation(paths, generation, root_uuid, files) { - Ok(manifest) => { - return Ok(Some(ManifestSelection { + None => { + let reason = classify_current_failure(paths, root_uuid); + let mut generations = segment::list_manifest_generations(paths)?; + generations.sort_unstable_by(|a, b| b.cmp(a)); + let Some(generation) = generations.into_iter().next() else { + return Ok(None); + }; + match load_generation(paths, generation, root_uuid, files) { + Ok(manifest) => Ok(Some(ManifestSelection { manifest, generation, path: paths.manifest(generation), - source: ManifestSource::Fallback { - reason: reason.clone(), - }, + source: ManifestSource::Fallback { reason }, rejected, - })) + })), + Err(cause) => { + rejected.push((generation, cause)); + Ok(None) + } } - Err(cause) => rejected.push((generation, cause)), } } - Ok(None) } // =========================================================================== @@ -408,6 +641,80 @@ pub fn load_checkpoint( crate::checkpoint::load_newest_valid(checkpoint_dir, root_uuid, shard_index, max_candidates) } +/// Load checkpoints only through the selected manifest's authoritative rows. +/// +/// A valid file in the directory is not sufficient authority: it may belong +/// to a newer manifest generation that recovery rejected, or may be an orphan +/// left between checkpoint installation and manifest installation. Selecting +/// it would publish derived state beyond the selected journal/segment closure. +fn load_authoritative_checkpoint( + paths: &ShardPaths, + selection: Option<&ManifestSelection>, + root_uuid: &[u8; 16], + shard_index: u16, + checkpoint_retain: u32, +) -> Result { + let Some(selection) = selection else { + let stray = crate::checkpoint::list_generations(&paths.checkpoints())?; + if stray.is_empty() { + return Ok(CheckpointLoad::Empty); + } + return Ok(CheckpointLoad::OfflineRebuildRequired { + rejected: stray + .into_iter() + .map(|(_, path)| { + ( + path, + CheckpointError::Body( + "checkpoint is unreferenced because no authoritative manifest exists", + ), + ) + }) + .collect(), + }); + }; + + if selection.manifest.checkpoints.is_empty() { + return Ok(CheckpointLoad::Empty); + } + + let max_candidates = (checkpoint_retain.max(2) as usize) * 4; + let mut rejected = Vec::new(); + for (declared_sequence, filename) in selection + .manifest + .checkpoints + .iter() + .rev() + .take(max_candidates.max(1)) + { + let path = paths.checkpoints().join(filename); + let bytes = match std::fs::read(&path) { + Ok(bytes) => bytes, + Err(error) => { + rejected.push((path, CheckpointError::Unreadable(error.to_string()))); + continue; + } + }; + match Checkpoint::decode(&bytes, root_uuid, shard_index) { + Ok(checkpoint) if checkpoint.shard_committed_sequence == *declared_sequence => { + return Ok(CheckpointLoad::Loaded { + checkpoint: Box::new(checkpoint), + path, + rejected, + }); + } + Ok(_) => rejected.push(( + path, + CheckpointError::Body( + "checkpoint shard sequence disagrees with authoritative manifest row", + ), + )), + Err(error) => rejected.push((path, error)), + } + } + Ok(CheckpointLoad::OfflineRebuildRequired { rejected }) +} + // =========================================================================== // Scope 3.4 — the interrupted seal // =========================================================================== @@ -671,9 +978,120 @@ impl FrameFacts { } } +/// Payload state needed for complete root reconstruction but not for the +/// sequence-verification seam represented by [`FrameFacts`]. +/// +/// Kept separate so adding D0-B recovery state does not break Wave A's public +/// `FrameFacts` struct literals. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RecoveredPayloadFacts { + pub ref_updates: Vec, + pub applied_refs: Vec, + pub objects_new: u64, + pub first_receipt_visibility_micros: Option, + pub staged_projection_install: Option, + pub objects: Vec<(ObjectId, u8)>, +} + /// Extracts the payload half of [`FrameFacts`]. pub trait PayloadFacts { fn extend(&self, facts: &mut FrameFacts, payload: &[u8]) -> Result<(), StoreError>; + + fn recovered(&self, _payload: &[u8]) -> Result { + Ok(RecoveredPayloadFacts::default()) + } + + /// The drive seam predates canonical transaction payloads and can create + /// opaque scripted frames. Production extractors leave this false: + /// appearing without a repository-create frame is then corruption. + fn permits_implicit_namespace_anchor(&self) -> bool { + false + } +} + +/// Production payload extractor. +/// +/// A physical frame is complete without interpreting its payload, but a frame +/// cannot enter a recovered logical root until its transaction payload has +/// also passed the canonical decoder. The drive seam injects its explicit +/// scripted-payload extractor through [`RecoveryConfig`]; production callers +/// use this one. +#[derive(Copy, Clone, Debug, Default)] +pub struct CanonicalPayloadFacts; + +impl PayloadFacts for CanonicalPayloadFacts { + fn extend(&self, facts: &mut FrameFacts, payload: &[u8]) -> Result<(), StoreError> { + let decoded = TransactionFramePayloadV1::decode_canonical(payload)?; + let extracted = decoded.facts()?; + if extracted.repo_id.0 != *facts.namespace.as_bytes() { + return Err(StoreError::Corruption(format!( + "frame header names namespace {} but its payload names repository {}", + facts.namespace.to_hex(), + hex::encode(extracted.repo_id.0) + ))); + } + if extracted.repo_sequence != facts.repo_sequence { + return Err(StoreError::Corruption(format!( + "frame header carries repo_sequence {} but its payload carries {}", + facts.repo_sequence, extracted.repo_sequence + ))); + } + + facts.event_digest = extracted.event_digest; + facts.previous_event_digest = extracted.previous_event_digest; + facts.creates_namespace = decoded.repository_create.is_some(); + facts.genesis_authority = decoded + .repository_create + .as_ref() + .map(|create| create.genesis_authority) + .unwrap_or(extracted.old_authority); + facts.current_authority = extracted.new_authority; + facts.retry_until_micros = extracted.retry_until_micros; + facts.objects = match decoded.objects { + FrameObjectsV1::Inline(objects) => objects + .into_iter() + .map(|object| (object.object_id, object_type_code(object.object_type))) + .collect(), + FrameObjectsV1::StagedProjectionInstall(_) => Vec::new(), + }; + Ok(()) + } + + fn recovered(&self, payload: &[u8]) -> Result { + let decoded = TransactionFramePayloadV1::decode_canonical(payload)?; + let extracted = decoded.facts()?; + let mut recovered = RecoveredPayloadFacts { + ref_updates: decoded.ref_cas.clone(), + applied_refs: decoded.committed.transaction.refs.clone(), + objects_new: extracted.objects_new, + first_receipt_visibility_micros: Some(extracted.first_receipt_visibility_micros), + ..RecoveredPayloadFacts::default() + }; + match decoded.objects { + FrameObjectsV1::Inline(objects) => { + recovered.objects = objects + .into_iter() + .map(|object| (object.object_id, object_type_code(object.object_type))) + .collect(); + } + FrameObjectsV1::StagedProjectionInstall(install) => { + recovered.staged_projection_install = Some(install); + } + } + Ok(recovered) + } +} + +fn object_type_code(value: ObjectType) -> u8 { + // Exhaustive so a new logical object type cannot acquire an accidental + // recovered-index representation. + match value { + ObjectType::Blob => 1, + ObjectType::Tree => 2, + ObjectType::Commit => 3, + ObjectType::Release => 4, + ObjectType::Authority => 5, + } } /// A shard-sequence fault. @@ -998,6 +1416,9 @@ pub struct ShardRecoveryReport { pub tail_stop: Option, pub quarantined: Option, pub quarantined_bytes: u64, + /// Durable hard link to the original active journal, before recovery + /// copied or sealed any prefix. The inode is byte-for-byte crash evidence. + pub preserved_journal: Option, pub promotions: Vec, /// Scope 3.8 step 12: readiness is true only after replay and /// catalog/genesis validation complete. A computed field, never a default. @@ -1017,12 +1438,215 @@ impl ShardRecoveryReport { tail_stop: None, quarantined: None, quarantined_bytes: 0, + preserved_journal: None, promotions: Vec::new(), ready: false, } } } +// =========================================================================== +// Production shard recovery +// =========================================================================== + +/// Bounded inputs to the shared recovery path. +/// +/// Both `StoreEngine::open` and the `store-internals` drive seam call +/// [`recover_shard`] with this type. The payload extractor is the only +/// deliberate variation: production uses [`CanonicalPayloadFacts`], while the +/// drive also supports the opaque scripted frames its writer API predates. +pub struct RecoveryConfig<'a> { + pub max_manifest_candidates: usize, + pub manifest_retain: u32, + pub checkpoint_retain: u32, + pub journal_preallocate_bytes: u64, + pub terminal_status_grace_micros: i64, + pub max_active_index_entries: u64, + pub max_active_index_bytes: u64, + pub max_index_runs: u32, + pub max_open_index_runs: u32, + pub max_replay_frames: u64, + pub max_replay_bytes: u64, + pub payload_facts: &'a dyn PayloadFacts, + /// Staging-owned read-only resolution and lifecycle seam. + /// + /// A canonical committed staged-install frame cannot become ready without + /// this resolver supplying its exact namespace membership and live + /// artifact ownership. + pub(crate) projection_recovery_resolver: Option<&'a dyn ProjectionRecoveryResolver>, +} + +impl<'a> RecoveryConfig<'a> { + pub fn from_store_options(options: &StoreOptions) -> Self { + static CANONICAL: CanonicalPayloadFacts = CanonicalPayloadFacts; + Self { + max_manifest_candidates: options.manifest_retain as usize, + manifest_retain: options.manifest_retain, + checkpoint_retain: options.checkpoint_retain, + journal_preallocate_bytes: options.journal_preallocate_bytes, + terminal_status_grace_micros: options.terminal_status_grace_micros, + max_active_index_entries: options.max_active_index_entries, + max_active_index_bytes: options.max_active_index_bytes, + max_index_runs: options.max_index_runs, + max_open_index_runs: options.max_open_index_runs, + max_replay_frames: options.max_replay_frames, + max_replay_bytes: options.max_replay_bytes, + payload_facts: &CANONICAL, + projection_recovery_resolver: None, + } + } + + pub(crate) fn with_projection_recovery_resolver( + mut self, + resolver: &'a dyn ProjectionRecoveryResolver, + ) -> Self { + self.projection_recovery_resolver = Some(resolver); + self + } +} + +/// Root-wide recovery and ownership session. +/// +/// A store root may contain several shards, but `LOCK` is root-wide. B1 opens +/// one session, recovers every shard through it, and retains the session for +/// the engine's lifetime. That prevents another process from entering between +/// shard recoveries or immediately after readiness. The drive's one-shot +/// [`recover_shard`] wrapper uses the same type and simply drops it afterward. +pub struct RecoverySession { + layout: RootLayout, + root_uuid: [u8; 16], + shard_count: u16, + _lock: File, +} + +impl std::fmt::Debug for RecoverySession { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RecoverySession") + .field("root", &self.layout.root) + .field("root_uuid", &hex::encode(self.root_uuid)) + .field("shard_count", &self.shard_count) + .finish_non_exhaustive() + } +} + +impl RecoverySession { + pub fn open(root: &Path) -> Result { + let layout = RootLayout::new(root); + // Validate FORMAT before taking LOCK so an unrecognized root is never + // modified merely by attempting to open it. + let marker = segment::read_format(&layout)?; + let lock = segment::lock_root(&layout)?; + Ok(Self { + layout, + root_uuid: marker.root_uuid, + shard_count: marker.shard_count, + _lock: lock, + }) + } + + pub fn root(&self) -> &Path { + &self.layout.root + } + + pub fn root_uuid(&self) -> [u8; 16] { + self.root_uuid + } + + pub fn shard_count(&self) -> u16 { + self.shard_count + } + + pub fn recover_shard( + &self, + shard: u16, + config: &RecoveryConfig<'_>, + ) -> Result { + recover_shard_under_lock(self, shard, config) + } +} + +/// One manifest-retained sealed segment, kept open for the recovered root's +/// lifetime. +#[derive(Clone)] +pub struct RecoveredSegment { + pub generation: u64, + pub first_shard_sequence: u64, + pub last_shard_sequence: u64, + pub path: PathBuf, + pub reader: Arc, +} + +impl std::fmt::Debug for RecoveredSegment { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RecoveredSegment") + .field("generation", &self.generation) + .field("first_shard_sequence", &self.first_shard_sequence) + .field("last_shard_sequence", &self.last_shard_sequence) + .field("path", &self.path) + .finish() + } +} + +impl PartialEq for RecoveredSegment { + fn eq(&self, other: &Self) -> bool { + self.generation == other.generation + && self.first_shard_sequence == other.first_shard_sequence + && self.last_shard_sequence == other.last_shard_sequence + && self.path == other.path + } +} + +impl Eq for RecoveredSegment {} + +/// The fresh active journal recovery fenced before reporting write readiness. +#[derive(Clone, Debug)] +pub struct RecoveredTail { + pub logical_generation: u64, + pub journal_id: [u8; 16], + pub path: PathBuf, + pub validated_through: u64, + file: Arc, +} + +impl RecoveredTail { + pub fn file(&self) -> &File { + &self.file + } +} + +impl PartialEq for RecoveredTail { + fn eq(&self, other: &Self) -> bool { + self.logical_generation == other.logical_generation + && self.journal_id == other.journal_id + && self.path == other.path + && self.validated_through == other.validated_through + } +} + +impl Eq for RecoveredTail {} + +/// Everything needed to construct one shard of `CommittedRoot`. +#[derive(Clone, Debug)] +pub struct RecoveredShard { + pub shard_index: u16, + pub catalog: NamespaceCatalog, + pub refs: Vec, + pub receipts: Vec, + pub index: LayeredObjectIndex, + /// Live ownership of every artifact retained by the selected generation. + pub retained_generation: Arc, + pub segments: Vec, + pub tail: Option, + pub manifest_generation: Option, + pub manifest_path: Option, + pub committed_shard_sequence: Option, + pub repo_sequences: BTreeMap, + pub projection_resolutions: Vec, + pub tail_stop_offset: Option, + pub report: ShardRecoveryReport, +} + /// Turn a checkpoint load into the state the later steps consume. /// /// Returns `None` and sets `offline_rebuild_required` when every generation @@ -1048,3 +1672,1422 @@ pub fn checkpoint_for_recovery( } } } + +#[derive(Clone, Debug)] +struct ReplayedFrame { + facts: FrameFacts, + payload: RecoveredPayloadFacts, + generation: u64, + offset: u64, + len: u64, +} + +/// Shared, non-feature-gated production recovery entry point. +/// +/// This function is the one ordering of recovery steps 1–12. The engine and +/// drive seam may choose different payload extractors, but neither owns a +/// manifest, checkpoint, tail, sequence, visibility, or index decision. +pub fn recover_shard( + root: &Path, + shard: u16, + config: &RecoveryConfig<'_>, +) -> Result { + RecoverySession::open(root)?.recover_shard(shard, config) +} + +fn recover_shard_under_lock( + session: &RecoverySession, + shard: u16, + config: &RecoveryConfig<'_>, +) -> Result { + if shard >= session.shard_count { + return Err(StoreError::FormatMismatch(format!( + "shard {shard} is outside the root's frozen topology of {} shards", + session.shard_count + ))); + } + let layout = &session.layout; + let root_uuid = session.root_uuid; + let paths = layout.shard(shard); + let counters = Arc::new(DurabilityCounters::default()); + let mut report = ShardRecoveryReport::new(shard); + + let referents = ProductionReferentValidator { + root_uuid, + shard_index: shard, + }; + let selection = resolve_manifest( + &paths, + &root_uuid, + &referents, + config.max_manifest_candidates, + )?; + let mut selection = match selection { + Some(selection) => { + report.manifest_generation = Some(selection.generation); + report.manifest_source = Some(selection.source.clone()); + Some(selection) + } + None if segment::list_manifest_generations(&paths)?.is_empty() => None, + None => return Err(StoreError::RecoveryRequired), + }; + + let mut segments = Vec::new(); + let mut sealed_runs_newest_first = Vector::new(); + let mut retained_index_runs = Vec::new(); + let mut retained_checkpoints = Vec::new(); + if let Some(selection) = &selection { + if selection.manifest.index_runs.len() as u64 > config.max_index_runs as u64 { + return Err(StoreError::LimitExceeded { + limit: "max_index_runs", + observed: selection.manifest.index_runs.len() as u64, + allowed: config.max_index_runs as u64, + }); + } + if selection.manifest.index_runs.len() as u64 > config.max_open_index_runs as u64 { + return Err(StoreError::LimitExceeded { + limit: "max_open_index_runs", + observed: selection.manifest.index_runs.len() as u64, + allowed: config.max_open_index_runs as u64, + }); + } + for (generation, filename) in selection.manifest.index_runs.iter().rev() { + let path = paths.indexes().join(filename); + let run = Arc::new(IndexRun::open(&path, &root_uuid)?); + if run.generation() != *generation { + return Err(StoreError::Corruption(format!( + "manifest names index generation {generation}, but {} carries generation {}", + path.display(), + run.generation() + ))); + } + sealed_runs_newest_first.push_back(Arc::clone(&run)); + retained_index_runs.push(RetainedIndexRun::new(path, run)); + } + for (_, filename) in &selection.manifest.checkpoints { + let path = paths.checkpoints().join(filename); + retained_checkpoints.push(PinnedFile::open(path)?); + } + } + + let load = load_authoritative_checkpoint( + &paths, + selection.as_ref(), + &root_uuid, + shard, + config.checkpoint_retain, + )?; + let selected_checkpoint = match &load { + CheckpointLoad::Loaded { + checkpoint, path, .. + } => Some((checkpoint.shard_committed_sequence, path.clone())), + CheckpointLoad::Empty | CheckpointLoad::OfflineRebuildRequired { .. } => None, + }; + let checkpoint = match checkpoint_for_recovery(load, &root_uuid, shard, &mut report) { + Some(checkpoint) => checkpoint, + None => { + debug_assert!(report.offline_rebuild_required); + return Err(StoreError::RecoveryRequired); + } + }; + if let Some((_, path)) = selected_checkpoint { + if !retained_checkpoints + .iter() + .any(|entry| entry.path() == path) + { + retained_checkpoints.push(PinnedFile::open(path)?); + } + } + + let checkpointed = report.checkpoint_sequence.is_some(); + let committed = checkpoint.shard_committed_sequence; + let mut adopted_shard_sequences = Vec::new(); + let mut replayed = Vec::new(); + let mut replayed_bytes = 0u64; + let mut retained_segments = Vec::new(); + let mut retained_tails = Vec::new(); + if checkpointed { + let mut accounted: Vec = checkpoint + .receipts + .iter() + .map(|receipt| receipt.shard_sequence) + .filter(|sequence| *sequence <= committed) + .collect(); + accounted.sort_unstable(); + adopted_shard_sequences.extend(accounted); + } + + if let Some(selection) = &selection { + for range in &selection.manifest.retained_tail_ranges { + let path = paths.segments().join(&range.filename); + let reader = Arc::new(SegmentReader::open(&path, &root_uuid)?); + validate_journal_binding( + &reader.journal_header()?, + &root_uuid, + shard, + Some(&reader.footer().journal_id), + None, + )?; + if reader.footer().generation != range.generation + || reader.footer().first_shard_sequence != range.first_shard_sequence + || reader.footer().last_shard_sequence != range.last_shard_sequence + { + return Err(StoreError::Corruption(format!( + "manifest range does not match sealed segment {}", + path.display() + ))); + } + + if !checkpointed || range.last_shard_sequence > committed { + for (sequence, offset, len) in reader.footer().offsets.clone() { + if checkpointed && sequence <= committed { + continue; + } + let frame = reader.read_frame(sequence)?; + let (facts, payload) = frame_facts(&frame, config.payload_facts)?; + account_replay(config, &mut replayed_bytes, len, replayed.len() as u64 + 1)?; + adopted_shard_sequences.push(sequence); + replayed.push(ReplayedFrame { + facts, + payload, + generation: range.generation, + offset, + len, + }); + } + } + retained_segments.push(RootRetainedSegment::new( + range.generation, + reader.footer().journal_id, + range.first_shard_sequence, + range.last_shard_sequence, + PinnedFile::open(paths.segments().join(&range.filename))?, + )); + segments.push(RecoveredSegment { + generation: range.generation, + first_shard_sequence: range.first_shard_sequence, + last_shard_sequence: range.last_shard_sequence, + path, + reader, + }); + } + } + + let mut tail_stop_offset = None; + let mut quarantined_bytes = 0; + let mut recovered_tail = None; + let mut must_create_fresh = false; + let mut fresh_preallocation = config.journal_preallocate_bytes; + if let Some(path) = active_journal_path(&paths)? { + let file = File::open(&path)?; + let mut header_bytes = [0u8; JOURNAL_HEADER_LEN]; + sys::pread_exact(&file, 0, &mut header_bytes)?; + let header = JournalHeader::decode(&header_bytes)?; + fresh_preallocation = header.preallocated_len; + let manifest_covered_through = selection.as_ref().and_then(|selected| { + selected + .manifest + .retained_tail_ranges + .last() + .map(|range| range.last_shard_sequence) + }); + let covered_through = if checkpointed { + Some(manifest_covered_through.unwrap_or(committed).max(committed)) + } else { + manifest_covered_through + }; + validate_journal_binding(&header, &root_uuid, shard, None, covered_through)?; + + let disposition = match &selection { + Some(selection) => classify_active_journal( + &paths, + &selection.manifest, + &header.journal_id, + &root_uuid, + )?, + None => ActiveJournalDisposition::Replay, + }; + report.active_journal = Some(disposition.clone()); + match disposition { + ActiveJournalDisposition::AlreadySealed { .. } => { + report.preserved_journal = Some(preserve_crash_journal( + &path, + &layout.quarantine_dir(), + &header, + &counters, + )?); + drop(file); + complete_interrupted_seal(&path, &paths, &counters)?; + must_create_fresh = true; + } + ActiveJournalDisposition::Replay => { + let recovery_generation = recovery_generation_for_journal( + &paths, + &root_uuid, + &header.journal_id, + config, + )?; + let from = if checkpointed && checkpoint.active_journal_id == header.journal_id { + checkpoint + .active_journal_offset + .max(JOURNAL_HEADER_LEN as u64) + } else { + JOURNAL_HEADER_LEN as u64 + }; + let full_scan = + crate::journal::scan_journal(&file, &header, JOURNAL_HEADER_LEN as u64); + let resumes_checkpoint_journal = + checkpointed && checkpoint.active_journal_id == header.journal_id; + if !resumes_checkpoint_journal { + if let (Some(covered), Some(first)) = + (covered_through, full_scan.frames.first()) + { + if first.shard_sequence <= covered { + return Err(ShardSequenceFault::Duplicate { + shard_sequence: first.shard_sequence, + } + .into()); + } + let expected = covered.saturating_add(1); + if first.shard_sequence != expected { + return Err(ShardSequenceFault::Gap { + expected, + observed: first.shard_sequence, + } + .into()); + } + } + } + let (scan, record) = recover_journal_tail( + &layout.quarantine_dir(), + &file, + &header, + from, + &counters, + )?; + quarantined_bytes = record.as_ref().map(|record| record.bytes).unwrap_or(0); + report.quarantined = record.map(|record| record.path); + for scanned in &scan.frames { + let mut frame_bytes = + vec![0u8; usize::try_from(scanned.len).map_err(|_| FrameError::Length)?]; + sys::pread_exact(&file, scanned.offset, &mut frame_bytes)?; + let frame = Frame::decode(&frame_bytes, &header.journal_id)?; + let (facts, payload) = frame_facts(&frame, config.payload_facts)?; + account_replay( + config, + &mut replayed_bytes, + scanned.len, + replayed.len() as u64 + 1, + )?; + adopted_shard_sequences.push(scanned.shard_sequence); + replayed.push(ReplayedFrame { + facts, + payload, + generation: recovery_generation, + offset: scanned.offset, + len: scanned.len, + }); + } + tail_stop_offset = match scan.stop { + TailStop::EndOfPreallocation => None, + TailStop::NotAFrame => (quarantined_bytes > 0).then_some(scan.stop_offset), + TailStop::Incomplete(_) | TailStop::ReadError => Some(scan.stop_offset), + }; + report.tail_stop = Some(scan.stop.clone()); + + let damaged = matches!( + full_scan.stop, + TailStop::Incomplete(_) | TailStop::ReadError | TailStop::EndOfPreallocation + ) || quarantined_bytes > 0; + let must_replace = damaged || !full_scan.frames.is_empty(); + if must_replace { + report.preserved_journal = Some(preserve_crash_journal( + &path, + &layout.quarantine_dir(), + &header, + &counters, + )?); + + if !full_scan.frames.is_empty() { + let segment_path = segment::seal_recovered_prefix( + &file, + &header, + &full_scan, + &paths, + recovery_generation, + &counters, + )?; + let filename = segment_path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + StoreError::Corruption( + "recovery segment name is not valid UTF-8".into(), + ) + })? + .to_string(); + let first = full_scan.frames.first().expect("non-empty").shard_sequence; + let last = full_scan.frames.last().expect("non-empty").shard_sequence; + + let (mut retained_tail_ranges, index_runs, checkpoints) = selection + .as_ref() + .map(|selected| { + ( + selected.manifest.retained_tail_ranges.clone(), + selected.manifest.index_runs.clone(), + selected.manifest.checkpoints.clone(), + ) + }) + .unwrap_or((Vec::new(), Vec::new(), Vec::new())); + retained_tail_ranges.push(TailRange { + generation: recovery_generation, + first_shard_sequence: first, + last_shard_sequence: last, + filename, + }); + let manifest = Manifest { + root_uuid, + generation: recovery_generation, + base_generation: 0, + retained_tail_ranges, + index_runs, + checkpoints, + committed_shard_sequence: last, + }; + validate_manifest_sequence_coverage(&manifest) + .map_err(StoreError::Corruption)?; + segment::install_manifest( + &paths, + &manifest, + config.manifest_retain, + &counters, + )?; + + let reader = Arc::new(SegmentReader::open(&segment_path, &root_uuid)?); + retained_segments.push(RootRetainedSegment::new( + recovery_generation, + header.journal_id, + first, + last, + PinnedFile::open(segment_path.clone())?, + )); + segments.push(RecoveredSegment { + generation: recovery_generation, + first_shard_sequence: first, + last_shard_sequence: last, + path: segment_path, + reader, + }); + + let manifest_path = paths.manifest(recovery_generation); + selection = Some(ManifestSelection { + manifest, + generation: recovery_generation, + path: manifest_path, + source: ManifestSource::Current, + rejected: Vec::new(), + }); + report.manifest_generation = Some(recovery_generation); + } + + drop(file); + complete_interrupted_seal(&path, &paths, &counters)?; + must_create_fresh = true; + } else { + let shared_file = Arc::new(file); + let retained_tail = + PinnedFile::from_shared(path.clone(), Arc::clone(&shared_file)); + recovered_tail = Some(RecoveredTail { + logical_generation: recovery_generation, + journal_id: header.journal_id, + path, + validated_through: JOURNAL_HEADER_LEN as u64, + file: shared_file, + }); + retained_tails.push(RetainedTail::new(recovery_generation, retained_tail)); + } + } + } + } else { + must_create_fresh = true; + } + + if must_create_fresh { + let last_committed = selection + .as_ref() + .and_then(|selected| { + selected + .manifest + .retained_tail_ranges + .last() + .map(|range| range.last_shard_sequence) + }) + .or(checkpointed.then_some(committed)); + let first_shard_sequence = last_committed + .map(|sequence| sequence.saturating_add(1)) + .unwrap_or(0); + let logical_generation = selection + .as_ref() + .and_then(|selected| { + selected + .manifest + .retained_tail_ranges + .last() + .map(|range| range.generation) + }) + .unwrap_or(0) + .saturating_add(1); + let (tail, retained) = create_fresh_active_journal( + &paths, + root_uuid, + shard, + first_shard_sequence, + fresh_preallocation, + logical_generation, + Arc::clone(&counters), + )?; + recovered_tail = Some(tail); + retained_tails.push(retained); + } + + let replayed_facts: Vec = + replayed.iter().map(|frame| frame.facts.clone()).collect(); + let mut catalog = checkpoint.catalog.clone(); + verify_and_apply_catalog( + &replayed_facts, + checkpointed.then(|| committed.saturating_add(1)), + &mut catalog, + config.payload_facts.permits_implicit_namespace_anchor(), + )?; + + let recovery_publication_micros = now_micros(); + let mut receipts = checkpoint.receipts.clone(); + for frame in &replayed { + receipts.push(receipt_for_recovery(frame, recovery_publication_micros)); + } + report.promotions = promote_receipt_visibility( + &mut receipts, + recovery_publication_micros, + config.terminal_status_grace_micros, + )?; + + let mut refs = checkpoint.refs.clone(); + apply_recovered_refs(&mut refs, &replayed)?; + + let mut delta = IndexDelta::new( + config.max_active_index_entries, + config.max_active_index_bytes, + ); + for frame in &replayed { + let frame_len = u32::try_from(frame.len).map_err(|_| StoreError::LimitExceeded { + limit: "frame_len", + observed: frame.len, + allowed: u32::MAX as u64, + })?; + for (object, object_type) in &frame.payload.objects { + delta.insert( + IndexKey::new(frame.facts.namespace, *object), + IndexLocation { + segment_generation: frame.generation, + frame_offset: frame.offset, + frame_len, + object_type: *object_type, + shard_sequence: frame.facts.shard_sequence, + }, + )?; + } + } + + // Resolve committed staged projections newest-first so their membership + // layers preserve the same first-hit ordering as ordinary replay. A final + // frame is already authoritative here; missing staging ownership is + // corruption and cannot be converted into an empty projection. + let mut committed_sessions = BTreeSet::new(); + let mut recovered_projections: Vec<(u64, RecoveredProjectionArtifacts)> = Vec::new(); + for frame in replayed.iter().rev() { + let Some(descriptor) = frame.payload.staged_projection_install.as_ref() else { + continue; + }; + if !committed_sessions.insert(descriptor.session_id) { + return Err(StoreError::Corruption(format!( + "staged projection session {} appears in more than one committed frame", + hex::encode(descriptor.session_id) + ))); + } + let resolver = config.projection_recovery_resolver.ok_or_else(|| { + StoreError::Corruption(format!( + "committed staged projection session {} has no recovery resolver", + hex::encode(descriptor.session_id) + )) + })?; + let artifacts = resolver.resolve_committed(frame.facts.namespace, descriptor)?; + if artifacts.descriptor() != descriptor { + return Err(StoreError::Corruption(format!( + "staging recovery returned a different descriptor for committed session {}", + hex::encode(descriptor.session_id) + ))); + } + recovered_projections.push((frame.facts.shard_sequence, artifacts)); + } + + let mut projection_outcomes = BTreeMap::new(); + if let Some(resolver) = config.projection_recovery_resolver { + for session_id in resolver.transferred_sessions(shard)?.iter().copied() { + projection_outcomes + .entry(session_id) + .or_insert(RecoveredProjectionOutcome::ProvedAbsent); + } + } + for session_id in committed_sessions { + projection_outcomes.insert(session_id, RecoveredProjectionOutcome::Committed); + } + let projection_resolutions: Vec<_> = projection_outcomes + .into_iter() + .map(|(session_id, outcome)| RecoveredProjectionResolution { + session_id, + outcome, + }) + .collect(); + + report.adopted_shard_sequences = adopted_shard_sequences; + report.quarantined_bytes = quarantined_bytes; + report.ready = !report.offline_rebuild_required && recovered_tail.is_some(); + let committed_shard_sequence = replayed + .last() + .map(|frame| frame.facts.shard_sequence) + .or(checkpointed.then_some(committed)) + .or_else(|| { + selection.as_ref().and_then(|selected| { + selected + .manifest + .retained_tail_ranges + .last() + .map(|range| range.last_shard_sequence) + }) + }); + let repo_sequences = catalog + .iter() + .map(|(namespace, record)| (*namespace, record.repo_sequence)) + .collect(); + + let through_shard_sequence = committed_shard_sequence.unwrap_or(0); + let mut delta_layers_newest_first = Vector::new(); + if !delta.is_empty() { + delta_layers_newest_first.push_back(IndexDeltaLayer::new( + shard, + through_shard_sequence, + Arc::new(delta), + )); + } + let mut all_runs_newest_first = Vector::new(); + let mut retained_projection_artifacts = Vec::new(); + for (shard_sequence, artifacts) in &recovered_projections { + if !artifacts.index_delta().is_empty() { + delta_layers_newest_first.push_back(IndexDeltaLayer::new( + shard, + *shard_sequence, + Arc::clone(artifacts.index_delta()), + )); + } + for run in artifacts.index_runs_newest_first() { + all_runs_newest_first.push_back(Arc::clone(run)); + } + retained_index_runs.extend_from_slice(artifacts.retained_index_runs()); + retained_projection_artifacts.extend_from_slice(artifacts.retained_artifacts()); + } + for run in sealed_runs_newest_first { + all_runs_newest_first.push_back(run); + } + let open_run_count = u64::try_from(all_runs_newest_first.len()).unwrap_or(u64::MAX); + if open_run_count > u64::from(config.max_index_runs) { + return Err(StoreError::LimitExceeded { + limit: "max_index_runs", + observed: open_run_count, + allowed: u64::from(config.max_index_runs), + }); + } + if open_run_count > u64::from(config.max_open_index_runs) { + return Err(StoreError::LimitExceeded { + limit: "max_open_index_runs", + observed: open_run_count, + allowed: u64::from(config.max_open_index_runs), + }); + } + validate_retained_object_sources( + &retained_segments, + &retained_tails, + &retained_projection_artifacts, + )?; + let index = LayeredObjectIndex::new(delta_layers_newest_first, all_runs_newest_first); + let generation_id = GenerationId::new( + shard, + selection + .as_ref() + .map(|selection| selection.generation) + .unwrap_or(0), + ); + let retained_generation = Arc::new(RetainedGeneration::new( + generation_id, + retained_segments.into(), + retained_index_runs.into(), + retained_checkpoints.into(), + retained_tails.into(), + retained_projection_artifacts.into(), + )); + + let recovered = RecoveredShard { + shard_index: shard, + catalog, + refs, + receipts, + index, + retained_generation, + segments, + tail: recovered_tail, + manifest_generation: selection.as_ref().map(|selection| selection.generation), + manifest_path: selection.as_ref().map(|selection| selection.path.clone()), + committed_shard_sequence, + repo_sequences, + projection_resolutions, + tail_stop_offset, + report, + }; + + // Lifecycle notification is last. At this point the complete physical + // proof, lookup layers, and every live pin are held by `recovered`. + // Implementations must make notification idempotent because a later + // notification failure keeps the shard unready and recovery retries. + if let Some(resolver) = config.projection_recovery_resolver { + for resolution in recovered.projection_resolutions.iter().copied() { + resolver.notify_recovered(resolution)?; + } + } + + Ok(recovered) +} + +fn validate_retained_object_sources( + segments: &[RootRetainedSegment], + tails: &[RetainedTail], + projections: &[RetainedProjectionArtifact], +) -> Result<(), StoreError> { + let mut generations: BTreeMap = BTreeMap::new(); + let mut insert = |generation: u64, kind: &'static str, path: &Path| { + if let Some((existing_kind, existing_path)) = generations.get(&generation) { + if *existing_kind != kind || existing_path.as_path() != path { + return Err(StoreError::Corruption(format!( + "logical object generation {generation} names both {existing_kind} {} and \ + {kind} {}", + existing_path.display(), + path.display() + ))); + } + } else { + generations.insert(generation, (kind, path.to_path_buf())); + } + Ok(()) + }; + for segment in segments { + insert(segment.logical_generation, "segment", segment.path())?; + } + for tail in tails { + insert(tail.logical_generation, "active tail", tail.path())?; + } + for projection in projections { + insert( + projection.logical_generation, + "projection artifact", + projection.path(), + )?; + } + Ok(()) +} + +fn account_replay( + config: &RecoveryConfig<'_>, + bytes: &mut u64, + frame_len: u64, + frames: u64, +) -> Result<(), StoreError> { + if frames > config.max_replay_frames { + return Err(StoreError::LimitExceeded { + limit: "max_replay_frames", + observed: frames, + allowed: config.max_replay_frames, + }); + } + *bytes = bytes + .checked_add(frame_len) + .ok_or(StoreError::LimitExceeded { + limit: "max_replay_bytes", + observed: u64::MAX, + allowed: config.max_replay_bytes, + })?; + if *bytes > config.max_replay_bytes { + return Err(StoreError::LimitExceeded { + limit: "max_replay_bytes", + observed: *bytes, + allowed: config.max_replay_bytes, + }); + } + Ok(()) +} + +fn frame_facts( + frame: &Frame, + payload: &dyn PayloadFacts, +) -> Result<(FrameFacts, RecoveredPayloadFacts), StoreError> { + let mut facts = FrameFacts::from_header(&frame.header); + payload.extend(&mut facts, &frame.payload)?; + let recovered = payload.recovered(&frame.payload)?; + Ok((facts, recovered)) +} + +fn verify_and_apply_catalog( + replayed: &[FrameFacts], + first_expected: Option, + catalog: &mut NamespaceCatalog, + permits_implicit_namespace_anchor: bool, +) -> Result<(), StoreError> { + if let Some(first_expected) = + first_expected.or_else(|| replayed.first().map(|f| f.shard_sequence)) + { + verify_shard_sequence(replayed, first_expected)?; + } + + if permits_implicit_namespace_anchor { + let mut anchored = catalog.clone(); + let mut chained = Vec::new(); + for facts in replayed { + if anchored.get(&facts.namespace).is_none() { + anchored.bind(crate::index::NamespaceRecord { + namespace: facts.namespace, + genesis_authority: facts.genesis_authority, + current_authority: facts.current_authority, + lifecycle: crate::index::NamespaceLifecycle::Active, + storage_mode: crate::index::NamespaceStorageMode::Full, + repo_sequence: facts.repo_sequence, + previous_event_digest: facts.event_digest, + })?; + } else { + chained.push(facts.clone()); + } + } + verify_repo_chain(&chained, &anchored)?; + } else { + verify_repo_chain(replayed, catalog)?; + } + + for facts in replayed { + if catalog.get(&facts.namespace).is_none() { + if !facts.creates_namespace && !permits_implicit_namespace_anchor { + return Err(RepoSequenceFault::UnknownNamespace { + namespace: facts.namespace.to_hex(), + } + .into()); + } + catalog.bind(crate::index::NamespaceRecord { + namespace: facts.namespace, + genesis_authority: facts.genesis_authority, + current_authority: facts.current_authority, + lifecycle: crate::index::NamespaceLifecycle::Active, + storage_mode: crate::index::NamespaceStorageMode::Full, + repo_sequence: facts.repo_sequence, + previous_event_digest: facts.event_digest, + })?; + } else { + catalog.advance( + &facts.namespace, + facts.current_authority, + facts.repo_sequence, + facts.event_digest, + )?; + } + } + Ok(()) +} + +fn receipt_for_recovery(frame: &ReplayedFrame, visible_at: i64) -> ReceiptRecord { + let facts = &frame.facts; + ReceiptRecord { + namespace: facts.namespace, + operation_id: facts.operation_id, + operation_digest: facts.operation_digest, + repo_sequence: facts.repo_sequence, + shard_sequence: facts.shard_sequence, + current_authority: facts.current_authority, + refs: frame.payload.applied_refs.clone(), + objects_new: frame.payload.objects_new, + retry_until_micros: facts.retry_until_micros, + // A replayed receipt's original first-visibility instant was not + // durably checkpointed. Step 10 promotes it to publication below. + first_receipt_visibility_micros: None, + receipt_visible_until_micros: visible_at, + } +} + +fn apply_recovered_refs( + refs: &mut Vec, + replayed: &[ReplayedFrame], +) -> Result<(), StoreError> { + let mut state: BTreeMap<(NamespaceId, u8, Vec), ObjectId> = refs + .iter() + .map(|record| { + ( + (record.namespace, record.ref_kind, record.name.clone()), + record.target, + ) + }) + .collect(); + + for frame in replayed { + let facts = &frame.facts; + for update in &frame.payload.ref_updates { + let (kind, name) = match &update.target { + RefTarget::Branch(name) => (1u8, name.as_bytes().to_vec()), + RefTarget::Release(name) => (2u8, name.as_bytes().to_vec()), + }; + let key = (facts.namespace, kind, name); + let observed = state.get(&key).copied(); + if observed != update.expected { + return Err(StoreError::Corruption(format!( + "committed ref CAS for namespace {} does not match recovered state", + facts.namespace.to_hex() + ))); + } + match update.mutation { + RefMutation::Set(target) => { + state.insert(key, target); + } + RefMutation::Delete => { + state.remove(&key); + } + } + } + } + + *refs = state + .into_iter() + .map(|((namespace, ref_kind, name), target)| RefRecord { + namespace, + ref_kind, + name, + target, + }) + .collect(); + Ok(()) +} + +fn recovery_generation_for_journal( + paths: &ShardPaths, + root_uuid: &[u8; 16], + active_journal_id: &[u8; 16], + config: &RecoveryConfig<'_>, +) -> Result { + let per_manifest = usize::try_from(config.max_index_runs) + .unwrap_or(usize::MAX) + .saturating_add(config.checkpoint_retain as usize) + .saturating_add(16); + let budget = config + .max_manifest_candidates + .max(1) + .saturating_mul(per_manifest) + .max(64); + let mut inspected = 0usize; + let mut maximum = 0u64; + let mut resumable = BTreeSet::new(); + + let manifests = segment::list_manifest_generations(paths)?; + inspected = inspected.saturating_add(manifests.len()); + if inspected > budget { + return Err(StoreError::LimitExceeded { + limit: "recovery_generation_artifacts", + observed: inspected as u64, + allowed: budget as u64, + }); + } + maximum = maximum.max(manifests.into_iter().max().unwrap_or(0)); + + for (dir, extension) in [(paths.segments(), "seg"), (paths.indexes(), "idx")] { + let entries = match std::fs::read_dir(&dir) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(error.into()), + }; + for entry in entries { + let path = entry?.path(); + if path.extension().and_then(|value| value.to_str()) != Some(extension) { + continue; + } + inspected = inspected.saturating_add(1); + if inspected > budget { + return Err(StoreError::LimitExceeded { + limit: "recovery_generation_artifacts", + observed: inspected as u64, + allowed: budget as u64, + }); + } + let name = path + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| { + StoreError::Corruption(format!( + "immutable artifact name {} is not UTF-8", + path.display() + )) + })?; + let stem = name.strip_suffix(&format!(".{extension}")).ok_or_else(|| { + StoreError::Corruption(format!( + "immutable artifact {} has an invalid extension", + path.display() + )) + })?; + let decoded_generation = if extension == "seg" { + SegmentReader::open(&path, root_uuid).ok().map(|reader| { + if &reader.footer().journal_id == active_journal_id { + resumable.insert(reader.footer().generation); + } + reader.footer().generation + }) + } else { + IndexRun::open(&path, root_uuid) + .ok() + .map(|run| run.generation()) + }; + let generation = decoded_generation + .or_else(|| { + let text = if extension == "seg" { + stem.split('-').next().unwrap_or("") + } else { + stem + }; + text.parse::().ok() + }) + .ok_or_else(|| { + StoreError::Corruption(format!( + "immutable artifact {} is invalid and has no generation in its name", + path.display() + )) + })?; + maximum = maximum.max(generation); + } + } + + let prefix = format!(".recovery-{}-", hex::encode(active_journal_id)); + let entries = std::fs::read_dir(paths.segments())?; + for entry in entries { + let path = entry?.path(); + let Some(name) = path.file_name().and_then(|value| value.to_str()) else { + continue; + }; + let Some(generation_text) = name + .strip_prefix(&prefix) + .and_then(|suffix| suffix.strip_suffix(".prefix")) + else { + continue; + }; + inspected = inspected.saturating_add(1); + if inspected > budget { + return Err(StoreError::LimitExceeded { + limit: "recovery_generation_artifacts", + observed: inspected as u64, + allowed: budget as u64, + }); + } + let generation = generation_text.parse::().map_err(|_| { + StoreError::Corruption(format!( + "recovery prefix {} has an invalid generation", + path.display() + )) + })?; + maximum = maximum.max(generation); + resumable.insert(generation); + } + + if resumable.len() > 1 { + return Err(StoreError::Corruption(format!( + "active journal {} has recovery artifacts in multiple generations: {:?}", + hex::encode(active_journal_id), + resumable + ))); + } + if let Some(generation) = resumable.into_iter().next() { + if generation != maximum { + return Err(StoreError::Corruption(format!( + "recovery artifact generation {generation} for journal {} is below occupied \ + immutable generation {maximum}", + hex::encode(active_journal_id) + ))); + } + return Ok(generation); + } + + maximum.checked_add(1).ok_or_else(|| { + StoreError::Corruption("no generation remains for recovery artifacts".into()) + }) +} + +fn preserve_crash_journal( + active_path: &Path, + quarantine_dir: &Path, + header: &JournalHeader, + counters: &DurabilityCounters, +) -> Result { + use std::os::unix::fs::MetadataExt; + + std::fs::create_dir_all(quarantine_dir)?; + let evidence = quarantine_dir.join(format!( + "{}-{}-{}.journal.evidence", + hex::encode(header.journal_id), + header.shard_index, + header.first_shard_sequence + )); + match sys::link_noreplace(active_path, &evidence) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + let original = std::fs::metadata(active_path)?; + let retained = std::fs::metadata(&evidence)?; + if original.dev() != retained.dev() || original.ino() != retained.ino() { + return Err(StoreError::Corruption(format!( + "journal evidence name {} is occupied by another inode", + evidence.display() + ))); + } + } + Err(error) => return Err(error.into()), + } + sys::fsync_dir(quarantine_dir, counters)?; + Ok(evidence) +} + +#[allow(clippy::too_many_arguments)] +fn create_fresh_active_journal( + paths: &ShardPaths, + root_uuid: [u8; 16], + shard: u16, + first_shard_sequence: u64, + preallocated_len: u64, + logical_generation: u64, + counters: Arc, +) -> Result<(RecoveredTail, RetainedTail), StoreError> { + let journal_id = fresh_recovery_journal_id(root_uuid, shard, first_shard_sequence); + let journal = Journal::create( + &paths.active(), + journal_id, + first_shard_sequence, + shard, + root_uuid, + preallocated_len, + now_micros(), + Arc::clone(&counters), + )?; + let path = journal.path().to_path_buf(); + let shared_file = Arc::new(journal.file().try_clone()?); + let retained = RetainedTail::new( + logical_generation, + PinnedFile::from_shared(path.clone(), Arc::clone(&shared_file)), + ); + Ok(( + RecoveredTail { + logical_generation, + journal_id, + path, + validated_through: JOURNAL_HEADER_LEN as u64, + file: shared_file, + }, + retained, + )) +} + +fn fresh_recovery_journal_id( + root_uuid: [u8; 16], + shard: u16, + first_shard_sequence: u64, +) -> [u8; 16] { + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT: AtomicU64 = AtomicU64::new(0); + + let mut hasher = blake3::Hasher::new(); + hasher.update(b"levcs-recovery-journal-id/v1\0"); + hasher.update(&root_uuid); + hasher.update(&shard.to_le_bytes()); + hasher.update(&first_shard_sequence.to_le_bytes()); + hasher.update(&now_micros().to_le_bytes()); + hasher.update(&std::process::id().to_le_bytes()); + hasher.update(&NEXT.fetch_add(1, Ordering::Relaxed).to_le_bytes()); + let digest = hasher.finalize(); + let mut out = [0u8; 16]; + out.copy_from_slice(&digest.as_bytes()[..16]); + out +} + +fn active_journal_path(paths: &ShardPaths) -> Result, StoreError> { + let dir = match std::fs::read_dir(paths.active()) { + Ok(dir) => dir, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + let mut found = Vec::new(); + for entry in dir { + let path = entry?.path(); + if path + .extension() + .is_some_and(|extension| extension == "journal") + { + found.push(path); + } + } + match found.len() { + 0 => Ok(None), + 1 => Ok(found.pop()), + count => Err(StoreError::Corruption(format!( + "shard directory {} holds {count} active journals; exactly one is expected", + paths.active().display() + ))), + } +} + +fn now_micros() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_micros() as i64) + .unwrap_or(0) +} + +#[cfg(test)] +mod production_session_tests { + use std::sync::Mutex; + + use levcs_protocol::v2::ProjectionMode; + + use super::*; + use crate::format::{frame_total_len, Frame}; + use crate::roots::{ProjectionArtifactFormat, RetainedProjectionArtifact}; + use crate::staging::RecoveredProjectionArtifacts; + + struct StagedFacts { + descriptor: StagedProjectionInstallV1, + } + + impl PayloadFacts for StagedFacts { + fn extend(&self, facts: &mut FrameFacts, _payload: &[u8]) -> Result<(), StoreError> { + facts.event_digest = ObjectId([12; 32]); + facts.previous_event_digest = ObjectId([0; 32]); + facts.creates_namespace = true; + facts.genesis_authority = ObjectId([13; 32]); + facts.current_authority = ObjectId([13; 32]); + facts.retry_until_micros = i64::MAX; + Ok(()) + } + + fn recovered(&self, _payload: &[u8]) -> Result { + Ok(RecoveredPayloadFacts { + objects_new: self.descriptor.object_count, + staged_projection_install: Some(self.descriptor.clone()), + ..RecoveredPayloadFacts::default() + }) + } + } + + struct StagedResolver { + namespace: NamespaceId, + descriptor: StagedProjectionInstallV1, + artifacts: RecoveredProjectionArtifacts, + absent_session: [u8; 16], + notifications: Mutex>, + } + + impl ProjectionRecoveryResolver for StagedResolver { + fn transferred_sessions(&self, _shard_index: u16) -> Result, StoreError> { + Ok(Arc::from([self.descriptor.session_id, self.absent_session])) + } + + fn resolve_committed( + &self, + namespace: NamespaceId, + descriptor: &StagedProjectionInstallV1, + ) -> Result { + assert_eq!(namespace, self.namespace); + assert_eq!(descriptor, &self.descriptor); + Ok(self.artifacts.clone()) + } + + fn notify_recovered( + &self, + resolution: RecoveredProjectionResolution, + ) -> Result<(), StoreError> { + self.notifications + .lock() + .expect("notification mutex") + .push(resolution); + Ok(()) + } + } + + #[test] + fn one_session_holds_lock_continuously_across_all_shards() { + let dir = tempfile::tempdir().expect("tempdir"); + let counters = DurabilityCounters::default(); + segment::initialize_root(&RootLayout::new(dir.path()), 2, [7u8; 16], 1, &counters) + .expect("initialize"); + + let mut options = StoreOptions::new(dir.path()); + options.shard_count = 2; + let config = RecoveryConfig::from_store_options(&options); + let session = RecoverySession::open(dir.path()).expect("first opener"); + + assert!(matches!( + RecoverySession::open(dir.path()), + Err(StoreError::AlreadyLocked) + )); + let shard0 = session.recover_shard(0, &config).expect("recover shard 0"); + assert!(shard0.report.ready); + assert!(matches!( + RecoverySession::open(dir.path()), + Err(StoreError::AlreadyLocked) + )); + let shard1 = session.recover_shard(1, &config).expect("recover shard 1"); + assert!(shard1.report.ready); + assert!(matches!( + RecoverySession::open(dir.path()), + Err(StoreError::AlreadyLocked) + )); + + drop(session); + RecoverySession::open(dir.path()).expect("lock released only with session"); + } + + #[test] + fn committed_staged_projection_is_indexed_pinned_and_notified_before_readiness() { + let dir = tempfile::tempdir().expect("tempdir"); + let root_uuid = [7u8; 16]; + let counters = Arc::new(DurabilityCounters::default()); + let layout = RootLayout::new(dir.path()); + segment::initialize_root(&layout, 1, root_uuid, 1, &counters).expect("initialize"); + let paths = layout.shard(0); + + let namespace = NamespaceId([21; 32]); + let descriptor = StagedProjectionInstallV1 { + session_id: [22; 16], + manifest_digest: ObjectId([23; 32]), + projection: ProjectionMode::Full, + object_count: 1, + object_bytes: 17, + membership_root: ObjectId([24; 32]), + artifact_set_digest: ObjectId([25; 32]), + }; + let object = ObjectId([26; 32]); + let artifact_path = dir.path().join("staged-projection.chunk"); + let artifact_file = File::create(&artifact_path).expect("artifact"); + let mut staged_delta = IndexDelta::new(8, 4096); + staged_delta + .insert( + IndexKey::new(namespace, object), + IndexLocation { + segment_generation: 55, + frame_offset: 0, + frame_len: 17, + object_type: 1, + shard_sequence: 0, + }, + ) + .expect("index staged object"); + let artifacts = RecoveredProjectionArtifacts::new( + descriptor.clone(), + Arc::new(staged_delta), + Arc::from([]), + Arc::from([RetainedProjectionArtifact::new( + 55, + ProjectionArtifactFormat::CanonicalStageChunkV1, + PinnedFile::new(artifact_path.clone(), artifact_file), + )]), + ) + .expect("resolved artifacts"); + let resolver = StagedResolver { + namespace, + descriptor: descriptor.clone(), + artifacts, + absent_session: [27; 16], + notifications: Mutex::new(Vec::new()), + }; + let facts = StagedFacts { + descriptor: descriptor.clone(), + }; + + let journal_id = [28; 16]; + let mut journal = Journal::create( + &paths.active(), + journal_id, + 0, + 0, + root_uuid, + 1024 * 1024, + 1, + Arc::clone(&counters), + ) + .expect("journal"); + let payload = vec![1]; + let frame = Frame { + header: FrameHeader { + flags: 0, + total_len: frame_total_len(payload.len() as u64), + journal_id, + shard_sequence: 0, + repo_sequence: 0, + namespace: *namespace.as_bytes(), + operation_id: [29; 16], + operation_digest: ObjectId([30; 32]), + payload_len: payload.len() as u64, + payload_digest: ObjectId([0; 32]), + }, + payload, + }; + journal + .append_group_and_fence(&[frame]) + .expect("append staged frame"); + drop(journal); + + let mut options = StoreOptions::new(dir.path()); + options.shard_count = 1; + options.journal_preallocate_bytes = 1024 * 1024; + let mut config = RecoveryConfig::from_store_options(&options); + config.payload_facts = &facts; + let config = config.with_projection_recovery_resolver(&resolver); + let session = RecoverySession::open(dir.path()).expect("recovery session"); + let recovered = session.recover_shard(0, &config).expect("recover"); + + assert!(recovered.report.ready); + let location = recovered + .index + .lookup(&IndexKey::new(namespace, object)) + .location + .expect("staged object membership"); + assert_eq!(location.segment_generation, 55); + assert_eq!( + recovered + .retained_generation + .projection_artifacts + .first() + .expect("projection pin") + .path(), + artifact_path + ); + assert_eq!( + recovered.projection_resolutions, + vec![ + RecoveredProjectionResolution { + session_id: descriptor.session_id, + outcome: RecoveredProjectionOutcome::Committed, + }, + RecoveredProjectionResolution { + session_id: [27; 16], + outcome: RecoveredProjectionOutcome::ProvedAbsent, + }, + ] + ); + assert_eq!( + *resolver.notifications.lock().expect("notifications"), + recovered.projection_resolutions + ); + } +} diff --git a/crates/levcs-store/src/roots.rs b/crates/levcs-store/src/roots.rs new file mode 100644 index 0000000..6ae797d --- /dev/null +++ b/crates/levcs-store/src/roots.rs @@ -0,0 +1,1345 @@ +//! Immutable publication roots for committed and in-flight store state. +//! +//! **Lead-owned (D0-B).** The types in this file are the boundary between the +//! shard writers and every reader. Writers build a complete [`ShardSubtree`], +//! merge it into the last root they loaded, and publish the resulting +//! [`CommittedRoot`] through `ArcSwap`. Nothing reachable from an already +//! published root is mutated. +//! +//! The persistent collections are intentional. A group touching a few +//! repositories, receipts, or refs must share the untouched majority rather +//! than clone it. Typed refs use [`im::OrdMap`] because their canonical order +//! feeds checkpoint/digest construction; hot lookup tables use +//! [`im::HashMap`]. + +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use im::{HashMap, OrdMap, Vector}; +use levcs_core::ObjectId; +use levcs_protocol::v2::RefTarget; + +use crate::index::{ + IndexDelta, IndexKey, IndexLocation, IndexRun, LookupResult, NamespaceLifecycle, + NamespaceStorageMode, +}; +use crate::types::{ + CommitReceipt, NamespaceId, OperationId, PendingPhase, StoreError, TransactionStatus, +}; + +/// Key shared by the committed terminal table and the transient status root. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub struct OperationKey { + pub namespace: NamespaceId, + pub operation_id: OperationId, +} + +impl OperationKey { + pub const fn new(namespace: NamespaceId, operation_id: OperationId) -> Self { + Self { + namespace, + operation_id, + } + } +} + +/// Canonically ordered typed refs. +/// +/// Ref order is observable anywhere a state digest or checkpoint is encoded, +/// so this is the one hot publication map for which `OrdMap`, rather than the +/// faster unordered HAMT, is required. +pub type TypedRefMap = OrdMap; + +/// One repository's complete logical state at a committed publication. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoState { + pub repo_sequence: u64, + pub current_authority: ObjectId, + pub genesis_authority: ObjectId, + pub refs: TypedRefMap, + pub lifecycle: NamespaceLifecycle, + pub storage_mode: NamespaceStorageMode, + pub previous_event_digest: ObjectId, +} + +/// A committed operation retained while its receipt remains visible. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RetainedReceipt { + pub operation_digest: ObjectId, + pub receipt: CommitReceipt, + pub shard_sequence: u64, + pub retry_until_micros: i64, + pub first_visible_at_micros: i64, + pub receipt_visible_until_micros: i64, +} + +/// A committed operation whose receipt aged out but whose ID/digest tombstone +/// still prevents ambiguous reuse. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReceiptTombstone { + pub operation_digest: ObjectId, + pub retry_until_micros: i64, + pub tombstone_until_micros: i64, +} + +/// Durable terminal state for one `(namespace, operation ID)`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TerminalStatusEntry { + Committed(RetainedReceipt), + Expired(ReceiptTombstone), +} + +impl TerminalStatusEntry { + pub fn operation_digest(&self) -> ObjectId { + match self { + Self::Committed(receipt) => receipt.operation_digest, + Self::Expired(tombstone) => tombstone.operation_digest, + } + } + + pub fn transaction_status(&self) -> TransactionStatus { + match self { + Self::Committed(receipt) => TransactionStatus::Committed(receipt.receipt.clone()), + Self::Expired(tombstone) => TransactionStatus::Expired { + operation_digest: tombstone.operation_digest, + retry_until_micros: tombstone.retry_until_micros, + tombstone_until_micros: tombstone.tombstone_until_micros, + }, + } + } +} + +/// A file pin retained by a committed generation. +/// +/// The live `File`, not the path, is the ownership. On the supported Unix +/// storage targets it keeps the inode alive even after reclamation removes a +/// directory entry. The path remains for diagnostics and reference proofs. +#[derive(Clone, Debug)] +pub struct PinnedFile { + path: PathBuf, + file: Arc, +} + +impl PinnedFile { + pub fn new(path: PathBuf, file: File) -> Self { + Self { + path, + file: Arc::new(file), + } + } + + pub fn from_shared(path: PathBuf, file: Arc) -> Self { + Self { path, file } + } + + pub fn open(path: impl Into) -> Result { + let path = path.into(); + let file = File::open(&path)?; + Ok(Self::new(path, file)) + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn file(&self) -> &Arc { + &self.file + } +} + +/// A validated immutable index run together with its installed name. +/// +/// `IndexRun` owns its mapping; retaining the `Arc` therefore retains the +/// mapped inode. Keeping the path alongside it makes adoption cleanup's +/// committed-root reference proof possible without consulting mutable +/// staging bookkeeping. +#[derive(Clone, Debug)] +pub struct RetainedIndexRun { + path: PathBuf, + run: Arc, +} + +impl RetainedIndexRun { + pub fn new(path: PathBuf, run: Arc) -> Self { + Self { path, run } + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn run(&self) -> &Arc { + &self.run + } +} + +/// A validated sealed segment addressed by the generation stored in index +/// locations. +/// +/// The pin keeps the inode alive; the logical generation is what lets a +/// reader resolve `IndexLocation::segment_generation` without parsing a file +/// name or consulting mutable manifest state. +#[derive(Clone, Debug)] +pub struct RetainedSegment { + pub logical_generation: u64, + pub journal_id: [u8; 16], + pub first_shard_sequence: u64, + pub last_shard_sequence: u64, + pub file: PinnedFile, +} + +impl RetainedSegment { + pub fn new( + logical_generation: u64, + journal_id: [u8; 16], + first_shard_sequence: u64, + last_shard_sequence: u64, + file: PinnedFile, + ) -> Self { + Self { + logical_generation, + journal_id, + first_shard_sequence, + last_shard_sequence, + file, + } + } + + pub fn path(&self) -> &Path { + self.file.path() + } +} + +/// A validated active journal prefix addressed as one logical segment. +/// +/// Replayed index entries use `IndexLocation::segment_generation`, so the +/// generation assigned during recovery must travel with the file pin into +/// every committed root. A bare `PinnedFile` would keep the bytes alive but +/// leave readers unable to resolve those locations without reopening or +/// guessing a path. +#[derive(Clone, Debug)] +pub struct RetainedTail { + pub logical_generation: u64, + pub file: PinnedFile, +} + +impl RetainedTail { + pub fn new(logical_generation: u64, file: PinnedFile) -> Self { + Self { + logical_generation, + file, + } + } + + pub fn path(&self) -> &Path { + self.file.path() + } +} + +/// Physical encoding used by an adopted projection artifact. +/// +/// Staged chunks are independently canonical and checksummed; they are not +/// transaction frames. The source kind therefore travels with the generation +/// pin so a reader never applies the journal-frame decoder to chunk bytes. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum ProjectionArtifactFormat { + CanonicalStageChunkV1, +} + +/// One adopted projection artifact addressable by index locations. +#[derive(Clone, Debug)] +pub struct RetainedProjectionArtifact { + pub logical_generation: u64, + pub format: ProjectionArtifactFormat, + pub file: PinnedFile, +} + +impl RetainedProjectionArtifact { + pub fn new( + logical_generation: u64, + format: ProjectionArtifactFormat, + file: PinnedFile, + ) -> Self { + Self { + logical_generation, + format, + file, + } + } + + pub fn path(&self) -> &Path { + self.file.path() + } +} + +/// The physical record decoder selected by one index generation. +#[derive(Copy, Clone, Debug)] +pub enum RetainedObjectSource<'a> { + Segment(&'a RetainedSegment), + ActiveTail(&'a RetainedTail), + ProjectionArtifact(&'a RetainedProjectionArtifact), +} + +impl RetainedObjectSource<'_> { + pub fn path(&self) -> &Path { + match self { + Self::Segment(source) => source.path(), + Self::ActiveTail(source) => source.path(), + Self::ProjectionArtifact(source) => source.path(), + } + } +} + +/// Stable identity of one retained manifest generation. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub struct GenerationId { + pub shard_index: u16, + pub manifest_generation: u64, +} + +impl GenerationId { + pub const fn new(shard_index: u16, manifest_generation: u64) -> Self { + Self { + shard_index, + manifest_generation, + } + } +} + +/// Live ownership of every artifact retained by one manifest generation. +/// +/// Recovery constructs this only after validating all referents. Moving it +/// into a root leaves no interval where the artifacts are merely named but +/// unowned and therefore reclaimable. +#[derive(Clone, Debug)] +pub struct RetainedGeneration { + pub id: GenerationId, + pub segments: Arc<[RetainedSegment]>, + pub index_runs: Arc<[RetainedIndexRun]>, + pub checkpoints: Arc<[PinnedFile]>, + pub active_tails: Arc<[RetainedTail]>, + pub projection_artifacts: Arc<[RetainedProjectionArtifact]>, +} + +impl RetainedGeneration { + pub fn new( + id: GenerationId, + segments: Arc<[RetainedSegment]>, + index_runs: Arc<[RetainedIndexRun]>, + checkpoints: Arc<[PinnedFile]>, + active_tails: Arc<[RetainedTail]>, + projection_artifacts: Arc<[RetainedProjectionArtifact]>, + ) -> Self { + Self { + id, + segments, + index_runs, + checkpoints, + active_tails, + projection_artifacts, + } + } + + pub fn references_path(&self, path: &Path) -> bool { + self.segments.iter().any(|segment| segment.path() == path) + || self.index_runs.iter().any(|run| run.path() == path) + || self.checkpoints.iter().any(|file| file.path() == path) + || self.active_tails.iter().any(|tail| tail.path() == path) + || self + .projection_artifacts + .iter() + .any(|artifact| artifact.path() == path) + } + + pub fn retained_tail(&self, logical_generation: u64) -> Option<&RetainedTail> { + self.active_tails + .iter() + .find(|tail| tail.logical_generation == logical_generation) + } + + pub fn retained_segment(&self, logical_generation: u64) -> Option<&RetainedSegment> { + self.segments + .iter() + .find(|segment| segment.logical_generation == logical_generation) + } + + pub fn retained_projection_artifact( + &self, + logical_generation: u64, + ) -> Option<&RetainedProjectionArtifact> { + self.projection_artifacts + .iter() + .find(|artifact| artifact.logical_generation == logical_generation) + } +} + +/// Immutable object-index layering captured by every reader. +/// +/// Both vectors are newest-first. Group deltas sit above sealed runs, and the +/// first hit wins. Each layer/run is `Arc`-shared, so a merge adds only the +/// new group's layer and shares every older one. Delta layers carry their +/// shard and upper sequence so installing a sealed run can discard exactly +/// the layers that run covers instead of allowing lookup fan-out to grow once +/// per committed group forever. +#[derive(Clone, Debug)] +pub struct IndexDeltaLayer { + pub shard_index: u16, + pub through_shard_sequence: u64, + pub delta: Arc, +} + +impl IndexDeltaLayer { + pub fn new(shard_index: u16, through_shard_sequence: u64, delta: Arc) -> Self { + Self { + shard_index, + through_shard_sequence, + delta, + } + } +} + +#[derive(Clone, Debug, Default)] +pub struct LayeredObjectIndex { + delta_layers_newest_first: Vector, + sealed_runs_newest_first: Vector>, +} + +impl LayeredObjectIndex { + pub fn new( + delta_layers_newest_first: Vector, + sealed_runs_newest_first: Vector>, + ) -> Self { + Self { + delta_layers_newest_first, + sealed_runs_newest_first, + } + } + + /// Construct the complete recovery lookup stack: replayed tail above the + /// selected generation's already ordered sealed runs. + pub fn from_recovery( + shard_index: u16, + through_shard_sequence: u64, + replayed_delta: Arc, + sealed_runs_newest_first: Vector>, + ) -> Self { + let mut delta_layers = Vector::new(); + if !replayed_delta.is_empty() { + delta_layers.push_back(IndexDeltaLayer::new( + shard_index, + through_shard_sequence, + replayed_delta, + )); + } + Self::new(delta_layers, sealed_runs_newest_first) + } + + pub fn delta_layers(&self) -> &Vector { + &self.delta_layers_newest_first + } + + pub fn sealed_runs(&self) -> &Vector> { + &self.sealed_runs_newest_first + } + + pub fn delta_layer_count(&self) -> usize { + self.delta_layers_newest_first.len() + } + + pub fn sealed_run_count(&self) -> usize { + self.sealed_runs_newest_first.len() + } + + pub fn lookup(&self, key: &IndexKey) -> LookupResult { + for layer in &self.delta_layers_newest_first { + if let Some(location) = layer.delta.get(key) { + return LookupResult { + location: Some(location), + runs_searched: 0, + runs_filtered: 0, + answered_by_delta: true, + }; + } + } + + let mut result = LookupResult::default(); + for run in &self.sealed_runs_newest_first { + if !run.may_contain(key) { + result.runs_filtered += 1; + continue; + } + result.runs_searched += 1; + if let Some(location) = run.get(key) { + result.location = Some(location); + return result; + } + } + result + } + + pub fn get(&self, key: &IndexKey) -> Option { + self.lookup(key).location + } + + fn with_subtree(&self, subtree: &ShardSubtree) -> Self { + let mut next = self.clone(); + if let Some(sealed_through) = subtree.sealed_through_shard_sequence { + next.delta_layers_newest_first.retain(|layer| { + layer.shard_index != subtree.shard_index + || layer.through_shard_sequence > sealed_through + }); + } + let delta_is_sealed = subtree + .sealed_through_shard_sequence + .is_some_and(|sealed_through| sealed_through >= subtree.shard_committed_sequence); + if !delta_is_sealed && !subtree.index_delta.is_empty() { + next.delta_layers_newest_first + .push_front(IndexDeltaLayer::new( + subtree.shard_index, + subtree.shard_committed_sequence, + Arc::clone(&subtree.index_delta), + )); + } + // `subtree` is already newest-first. Pushing from its back preserves + // that order at the front of the composed stack. + for run in subtree.sealed_runs_newest_first.iter().rev() { + next.sealed_runs_newest_first.push_front(Arc::clone(run)); + } + next + } +} + +pub type RepoMap = HashMap>; +pub type TerminalStatusMap = HashMap; +pub type ShardSequenceMap = HashMap; +pub type GenerationMap = HashMap>; + +/// The immutable visibility boundary for all committed reads. +#[derive(Clone, Debug, Default)] +pub struct CommittedRoot { + repositories: RepoMap, + index: LayeredObjectIndex, + terminal_statuses: TerminalStatusMap, + shard_committed_sequences: ShardSequenceMap, + retained_generations: GenerationMap, +} + +impl CommittedRoot { + pub fn new( + repositories: RepoMap, + index: LayeredObjectIndex, + terminal_statuses: TerminalStatusMap, + shard_committed_sequences: ShardSequenceMap, + retained_generations: GenerationMap, + ) -> Self { + Self { + repositories, + index, + terminal_statuses, + shard_committed_sequences, + retained_generations, + } + } + + pub fn repositories(&self) -> &RepoMap { + &self.repositories + } + + pub fn repo(&self, namespace: &NamespaceId) -> Option<&Arc> { + self.repositories.get(namespace) + } + + pub fn index(&self) -> &LayeredObjectIndex { + &self.index + } + + pub fn terminal_statuses(&self) -> &TerminalStatusMap { + &self.terminal_statuses + } + + pub fn terminal_status(&self, key: &OperationKey) -> Option<&TerminalStatusEntry> { + self.terminal_statuses.get(key) + } + + pub fn shard_committed_sequences(&self) -> &ShardSequenceMap { + &self.shard_committed_sequences + } + + pub fn shard_committed_sequence(&self, shard_index: u16) -> Option { + self.shard_committed_sequences.get(&shard_index).copied() + } + + pub fn retained_generations(&self) -> &GenerationMap { + &self.retained_generations + } + + /// Reference proof used by finalized-staging cleanup. + /// + /// The answer comes from the captured committed root's live generation + /// ownership, never from staging's mutable bookkeeping. + pub fn references_path(&self, path: &Path) -> bool { + self.retained_generations + .values() + .any(|generation| generation.references_path(path)) + } + + /// Staging-facing spelling of [`Self::references_path`]. + pub fn references_artifact(&self, path: &Path) -> bool { + self.references_path(path) + } + + pub fn references_generation(&self, id: GenerationId) -> bool { + self.retained_generations.contains_key(&id) + } + + /// Resolve a replayed active-tail index generation to its retained pin. + pub fn retained_tail( + &self, + shard_index: u16, + logical_generation: u64, + ) -> Option<&RetainedTail> { + self.retained_generations + .values() + .filter(|generation| generation.id.shard_index == shard_index) + .find_map(|generation| generation.retained_tail(logical_generation)) + } + + /// Resolve a sealed-run index generation to its retained segment pin. + pub fn retained_segment( + &self, + shard_index: u16, + logical_generation: u64, + ) -> Option<&RetainedSegment> { + self.retained_generations + .values() + .filter(|generation| generation.id.shard_index == shard_index) + .find_map(|generation| generation.retained_segment(logical_generation)) + } + + /// Resolve an adopted projection generation to its retained artifact. + pub fn retained_projection_artifact( + &self, + shard_index: u16, + logical_generation: u64, + ) -> Option<&RetainedProjectionArtifact> { + self.retained_generations + .values() + .filter(|generation| generation.id.shard_index == shard_index) + .find_map(|generation| generation.retained_projection_artifact(logical_generation)) + } + + /// Resolve the decoder and live pin for an index location. + /// + /// A logical generation may be retained by several manifest generations, + /// but every occurrence must name the same path and source kind. A + /// segment/chunk collision is corruption, never an arbitrary lookup + /// preference. + pub fn object_source( + &self, + shard_index: u16, + logical_generation: u64, + ) -> Result>, StoreError> { + let mut found: Option> = None; + for generation in self + .retained_generations + .values() + .filter(|generation| generation.id.shard_index == shard_index) + { + let candidates = [ + generation + .retained_segment(logical_generation) + .map(RetainedObjectSource::Segment), + generation + .retained_tail(logical_generation) + .map(RetainedObjectSource::ActiveTail), + generation + .retained_projection_artifact(logical_generation) + .map(RetainedObjectSource::ProjectionArtifact), + ]; + for candidate in candidates.into_iter().flatten() { + if let Some(existing) = found { + if std::mem::discriminant(&existing) != std::mem::discriminant(&candidate) + || existing.path() != candidate.path() + { + return Err(StoreError::Corruption(format!( + "shard {shard_index} generation {logical_generation} names \ + multiple object sources" + ))); + } + } else { + found = Some(candidate); + } + } + } + Ok(found) + } + + /// Purely merge one fenced shard publication into this root. + /// + /// Reapplying a subtree whose shard sequence is already present is a + /// no-op. This makes the CAS retry path idempotent in effect while still + /// allowing a subtree from another shard to merge against a newer root. + /// No object reachable through `self` is mutated. + pub fn merge(&self, subtree: &ShardSubtree) -> CommittedRoot { + if self + .shard_committed_sequence(subtree.shard_index) + .is_some_and(|published| published >= subtree.shard_committed_sequence) + { + return self.clone(); + } + + let mut repositories = self.repositories.clone(); + for (namespace, state) in &subtree.repositories { + repositories.insert(*namespace, Arc::clone(state)); + } + + let mut terminal_statuses = self.terminal_statuses.clone(); + for key in &subtree.terminal_status_removals { + terminal_statuses.remove(key); + } + for (key, status) in &subtree.terminal_statuses { + terminal_statuses.insert(*key, status.clone()); + } + + let mut shard_committed_sequences = self.shard_committed_sequences.clone(); + shard_committed_sequences.insert(subtree.shard_index, subtree.shard_committed_sequence); + + let mut retained_generations = self.retained_generations.clone(); + for id in &subtree.retained_generation_removals { + retained_generations.remove(id); + } + for (id, generation) in &subtree.retained_generations { + retained_generations.insert(*id, Arc::clone(generation)); + } + + Self { + repositories, + index: self.index.with_subtree(subtree), + terminal_statuses, + shard_committed_sequences, + retained_generations, + } + } +} + +/// Complete immutable output of one fenced group, before root merge. +#[derive(Clone, Debug)] +pub struct ShardSubtree { + pub shard_index: u16, + pub shard_committed_sequence: u64, + pub index_delta: Arc, + /// When present, the newly installed sealed runs cover every delta layer + /// from this shard through the named sequence, including this subtree's + /// delta when the value reaches `shard_committed_sequence`. + pub sealed_through_shard_sequence: Option, + /// Any newly installed runs, newest-first. Most groups leave this empty. + pub sealed_runs_newest_first: Vector>, + pub repositories: RepoMap, + /// Terminal rows whose tombstone horizon has ended. Removals are applied + /// before replacements, which keeps retrying the same subtree idempotent. + pub terminal_status_removals: Vector, + pub terminal_statuses: TerminalStatusMap, + /// Generation pins released by the same publication after every newer + /// root/index reference has been installed. + pub retained_generation_removals: Vector, + pub retained_generations: GenerationMap, +} + +impl ShardSubtree { + #[allow(clippy::too_many_arguments)] + pub fn new( + shard_index: u16, + shard_committed_sequence: u64, + index_delta: Arc, + sealed_through_shard_sequence: Option, + sealed_runs_newest_first: Vector>, + repositories: RepoMap, + terminal_statuses: TerminalStatusMap, + retained_generations: GenerationMap, + ) -> Self { + Self { + shard_index, + shard_committed_sequence, + index_delta, + sealed_through_shard_sequence, + sealed_runs_newest_first, + repositories, + terminal_status_removals: Vector::new(), + terminal_statuses, + retained_generation_removals: Vector::new(), + retained_generations, + } + } +} + +/// Phase stored in the bounded transient status root. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum StatusPhase { + Pending(PendingPhase), + Resolving, +} + +/// One transient operation reservation. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct StatusEntry { + pub operation_digest: ObjectId, + pub retry_until_micros: i64, + pub phase: StatusPhase, + pub shard_sequence: Option, +} + +impl StatusEntry { + pub fn pending( + operation_digest: ObjectId, + retry_until_micros: i64, + phase: PendingPhase, + ) -> Self { + Self { + operation_digest, + retry_until_micros, + phase: StatusPhase::Pending(phase), + shard_sequence: None, + } + } + + pub fn resolving( + operation_digest: ObjectId, + retry_until_micros: i64, + shard_sequence: Option, + ) -> Self { + Self { + operation_digest, + retry_until_micros, + phase: StatusPhase::Resolving, + shard_sequence, + } + } + + pub fn transaction_status(self) -> TransactionStatus { + match self.phase { + StatusPhase::Pending(phase) => TransactionStatus::Pending { + operation_digest: self.operation_digest, + retry_until_micros: self.retry_until_micros, + phase, + }, + StatusPhase::Resolving => TransactionStatus::Resolving { + operation_digest: self.operation_digest, + retry_until_micros: self.retry_until_micros, + shard_sequence: self.shard_sequence, + }, + } + } +} + +/// Result of atomically reconsidering a reservation against one status root. +/// +/// The engine reruns this operation after an `ArcSwap` CAS failure. Existing +/// IDs are examined before the capacity bound, preserving attachment and +/// mismatch semantics under overload. +#[derive(Clone, Debug)] +pub enum StatusReservation { + Attached(StatusEntry), + OperationIdMismatch { + existing_digest: ObjectId, + submitted_digest: ObjectId, + }, + AtCapacity { + limit: u64, + }, + Inserted(OperationStatusRoot), +} + +/// Observable capacity metrics for the status-root CAS owner. +/// +/// These counters deliberately live beside, not inside, the immutable root. +/// The engine records occupancy only after a successful CAS and records a +/// rejection only after the retry loop reaches a final `AtCapacity` result; +/// speculative CAS attempts therefore cannot inflate either value. +#[derive(Debug, Default)] +pub struct OperationStatusMetrics { + occupancy: AtomicU64, + rejections: AtomicU64, +} + +impl OperationStatusMetrics { + pub fn record_published(&self, root: &OperationStatusRoot) { + self.occupancy.store(root.len() as u64, Ordering::Relaxed); + } + + pub fn record_rejection(&self) { + self.rejections.fetch_add(1, Ordering::Relaxed); + } + + pub fn snapshot(&self) -> OperationStatusMetricSnapshot { + OperationStatusMetricSnapshot { + occupancy: self.occupancy.load(Ordering::Relaxed), + rejections: self.rejections.load(Ordering::Relaxed), + } + } +} + +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub struct OperationStatusMetricSnapshot { + pub occupancy: u64, + pub rejections: u64, +} + +/// Bounded immutable map of operations that have not reached terminal state. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OperationStatusRoot { + entries: HashMap, + max_entries: u64, +} + +impl OperationStatusRoot { + pub fn new(max_entries: u64) -> Self { + assert!( + max_entries > 0, + "max_status_entries is validated as nonzero at startup" + ); + Self { + entries: HashMap::new(), + max_entries, + } + } + + pub fn entries(&self) -> &HashMap { + &self.entries + } + + pub fn get(&self, key: &OperationKey) -> Option<&StatusEntry> { + self.entries.get(key) + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub fn max_entries(&self) -> u64 { + self.max_entries + } + + /// Reconsider and, when possible, immutably insert one reservation. + /// + /// The caller must publish an `Inserted` root with CAS and retry this + /// method on contention. That retry is what makes the bound and insertion + /// atomic under concurrent reservations. + pub fn reserve(&self, key: OperationKey, entry: StatusEntry) -> StatusReservation { + if let Some(existing) = self.entries.get(&key).copied() { + if existing.operation_digest == entry.operation_digest { + return StatusReservation::Attached(existing); + } + return StatusReservation::OperationIdMismatch { + existing_digest: existing.operation_digest, + submitted_digest: entry.operation_digest, + }; + } + + if self.entries.len() as u64 >= self.max_entries { + return StatusReservation::AtCapacity { + limit: self.max_entries, + }; + } + + let mut entries = self.entries.clone(); + entries.insert(key, entry); + StatusReservation::Inserted(Self { + entries, + max_entries: self.max_entries, + }) + } + + /// Replace an already reserved entry without applying the admission + /// bound. In particular, a transition to `Resolving` can never be evicted + /// or refused because unrelated entries filled the root afterward. + pub fn replace_existing(&self, key: OperationKey, entry: StatusEntry) -> Option { + let existing = self.entries.get(&key)?; + if existing.operation_digest != entry.operation_digest { + return None; + } + let mut entries = self.entries.clone(); + entries.insert(key, entry); + Some(Self { + entries, + max_entries: self.max_entries, + }) + } + + pub fn without(&self, key: &OperationKey) -> Self { + let mut entries = self.entries.clone(); + entries.remove(key); + Self { + entries, + max_entries: self.max_entries, + } + } + + pub fn without_all<'a>(&self, keys: impl IntoIterator) -> Self { + let mut entries = self.entries.clone(); + for key in keys { + entries.remove(key); + } + Self { + entries, + max_entries: self.max_entries, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::index::{IndexKey, IndexLocation}; + + fn id(byte: u8) -> ObjectId { + ObjectId([byte; 32]) + } + + fn namespace(byte: u8) -> NamespaceId { + NamespaceId([byte; 32]) + } + + fn operation(byte: u8) -> OperationId { + OperationId([byte; 16]) + } + + fn repo(sequence: u64) -> Arc { + Arc::new(RepoState { + repo_sequence: sequence, + current_authority: id(1), + genesis_authority: id(2), + refs: TypedRefMap::new(), + lifecycle: NamespaceLifecycle::Active, + storage_mode: NamespaceStorageMode::Full, + previous_event_digest: id(3), + }) + } + + fn delta(namespace: NamespaceId, object: ObjectId, location: IndexLocation) -> Arc { + let mut delta = IndexDelta::new(100, 1 << 20); + delta + .insert(IndexKey::new(namespace, object), location) + .expect("insert"); + Arc::new(delta) + } + + fn subtree( + shard: u16, + shard_sequence: u64, + namespace: NamespaceId, + repo_sequence: u64, + index_delta: Arc, + ) -> ShardSubtree { + let mut repositories = RepoMap::new(); + repositories.insert(namespace, repo(repo_sequence)); + ShardSubtree::new( + shard, + shard_sequence, + index_delta, + None, + Vector::new(), + repositories, + TerminalStatusMap::new(), + GenerationMap::new(), + ) + } + + #[test] + fn duplicate_subtree_is_idempotent_in_effect() { + let ns = namespace(1); + let object = id(7); + let location = IndexLocation { + segment_generation: 3, + frame_offset: 40, + frame_len: 50, + object_type: 1, + shard_sequence: 9, + }; + let mut subtree = subtree(0, 9, ns, 1, delta(ns, object, location)); + let key = OperationKey::new(ns, operation(4)); + subtree.terminal_statuses.insert( + key, + TerminalStatusEntry::Expired(ReceiptTombstone { + operation_digest: id(8), + retry_until_micros: 100, + tombstone_until_micros: 200, + }), + ); + + let once = CommittedRoot::default().merge(&subtree); + let twice = once.merge(&subtree); + + assert_eq!(twice.repositories().len(), 1); + assert_eq!(twice.terminal_statuses().len(), 1); + assert_eq!(twice.index().delta_layer_count(), 1); + assert_eq!(twice.shard_committed_sequence(0), Some(9)); + assert_eq!( + twice.index().get(&IndexKey::new(ns, object)), + Some(location) + ); + } + + #[test] + fn subtree_can_prune_terminal_rows_without_rebuilding_the_table() { + let ns = namespace(1); + let removed = OperationKey::new(ns, operation(4)); + let retained = OperationKey::new(ns, operation(5)); + let mut statuses = TerminalStatusMap::new(); + for key in [removed, retained] { + statuses.insert( + key, + TerminalStatusEntry::Expired(ReceiptTombstone { + operation_digest: id(key.operation_id.0[0]), + retry_until_micros: 100, + tombstone_until_micros: 200, + }), + ); + } + let root = CommittedRoot::new( + RepoMap::new(), + LayeredObjectIndex::default(), + statuses, + ShardSequenceMap::new(), + GenerationMap::new(), + ); + let mut prune = subtree(0, 1, ns, 1, Arc::new(IndexDelta::new(10, 1024))); + prune.terminal_status_removals.push_back(removed); + + let merged = root.merge(&prune); + assert!(merged.terminal_status(&removed).is_none()); + assert!(merged.terminal_status(&retained).is_some()); + } + + #[test] + fn concurrent_shards_merge_without_losing_the_other_subtree() { + let ns_a = namespace(1); + let ns_b = namespace(2); + let empty_a = Arc::new(IndexDelta::new(10, 1024)); + let empty_b = Arc::new(IndexDelta::new(10, 1024)); + let a = subtree(0, 4, ns_a, 4, empty_a); + let b = subtree(1, 8, ns_b, 7, empty_b); + + let after_a = CommittedRoot::default().merge(&a); + let after_b_retry = after_a.merge(&b); + + assert_eq!(after_b_retry.repo(&ns_a).unwrap().repo_sequence, 4); + assert_eq!(after_b_retry.repo(&ns_b).unwrap().repo_sequence, 7); + assert_eq!(after_b_retry.shard_committed_sequence(0), Some(4)); + assert_eq!(after_b_retry.shard_committed_sequence(1), Some(8)); + } + + #[test] + fn sealed_publication_discards_only_covered_layers_from_its_shard() { + let ns_a = namespace(1); + let ns_b = namespace(2); + let a = subtree( + 0, + 1, + ns_a, + 1, + delta( + ns_a, + id(10), + IndexLocation { + segment_generation: 1, + frame_offset: 1, + frame_len: 1, + object_type: 1, + shard_sequence: 1, + }, + ), + ); + let b = subtree( + 1, + 1, + ns_b, + 1, + delta( + ns_b, + id(11), + IndexLocation { + segment_generation: 1, + frame_offset: 2, + frame_len: 1, + object_type: 1, + shard_sequence: 1, + }, + ), + ); + let root = CommittedRoot::default().merge(&a).merge(&b); + assert_eq!(root.index().delta_layer_count(), 2); + + let mut seal = subtree(0, 2, ns_a, 2, Arc::new(IndexDelta::new(10, 1024))); + seal.sealed_through_shard_sequence = Some(2); + let sealed = root.merge(&seal); + + assert_eq!(sealed.index().delta_layer_count(), 1); + let survivor = sealed.index().delta_layers().front().unwrap(); + assert_eq!(survivor.shard_index, 1); + } + + #[test] + fn untouched_repository_arc_is_shared_across_merge() { + let untouched_namespace = namespace(1); + let changed_namespace = namespace(2); + let untouched = repo(5); + let mut repositories = RepoMap::new(); + repositories.insert(untouched_namespace, Arc::clone(&untouched)); + let root = CommittedRoot::new( + repositories, + LayeredObjectIndex::default(), + TerminalStatusMap::new(), + ShardSequenceMap::new(), + GenerationMap::new(), + ); + let update = subtree( + 1, + 1, + changed_namespace, + 1, + Arc::new(IndexDelta::new(10, 1024)), + ); + + let merged = root.merge(&update); + assert!(Arc::ptr_eq( + merged.repo(&untouched_namespace).unwrap(), + &untouched + )); + } + + #[test] + fn status_reservation_checks_identity_before_capacity() { + let ns = namespace(1); + let key = OperationKey::new(ns, operation(1)); + let entry = StatusEntry::pending(id(1), 100, PendingPhase::Queued); + let root = match OperationStatusRoot::new(1).reserve(key, entry) { + StatusReservation::Inserted(root) => root, + other => panic!("expected insertion, got {other:?}"), + }; + + assert!(matches!( + root.reserve(key, entry), + StatusReservation::Attached(found) if found == entry + )); + assert!(matches!( + root.reserve(key, StatusEntry::pending(id(2), 100, PendingPhase::Queued)), + StatusReservation::OperationIdMismatch { .. } + )); + assert!(matches!( + root.reserve( + OperationKey::new(ns, operation(2)), + StatusEntry::pending(id(3), 100, PendingPhase::Queued) + ), + StatusReservation::AtCapacity { limit: 1 } + )); + } + + #[test] + fn resolving_transition_is_not_subject_to_the_capacity_bound() { + let ns = namespace(1); + let key = OperationKey::new(ns, operation(1)); + let root = match OperationStatusRoot::new(1).reserve( + key, + StatusEntry::pending(id(1), 100, PendingPhase::Sequenced), + ) { + StatusReservation::Inserted(root) => root, + other => panic!("expected insertion, got {other:?}"), + }; + + let resolving = StatusEntry::resolving(id(1), 100, Some(9)); + let root = root + .replace_existing(key, resolving) + .expect("existing reservations transition despite full capacity"); + assert_eq!(root.get(&key), Some(&resolving)); + } + + #[test] + fn status_metrics_observe_only_final_publications_and_rejections() { + let metrics = OperationStatusMetrics::default(); + let root = OperationStatusRoot::new(1); + metrics.record_published(&root); + metrics.record_rejection(); + metrics.record_rejection(); + assert_eq!( + metrics.snapshot(), + OperationStatusMetricSnapshot { + occupancy: 0, + rejections: 2, + } + ); + } + + #[test] + fn retained_generation_reference_proof_uses_the_committed_root() { + let temp = tempfile::tempdir().expect("tempdir"); + let segment_path = temp.path().join("1-1-1.seg"); + let tail_path = temp.path().join("2.journal"); + let projection_path = temp.path().join("3.stage-chunk"); + std::fs::write(&segment_path, b"segment").expect("write"); + std::fs::write(&tail_path, b"tail").expect("write"); + std::fs::write(&projection_path, b"projection").expect("write"); + let pinned = RetainedSegment::new( + 1, + [7; 16], + 1, + 1, + PinnedFile::open(segment_path.clone()).expect("pin"), + ); + let tail = RetainedTail::new( + 2, + PinnedFile::open(tail_path.clone()).expect("pin active tail"), + ); + let generation = Arc::new(RetainedGeneration::new( + GenerationId::new(0, 1), + Arc::from([pinned]), + Arc::from([]), + Arc::from([]), + Arc::from([tail]), + Arc::from([RetainedProjectionArtifact::new( + 3, + ProjectionArtifactFormat::CanonicalStageChunkV1, + PinnedFile::open(projection_path.clone()).expect("pin projection artifact"), + )]), + )); + let mut generations = GenerationMap::new(); + generations.insert(generation.id, generation); + let root = CommittedRoot::new( + RepoMap::new(), + LayeredObjectIndex::default(), + TerminalStatusMap::new(), + ShardSequenceMap::new(), + generations, + ); + + assert!(root.references_path(&segment_path)); + assert!(root.references_path(&tail_path)); + assert!(root.references_path(&projection_path)); + assert!(root.references_generation(GenerationId::new(0, 1))); + assert_eq!( + root.retained_tail(0, 2).map(RetainedTail::path), + Some(tail_path.as_path()) + ); + assert_eq!( + root.retained_segment(0, 1).map(RetainedSegment::path), + Some(segment_path.as_path()) + ); + assert_eq!( + root.retained_projection_artifact(0, 3) + .map(RetainedProjectionArtifact::path), + Some(projection_path.as_path()) + ); + assert!(matches!( + root.object_source(0, 3).expect("unambiguous source"), + Some(RetainedObjectSource::ProjectionArtifact(_)) + )); + assert!(root.retained_tail(1, 2).is_none()); + assert!(root.retained_segment(1, 1).is_none()); + assert!(!root.references_path(&temp.path().join("unreferenced.seg"))); + } +} diff --git a/crates/levcs-store/src/segment.rs b/crates/levcs-store/src/segment.rs index 562e81f..9697aef 100644 --- a/crates/levcs-store/src/segment.rs +++ b/crates/levcs-store/src/segment.rs @@ -18,7 +18,7 @@ use crate::format::{ CurrentPointer, Frame, FrameError, JournalHeader, Manifest, SegmentFooter, FORMAT_MARKER_LEN, JOURNAL_HEADER_LEN, SEGMENT_FOOTER_LOCATOR_LEN, STORAGE_VERSION, }; -use crate::journal::Journal; +use crate::journal::{Journal, TailScan}; use crate::sys; use crate::types::{DurabilityCounters, StoreError}; @@ -304,6 +304,228 @@ pub fn seal_journal( Ok(destination) } +/// Seal a recovery-validated prefix without ever modifying its source journal. +/// +/// The crash image is forensic evidence. Recovery therefore copies exactly +/// `0..scan.stop_offset` into a uniquely named, fenced artifact, opens that +/// copy through the normal journal scanner, and seals the copy through +/// [`seal_journal`]. The original descriptor is read-only throughout. +/// +/// A crash may leave the deterministic final segment installed but not yet +/// referenced by a manifest. Re-recovery accepts that artifact only after its +/// footer, frame index, and every byte of the validated prefix agree with the +/// source. An occupied name with different contents is corruption, never an +/// overwrite. +pub fn seal_recovered_prefix( + source: &File, + header: &JournalHeader, + scan: &TailScan, + paths: &ShardPaths, + generation: u64, + counters: &Arc, +) -> Result { + if scan.frames.is_empty() { + return Err(StoreError::Corruption( + "recovery sealing refused: the validated prefix contains no frame".into(), + )); + } + if scan.frames.first().map(|frame| frame.offset) != Some(JOURNAL_HEADER_LEN as u64) { + return Err(StoreError::Corruption( + "recovery sealing refused: the validated prefix does not start after the header".into(), + )); + } + let last_end = scan + .frames + .last() + .and_then(|frame| frame.offset.checked_add(frame.len)) + .ok_or_else(|| { + StoreError::Corruption("recovery sealing refused: invalid frame range".into()) + })?; + if last_end != scan.stop_offset { + return Err(StoreError::Corruption(format!( + "recovery sealing refused: frame prefix ends at {last_end}, scan stops at {}", + scan.stop_offset + ))); + } + + let first = scan.frames.first().expect("non-empty").shard_sequence; + let last = scan.frames.last().expect("non-empty").shard_sequence; + let destination = paths.segment(generation, first, last); + let artifact = recovery_prefix_path(paths, header, generation); + if destination.exists() { + validate_recovered_segment(source, header, scan, &destination, generation)?; + if artifact.exists() { + let artifact_file = File::open(&artifact)?; + let compare_len = artifact_file.metadata()?.len().min(scan.stop_offset); + compare_prefix(source, &artifact_file, compare_len, &artifact)?; + sys::unlink(&artifact)?; + sys::fsync_dir(&paths.segments(), counters)?; + } + return Ok(destination); + } + + let mut copied = File::options() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(&artifact)?; + let existing_len = copied.metadata()?.len(); + let compare_len = existing_len.min(scan.stop_offset); + compare_prefix(source, &copied, compare_len, &artifact)?; + if existing_len > scan.stop_offset { + // A crash after footer append but before publishing the segment may + // leave a sealed or partially sealed construction artifact. Its exact + // prefix proves ownership; discard only the construction suffix and + // run the normal seal again. + sys::truncate(&copied, scan.stop_offset, counters)?; + } else if existing_len < scan.stop_offset { + copy_exact_prefix( + source, + &mut copied, + existing_len, + scan.stop_offset, + counters, + )?; + } + sys::fdatasync(&copied, counters)?; + sys::fsync_dir(&paths.segments(), counters)?; + drop(copied); + + let (mut journal, copied_scan) = + Journal::open(&artifact, &header.root_uuid, Arc::clone(counters))?; + if journal.header() != header + || copied_scan.frames != scan.frames + || copied_scan.stop_offset != scan.stop_offset + { + return Err(StoreError::Corruption( + "the fenced recovery prefix did not reopen as the source prefix".into(), + )); + } + + let installed = seal_journal(&mut journal, paths, generation, counters)?; + validate_recovered_segment(source, header, scan, &installed, generation)?; + + // The installed segment is a hard link to the now-sealed copy. Once its + // own name is fenced the uniquely named construction artifact is dead. + sys::unlink(&artifact)?; + sys::fsync_dir(&paths.segments(), counters)?; + Ok(installed) +} + +fn recovery_prefix_path(paths: &ShardPaths, header: &JournalHeader, generation: u64) -> PathBuf { + paths.segments().join(format!( + ".recovery-{}-{generation}.prefix", + hex::encode(header.journal_id) + )) +} + +fn copy_exact_prefix( + source: &File, + destination: &mut File, + from: u64, + through: u64, + counters: &DurabilityCounters, +) -> Result<(), StoreError> { + const COPY_CHUNK: usize = 1 << 20; + let mut offset = from; + sys::seek_to(destination, from)?; + while offset < through { + let remaining = through - offset; + let len = usize::try_from(remaining.min(COPY_CHUNK as u64)) + .map_err(|_| StoreError::Corruption("recovery prefix length overflow".into()))?; + let mut bytes = vec![0u8; len]; + sys::pread_exact(source, offset, &mut bytes)?; + let end = sys::write_vectored_all(destination, &[IoSlice::new(&bytes)], counters)?; + let expected = offset + .checked_add(len as u64) + .ok_or_else(|| StoreError::Corruption("recovery prefix length overflow".into()))?; + if end != expected { + return Err(StoreError::Corruption(format!( + "short recovery-prefix copy: expected cursor {expected}, got {end}" + ))); + } + offset = expected; + } + Ok(()) +} + +fn compare_prefix( + source: &File, + artifact: &File, + through: u64, + artifact_path: &Path, +) -> Result<(), StoreError> { + const COMPARE_CHUNK: usize = 1 << 20; + let mut offset = 0u64; + while offset < through { + let remaining = through - offset; + let len = usize::try_from(remaining.min(COMPARE_CHUNK as u64)) + .map_err(|_| StoreError::Corruption("recovery prefix length overflow".into()))?; + let mut original = vec![0u8; len]; + let mut retained = vec![0u8; len]; + sys::pread_exact(source, offset, &mut original)?; + sys::pread_exact(artifact, offset, &mut retained)?; + if original != retained { + return Err(StoreError::Corruption(format!( + "recovery construction artifact {} differs from its source at offset {offset}", + artifact_path.display() + ))); + } + offset += len as u64; + } + Ok(()) +} + +fn validate_recovered_segment( + source: &File, + header: &JournalHeader, + scan: &TailScan, + destination: &Path, + generation: u64, +) -> Result<(), StoreError> { + let reader = SegmentReader::open(destination, &header.root_uuid)?; + let footer = reader.footer(); + let expected_offsets: Vec<(u64, u64, u64)> = scan + .frames + .iter() + .map(|frame| (frame.shard_sequence, frame.offset, frame.len)) + .collect(); + if reader.journal_header()? != *header + || footer.journal_id != header.journal_id + || footer.generation != generation + || footer.first_shard_sequence != scan.frames.first().expect("non-empty").shard_sequence + || footer.last_shard_sequence != scan.frames.last().expect("non-empty").shard_sequence + || footer.offsets != expected_offsets + { + return Err(StoreError::Corruption(format!( + "occupied recovery segment {} does not describe the validated prefix", + destination.display() + ))); + } + + const COMPARE_CHUNK: usize = 1 << 20; + let segment = File::open(destination)?; + let mut offset = 0u64; + while offset < scan.stop_offset { + let remaining = scan.stop_offset - offset; + let len = usize::try_from(remaining.min(COMPARE_CHUNK as u64)) + .map_err(|_| StoreError::Corruption("recovery prefix length overflow".into()))?; + let mut original = vec![0u8; len]; + let mut installed = vec![0u8; len]; + sys::pread_exact(source, offset, &mut original)?; + sys::pread_exact(&segment, offset, &mut installed)?; + if original != installed { + return Err(StoreError::Corruption(format!( + "occupied recovery segment {} differs from the crash prefix at offset {offset}", + destination.display() + ))); + } + offset += len as u64; + } + Ok(()) +} + /// Step 6 of scope 3.4: drop the `active/` name, now that the manifest /// generation naming the segment is durable. /// @@ -364,12 +586,37 @@ pub fn install_manifest( let final_path = paths.manifest(manifest.generation); let tmp = manifests.join(format!("{}.tmp", manifest_filename(manifest.generation))); + let encoded = manifest.encode()?; // 1, 2 - write_fenced(&tmp, &manifest.encode()?, counters)?; + if tmp.exists() { + let existing = std::fs::read(&tmp)?; + if existing != encoded { + return Err(StoreError::Corruption(format!( + "manifest temp for generation {} contains different bytes", + manifest.generation + ))); + } + let file = File::open(&tmp)?; + // Exact visible bytes do not prove the interrupted attempt reached its + // fence. Fence them again before publishing the final name. + sys::fdatasync(&file, counters)?; + } else { + write_fenced(&tmp, &encoded, counters)?; + } // 3 — never overwrite an existing generation. if let Err(e) = sys::rename_noreplace(&tmp, &final_path) { + if e.kind() != std::io::ErrorKind::AlreadyExists { + let _ = sys::unlink(&tmp); + return Err(e.into()); + } + let existing = std::fs::read(&final_path)?; let _ = sys::unlink(&tmp); - return Err(e.into()); + if existing != encoded { + return Err(StoreError::Corruption(format!( + "manifest generation {} already exists with different contents", + manifest.generation + ))); + } } // 4 sys::fsync_dir(&manifests, counters)?; diff --git a/crates/levcs-store/src/staging.rs b/crates/levcs-store/src/staging.rs index 1482114..a8d1548 100644 --- a/crates/levcs-store/src/staging.rs +++ b/crates/levcs-store/src/staging.rs @@ -1,9 +1,514 @@ -//! Bounded invisible projection-staging sessions: begin, idempotent chunk-put, -//! read-only resolver, seal to `StagedProjectionInstallV1`, abort, expiry, and -//! cleanup. +//! Bounded invisible projection-staging sessions. //! -//! **Owned by B1 NamespaceTxn** (scope 2.1, 6-B1). Empty in D0. -//! -//! Sealing cannot publish membership. Only `submit(ValidatedTransaction)` may -//! adopt a sealed descriptor, so possession of a session ID never authorizes -//! publication (plan §4 identity invariant 7, §8). +//! B3 owns the storage mechanism. D0-B owns and freezes the adoption seam in +//! this file so B1 never has to read B3's on-disk representation directly. + +use std::fmt; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use levcs_core::ObjectId; +use levcs_protocol::v2::{ + ProjectionStageManifestV1, ProjectionStageSessionV1, StagedProjectionInstallV1, +}; + +use crate::index::{IndexDelta, IndexRun}; +use crate::roots::{CommittedRoot, RetainedIndexRun, RetainedProjectionArtifact}; +use crate::types::{NamespaceId, StoreError}; + +/// One immutable artifact offered for adoption. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProjectionArtifact { + pub path: PathBuf, + pub digest: ObjectId, + pub bytes: u64, +} + +/// Everything B1 may inspect while revalidating a sealed staged projection. +/// +/// This is deliberately read-only. Adoption may reject this state but may +/// never repair or mutate it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProjectionAdoptionResolution { + pub session: ProjectionStageSessionV1, + pub manifest: ProjectionStageManifestV1, + pub artifacts: Arc<[ProjectionArtifact]>, +} + +/// The only three ways ownership of an admitted adoption pin may end. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum ProjectionAdoptionOutcome { + Adopted, + DefinitivePreAppendFailure, + TransferredToRecovery, +} + +/// Resolution delivered to staging after production recovery has made the +/// final frame authoritative or proved that no complete frame exists. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum RecoveredProjectionOutcome { + Committed, + ProvedAbsent, +} + +/// Recovery's resolution of a staging pin transferred across a poisoned +/// process boundary. +/// +/// B3 consumes these notifications before expiry or cleanup may inspect the +/// recovered shard. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct RecoveredProjectionResolution { + pub session_id: [u8; 16], + pub outcome: RecoveredProjectionOutcome, +} + +/// Read-only recovery result for one authoritative staged-install frame. +/// +/// A final frame carries only a canonical descriptor, not the installed +/// object's locations. Recovery obtains those locations and live file +/// ownership from B3 through this value. The vectors are newest-first where +/// order is observable by index lookup. +/// +/// The fields remain private so recovery can compose the result into its +/// recovered root but cannot mutate staging state or reinterpret B3's on-disk +/// representation. +#[derive(Clone, Debug)] +pub(crate) struct RecoveredProjectionArtifacts { + descriptor: StagedProjectionInstallV1, + index_delta: Arc, + /// Newest-first; each retained entry is also the lookup run, so lookup + /// membership cannot accidentally diverge from physical ownership. + retained_index_runs: Arc<[RetainedIndexRun]>, + retained_artifacts: Arc<[RetainedProjectionArtifact]>, +} + +impl RecoveredProjectionArtifacts { + /// Construct the complete physical result of resolving one descriptor. + /// + /// B3 calls this only after mechanically verifying the descriptor against + /// the sealed session, manifest, artifact hashes, and index. Identity, + /// graph, policy, authority, and federation decisions are deliberately + /// absent from this interface. + pub(crate) fn new( + descriptor: StagedProjectionInstallV1, + index_delta: Arc, + retained_index_runs: Arc<[RetainedIndexRun]>, + retained_artifacts: Arc<[RetainedProjectionArtifact]>, + ) -> Result { + let run_entries = retained_index_runs + .iter() + .try_fold(0u64, |count, retained| { + count.checked_add(retained.run().entry_count()) + }) + .ok_or_else(|| { + StoreError::Corruption( + "recovered staged-projection index entry count overflowed".into(), + ) + })?; + let delta_entries = u64::try_from(index_delta.len()).map_err(|_| { + StoreError::Corruption( + "recovered staged-projection delta count does not fit u64".into(), + ) + })?; + let total_entries = run_entries.checked_add(delta_entries).ok_or_else(|| { + StoreError::Corruption( + "recovered staged-projection total index count overflowed".into(), + ) + })?; + if total_entries == 0 { + return Err(StoreError::Corruption( + "committed staged projection resolved without object membership".into(), + )); + } + if total_entries != descriptor.object_count { + return Err(StoreError::Corruption(format!( + "committed staged projection declares {} objects but its recovered \ + index contains {total_entries}", + descriptor.object_count + ))); + } + if retained_artifacts.is_empty() { + return Err(StoreError::Corruption( + "committed staged projection resolved without live artifact ownership".into(), + )); + } + + Ok(Self { + descriptor, + index_delta, + retained_index_runs, + retained_artifacts, + }) + } + + pub(crate) fn descriptor(&self) -> &StagedProjectionInstallV1 { + &self.descriptor + } + + pub(crate) fn index_delta(&self) -> &Arc { + &self.index_delta + } + + pub(crate) fn index_runs_newest_first(&self) -> impl ExactSizeIterator> { + self.retained_index_runs.iter().map(RetainedIndexRun::run) + } + + pub(crate) fn retained_index_runs(&self) -> &[RetainedIndexRun] { + &self.retained_index_runs + } + + pub(crate) fn retained_artifacts(&self) -> &[RetainedProjectionArtifact] { + &self.retained_artifacts + } +} + +/// The complete staging-owned seam used by production recovery. +/// +/// `resolve_committed` is read-only: the complete frame is already the +/// durable authority and resolution may inspect, open, hash, and pin its +/// immutable artifacts but may not repair them. `notify_recovered` is the +/// separate lifecycle transition performed only after recovery has either +/// incorporated a committed result into the recovered root or proved absence +/// by scanning the complete authoritative journal history. +pub(crate) trait ProjectionRecoveryResolver: Send + Sync { + /// Sessions whose in-process adoption handle transferred responsibility + /// to recovery before the previous engine stopped. + fn transferred_sessions(&self, shard_index: u16) -> Result, StoreError>; + + /// Resolve a canonical committed descriptor into exact namespace-scoped + /// membership and live physical ownership. + fn resolve_committed( + &self, + namespace: NamespaceId, + descriptor: &StagedProjectionInstallV1, + ) -> Result; + + /// Finish one transferred session after the physical-state proof is + /// complete. This transition must be idempotent: recovery remains unready + /// if a later notification fails and repeats every notification on the + /// next attempt. + fn notify_recovered(&self, resolution: RecoveredProjectionResolution) + -> Result<(), StoreError>; +} + +/// B3's implementation behind the opaque handle. +/// +/// The trait and constructor are crate-private: external callers may carry a +/// handle issued by staging but cannot forge one. +pub(crate) trait ProjectionAdoptionLifecycle: Send + Sync { + fn resolution(&self) -> Result, StoreError>; + + fn finish(&self, outcome: ProjectionAdoptionOutcome) -> Result<(), StoreError>; + + fn dropped_without_outcome(&self); +} + +/// An unforgeable staging capability consumed alongside the wire descriptor. +/// +/// Holding this value pins the sealed artifacts. Exactly one terminal outcome +/// must be recorded before it is dropped. +pub struct ProjectionAdoption { + lifecycle: Arc, + finished: bool, +} + +impl ProjectionAdoption { + pub(crate) fn new(lifecycle: Arc) -> Self { + Self { + lifecycle, + finished: false, + } + } + + pub(crate) fn resolution(&self) -> Result, StoreError> { + self.lifecycle.resolution() + } + + /// Prove artifact retention against the state readers actually capture. + pub(crate) fn artifact_is_referenced(&self, root: &CommittedRoot, path: &Path) -> bool { + root.references_path(path) + } + + pub(crate) fn finish(mut self, outcome: ProjectionAdoptionOutcome) -> Result<(), StoreError> { + self.lifecycle.finish(outcome)?; + self.finished = true; + Ok(()) + } +} + +impl fmt::Debug for ProjectionAdoption { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ProjectionAdoption") + .field("finished", &self.finished) + .finish_non_exhaustive() + } +} + +impl Drop for ProjectionAdoption { + fn drop(&mut self) { + if !self.finished { + self.lifecycle.dropped_without_outcome(); + } + } +} + +/// D0-B's exact B1/B3 handoff payload. +pub(crate) struct StagedProjectionAdoption { + pub(crate) descriptor: StagedProjectionInstallV1, + pub(crate) handle: ProjectionAdoption, +} + +#[cfg(test)] +mod tests { + use std::fs::File; + use std::sync::Mutex; + + use levcs_protocol::v2::{ProjectionMode, StageSourceKindV1}; + use tempfile::TempDir; + + use super::*; + use crate::index::{IndexKey, IndexLocation}; + use crate::roots::{PinnedFile, ProjectionArtifactFormat}; + + struct RecordingLifecycle { + resolution: Arc, + outcomes: Mutex>, + dropped: Mutex, + } + + impl ProjectionAdoptionLifecycle for RecordingLifecycle { + fn resolution(&self) -> Result, StoreError> { + Ok(Arc::clone(&self.resolution)) + } + + fn finish(&self, outcome: ProjectionAdoptionOutcome) -> Result<(), StoreError> { + self.outcomes.lock().unwrap().push(outcome); + Ok(()) + } + + fn dropped_without_outcome(&self) { + *self.dropped.lock().unwrap() += 1; + } + } + + fn resolution() -> Arc { + Arc::new(ProjectionAdoptionResolution { + session: ProjectionStageSessionV1 { + session_id: [1; 16], + destination_repo: ObjectId([2; 32]), + destination_genesis: ObjectId([3; 32]), + expected_authority: ObjectId([4; 32]), + projection: ProjectionMode::Full, + source_kind: StageSourceKindV1::Mirror, + actor: [5; 32], + actor_key_epoch: 7, + source_generation_digest: ObjectId([6; 32]), + fork_proof: None, + final_operation_id: [8; 16], + final_operation_digest: ObjectId([9; 32]), + final_evidence_digest: ObjectId([10; 32]), + total_object_count: 1, + total_object_bytes: 1, + chunk_count: 1, + manifest_digest: ObjectId([11; 32]), + expires_at_micros: 12, + }, + manifest: ProjectionStageManifestV1 { + session_id: [1; 16], + chunk_digests: Vec::new(), + objects: Vec::new(), + membership_root: ObjectId([13; 32]), + }, + artifacts: Arc::from([]), + }) + } + + fn lifecycle() -> Arc { + Arc::new(RecordingLifecycle { + resolution: resolution(), + outcomes: Mutex::new(Vec::new()), + dropped: Mutex::new(0), + }) + } + + fn install() -> StagedProjectionInstallV1 { + StagedProjectionInstallV1 { + session_id: [20; 16], + manifest_digest: ObjectId([21; 32]), + projection: ProjectionMode::Full, + object_count: 1, + object_bytes: 32, + membership_root: ObjectId([22; 32]), + artifact_set_digest: ObjectId([23; 32]), + } + } + + fn recovered_artifacts( + directory: &TempDir, + ) -> Result { + let path = directory.path().join("projection-objects"); + File::create(&path)?; + let mut delta = IndexDelta::new(8, 4096); + delta.insert( + IndexKey::new(NamespaceId([24; 32]), ObjectId([25; 32])), + IndexLocation { + segment_generation: 7, + frame_offset: 128, + frame_len: 256, + object_type: 1, + shard_sequence: 9, + }, + )?; + RecoveredProjectionArtifacts::new( + install(), + Arc::new(delta), + Arc::from([]), + Arc::from([RetainedProjectionArtifact::new( + 7, + ProjectionArtifactFormat::CanonicalStageChunkV1, + PinnedFile::open(path)?, + )]), + ) + } + + struct RecordingRecoveryResolver { + artifacts: RecoveredProjectionArtifacts, + transferred: Arc<[[u8; 16]]>, + notifications: Mutex>, + } + + impl ProjectionRecoveryResolver for RecordingRecoveryResolver { + fn transferred_sessions(&self, _shard_index: u16) -> Result, StoreError> { + Ok(Arc::clone(&self.transferred)) + } + + fn resolve_committed( + &self, + _namespace: NamespaceId, + descriptor: &StagedProjectionInstallV1, + ) -> Result { + if self.artifacts.descriptor() != descriptor { + return Err(StoreError::Corruption( + "descriptor did not name the sealed staging session".into(), + )); + } + Ok(self.artifacts.clone()) + } + + fn notify_recovered( + &self, + resolution: RecoveredProjectionResolution, + ) -> Result<(), StoreError> { + self.notifications.lock().unwrap().push(resolution); + Ok(()) + } + } + + #[test] + fn handle_drop_without_outcome_is_observable() { + let lifecycle = lifecycle(); + let handle = ProjectionAdoption::new(lifecycle.clone()); + drop(handle); + assert_eq!(*lifecycle.dropped.lock().unwrap(), 1); + assert!(lifecycle.outcomes.lock().unwrap().is_empty()); + } + + #[test] + fn each_terminal_outcome_suppresses_the_drop_bug() { + for outcome in [ + ProjectionAdoptionOutcome::Adopted, + ProjectionAdoptionOutcome::DefinitivePreAppendFailure, + ProjectionAdoptionOutcome::TransferredToRecovery, + ] { + let lifecycle = lifecycle(); + ProjectionAdoption::new(lifecycle.clone()) + .finish(outcome) + .unwrap(); + assert_eq!(&*lifecycle.outcomes.lock().unwrap(), &[outcome]); + assert_eq!(*lifecycle.dropped.lock().unwrap(), 0); + } + } + + #[test] + fn recovery_resolution_is_read_only_and_carries_live_ownership() { + let directory = TempDir::new().unwrap(); + let artifacts = recovered_artifacts(&directory).unwrap(); + let retained_path = artifacts.retained_artifacts()[0].path().to_owned(); + let resolver = RecordingRecoveryResolver { + artifacts, + transferred: Arc::from([[20; 16], [26; 16]]), + notifications: Mutex::new(Vec::new()), + }; + + let transferred = resolver.transferred_sessions(3).unwrap(); + assert_eq!(&*transferred, &[[20; 16], [26; 16]]); + let recovered = resolver + .resolve_committed(NamespaceId([24; 32]), &install()) + .unwrap(); + assert_eq!(recovered.descriptor(), &install()); + assert_eq!(recovered.index_delta().len(), 1); + assert_eq!(recovered.index_runs_newest_first().len(), 0); + assert!(recovered.retained_index_runs().is_empty()); + assert_eq!(recovered.retained_artifacts()[0].path(), retained_path); + } + + #[test] + fn recovery_notifies_both_terminal_physical_outcomes() { + let directory = TempDir::new().unwrap(); + let resolver = RecordingRecoveryResolver { + artifacts: recovered_artifacts(&directory).unwrap(), + transferred: Arc::from([]), + notifications: Mutex::new(Vec::new()), + }; + for outcome in [ + RecoveredProjectionOutcome::Committed, + RecoveredProjectionOutcome::ProvedAbsent, + ] { + resolver + .notify_recovered(RecoveredProjectionResolution { + session_id: [20; 16], + outcome, + }) + .unwrap(); + } + assert_eq!( + &*resolver.notifications.lock().unwrap(), + &[ + RecoveredProjectionResolution { + session_id: [20; 16], + outcome: RecoveredProjectionOutcome::Committed, + }, + RecoveredProjectionResolution { + session_id: [20; 16], + outcome: RecoveredProjectionOutcome::ProvedAbsent, + }, + ] + ); + } + + #[test] + fn recovery_rejects_membership_without_live_artifact_ownership() { + let mut delta = IndexDelta::new(8, 4096); + delta + .insert( + IndexKey::new(NamespaceId([24; 32]), ObjectId([25; 32])), + IndexLocation { + segment_generation: 7, + frame_offset: 128, + frame_len: 256, + object_type: 1, + shard_sequence: 9, + }, + ) + .unwrap(); + let error = RecoveredProjectionArtifacts::new( + install(), + Arc::new(delta), + Arc::from([]), + Arc::from([]), + ) + .unwrap_err(); + assert!(matches!(error, StoreError::Corruption(message) if message.contains("ownership"))); + } +} diff --git a/crates/levcs-store/src/transaction.rs b/crates/levcs-store/src/transaction.rs index 926c916..5ea4bc9 100644 --- a/crates/levcs-store/src/transaction.rs +++ b/crates/levcs-store/src/transaction.rs @@ -7,6 +7,7 @@ use levcs_core::{ObjectId, ObjectType}; use levcs_protocol::v2::{TransactionEvidenceV1, TypedRefCas}; +use crate::staging::{ProjectionAdoptionOutcome, StagedProjectionAdoption}; use crate::types::{NamespaceId, OperationId, PrivilegedConstruction, StoreError}; /// The immutable output of the instance pipeline's stages 4-8 (plan §7). @@ -32,12 +33,12 @@ impl ValidatedTransaction { /// Begin construction. Requires a capability token that is not reachable /// from a `&StoreEngine`; see `PrivilegedConstruction` (scope 2.2). pub fn builder(_token: PrivilegedConstruction) -> ValidatedTransactionBuilder { - ValidatedTransactionBuilder { _private: () } + ValidatedTransactionBuilder { adoption: None } } } pub struct ValidatedTransactionBuilder { - _private: (), + adoption: Option, } impl ValidatedTransactionBuilder { @@ -76,9 +77,111 @@ impl ValidatedTransactionBuilder { self } - pub fn build(self) -> Result { + /// Adopt one sealed staged projection. The opaque handle is the adoption + /// pin and is consumed with the canonical wire descriptor so neither half + /// can be forgotten independently. + pub fn adopt_projection( + mut self, + descriptor: levcs_protocol::v2::StagedProjectionInstallV1, + handle: crate::staging::ProjectionAdoption, + ) -> Result { + if let Some(previous) = self.adoption.take() { + // Both pins were admitted, so both receive a terminal outcome + // even though the builder rejects the duplicate. Returning early + // after finishing only one would turn the other drop into the + // lifecycle bug this handle exists to expose. + let previous_result = previous + .handle + .finish(ProjectionAdoptionOutcome::DefinitivePreAppendFailure); + let submitted_result = + handle.finish(ProjectionAdoptionOutcome::DefinitivePreAppendFailure); + previous_result?; + submitted_result?; + return Err(StoreError::Conflict( + "a transaction may adopt exactly one staged projection".into(), + )); + } + self.adoption = Some(StagedProjectionAdoption { descriptor, handle }); + Ok(self) + } + + pub fn build(mut self) -> Result { + if let Some(adoption) = self.adoption.take() { + // D0-B freezes the lifecycle while B1 still owns the successful + // builder body. `NotImplemented` is definitive and pre-append. + adoption + .handle + .finish(ProjectionAdoptionOutcome::DefinitivePreAppendFailure)?; + } Err(StoreError::NotImplemented( "ValidatedTransactionBuilder::build — B1 NamespaceTxn, scope 6-B1 deliverable 3", )) } } + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use levcs_protocol::v2::{ProjectionMode, StagedProjectionInstallV1}; + + use super::*; + use crate::staging::{ + ProjectionAdoption, ProjectionAdoptionLifecycle, ProjectionAdoptionResolution, + }; + + #[derive(Default)] + struct Lifecycle { + outcomes: Mutex>, + dropped: Mutex, + } + + impl ProjectionAdoptionLifecycle for Lifecycle { + fn resolution(&self) -> Result, StoreError> { + Err(StoreError::NotImplemented( + "not needed by this contract test", + )) + } + + fn finish(&self, outcome: ProjectionAdoptionOutcome) -> Result<(), StoreError> { + self.outcomes.lock().unwrap().push(outcome); + Ok(()) + } + + fn dropped_without_outcome(&self) { + *self.dropped.lock().unwrap() += 1; + } + } + + fn descriptor(session_id: [u8; 16]) -> StagedProjectionInstallV1 { + StagedProjectionInstallV1 { + session_id, + manifest_digest: ObjectId([1; 32]), + projection: ProjectionMode::Full, + object_count: 1, + object_bytes: 1, + membership_root: ObjectId([2; 32]), + artifact_set_digest: ObjectId([3; 32]), + } + } + + #[test] + fn duplicate_projection_adoption_releases_both_pins() { + let first = Arc::new(Lifecycle::default()); + let second = Arc::new(Lifecycle::default()); + let builder = ValidatedTransaction::builder(PrivilegedConstruction::internal()) + .adopt_projection(descriptor([1; 16]), ProjectionAdoption::new(first.clone())) + .expect("first adoption"); + + let result = + builder.adopt_projection(descriptor([2; 16]), ProjectionAdoption::new(second.clone())); + assert!(matches!(result, Err(StoreError::Conflict(_)))); + for lifecycle in [first, second] { + assert_eq!( + &*lifecycle.outcomes.lock().unwrap(), + &[ProjectionAdoptionOutcome::DefinitivePreAppendFailure] + ); + assert_eq!(*lifecycle.dropped.lock().unwrap(), 0); + } + } +} diff --git a/crates/levcs-store/src/types.rs b/crates/levcs-store/src/types.rs index f604bd2..ba584f6 100644 --- a/crates/levcs-store/src/types.rs +++ b/crates/levcs-store/src/types.rs @@ -6,6 +6,7 @@ //! The whole file is the Phase 1 realization of plan §5.1's frozen public API. use std::fmt; +use std::sync::Arc; use levcs_core::ObjectId; @@ -140,7 +141,7 @@ pub enum TransactionStatus { /// Inability to answer. Corruption, unavailable recovery state, refusal to /// open, or a definitive rejection — never a lifecycle state. -#[derive(Debug, thiserror::Error)] +#[derive(Clone, Debug, thiserror::Error)] pub enum StoreError { #[error("not implemented: {0}")] NotImplemented(&'static str), @@ -174,7 +175,7 @@ pub enum StoreError { AlreadyLocked, #[error("io: {0}")] - Io(#[from] std::io::Error), + Io(#[source] Arc), #[error("no space left on the store device")] NoSpace, @@ -186,6 +187,12 @@ pub enum StoreError { allowed: u64, }, + #[error("store overloaded at {limit}; retry after {retry_after_micros} microseconds")] + Overloaded { + limit: &'static str, + retry_after_micros: u64, + }, + #[error("mutable-state conflict: {0}")] Conflict(String), @@ -199,7 +206,7 @@ pub enum StoreError { InvalidConfiguration(String), } -#[derive(Debug, thiserror::Error)] +#[derive(Clone, Debug, thiserror::Error)] pub enum SignerError { #[error("signer unavailable")] Unavailable, @@ -207,6 +214,12 @@ pub enum SignerError { Rejected(String), } +impl From for StoreError { + fn from(error: std::io::Error) -> Self { + Self::Io(Arc::new(error)) + } +} + /// Signs `CommittedTransactionV1` event digests on behalf of the instance. /// /// Registered by instance composition (plan §5.2). The store calls it before @@ -278,3 +291,44 @@ pub struct DurabilityCounterSnapshot { pub short_writes: u64, pub bytes_written: u64, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cloned_io_error_shares_the_original_error_losslessly() { + use std::error::Error; + + let error = StoreError::from(std::io::Error::from_raw_os_error(libc::EIO)); + let clone = error.clone(); + + let (StoreError::Io(original), StoreError::Io(cloned)) = (&error, &clone) else { + panic!("expected two shared I/O errors"); + }; + assert!(Arc::ptr_eq(original, cloned)); + assert_eq!(cloned.raw_os_error(), Some(libc::EIO)); + let source = clone.source().expect("I/O source remains available"); + let shared = source + .downcast_ref::>() + .expect("shared I/O source remains downcastable without reconstruction"); + assert!(Arc::ptr_eq(original, shared)); + } + + #[test] + fn overloaded_error_is_cloneable_and_preserves_retry_guidance() { + let error = StoreError::Overloaded { + limit: "max_status_entries", + retry_after_micros: 1_000, + }; + let StoreError::Overloaded { + limit, + retry_after_micros, + } = error.clone() + else { + panic!("clone changed the error variant"); + }; + assert_eq!(limit, "max_status_entries"); + assert_eq!(retry_after_micros, 1_000); + } +} diff --git a/crates/levcs-store/tests/recovery_checkpoint.rs b/crates/levcs-store/tests/recovery_checkpoint.rs index a27a77e..a0858cb 100644 --- a/crates/levcs-store/tests/recovery_checkpoint.rs +++ b/crates/levcs-store/tests/recovery_checkpoint.rs @@ -58,6 +58,7 @@ fn checkpoint(sequence: u64) -> Checkpoint { repo_sequence: sequence, shard_sequence: sequence, current_authority: oid(0xA1), + refs: Vec::new(), objects_new: 1, retry_until_micros: 1_700_000_900_000_000, first_receipt_visibility_micros: Some(1_700_000_000_100_000), diff --git a/crates/levcs-store/tests/recovery_checkpoint_faults.rs b/crates/levcs-store/tests/recovery_checkpoint_faults.rs index 99a9bd8..f777193 100644 --- a/crates/levcs-store/tests/recovery_checkpoint_faults.rs +++ b/crates/levcs-store/tests/recovery_checkpoint_faults.rs @@ -76,6 +76,7 @@ mod faults { repo_sequence: sequence, shard_sequence: sequence, current_authority: oid(0xB1), + refs: Vec::new(), objects_new: 1, retry_until_micros: 1_700_000_900_000_000, first_receipt_visibility_micros: Some(1_700_000_000_100_000), diff --git a/crates/levcs-store/tests/recovery_manifest.rs b/crates/levcs-store/tests/recovery_manifest.rs index ed588aa..6f79d6d 100644 --- a/crates/levcs-store/tests/recovery_manifest.rs +++ b/crates/levcs-store/tests/recovery_manifest.rs @@ -135,33 +135,17 @@ fn a_corrupt_current_falls_back_to_the_newest_valid_predecessor_manifest() { } #[test] -fn a_corrupt_current_whose_newest_manifest_is_also_corrupt_falls_further_back() { +fn a_corrupt_current_with_a_higher_invalid_manifest_refuses_lossy_fallback() { let shard = two_generations(); shard.corrupt("CURRENT", 12); shard.corrupt("manifests/4.manifest", 30); - let selection = resolve(&shard).expect("generation 3 must be reachable"); - assert_eq!(selection.generation, 3); - match selection.source { - ManifestSource::Fallback { - reason: ManifestFallbackReason::CurrentCorrupt(_), - } => {} - ref other => panic!( - "the reported reason is why CURRENT was abandoned, not why an \ - intermediate generation was skipped; got {other:?}" - ), - } assert_eq!( - selection.rejected.len(), - 1, - "the skipped generation must be reported, not silently passed over" + resolve(&shard), + None, + "a finalized higher manifest exists but its committed closure cannot be \ + proved; generation 3 may be a shorter acknowledged prefix" ); - assert_eq!(selection.rejected[0].0, 4); - match selection.rejected[0].1 { - ManifestFallbackReason::ReferentCorrupt { generation, .. } => assert_eq!(generation, 4), - ref other => panic!("expected a corrupt referent, got {other:?}"), - } - assert_agrees_with_a1(&shard); } #[test] @@ -202,26 +186,18 @@ fn a_current_naming_another_store_root_is_refused_as_a_pointer_failure() { // =========================================================================== #[test] -fn a_valid_current_naming_a_missing_manifest_generation_falls_back() { +fn a_valid_current_naming_a_missing_manifest_generation_refuses_to_guess() { let shard = two_generations(); // The pointer stays intact and keeps naming generation 9, which was never // written. This is a failure the single-`CURRENT` layout could not express. shard.write_current(9, ROOT_UUID); - let selection = resolve(&shard).expect("the newest present generation must be reachable"); - assert_eq!(selection.generation, 4); assert_eq!( - selection.source, - ManifestSource::Fallback { - reason: ManifestFallbackReason::ReferentMissing { generation: 9 } - }, - "an absent referent is not the same fault as a pointer that does not validate" + resolve(&shard), + None, + "without generation 9's manifest bytes recovery cannot prove that \ + generation 4 has the same committed closure" ); - assert_eq!( - selection.rejected, - vec![(9, ManifestFallbackReason::ReferentMissing { generation: 9 })] - ); - assert_agrees_with_a1(&shard); } // =========================================================================== @@ -229,49 +205,45 @@ fn a_valid_current_naming_a_missing_manifest_generation_falls_back() { // =========================================================================== #[test] -fn a_valid_current_naming_a_manifest_that_fails_its_own_checksum_falls_back() { +fn a_valid_current_naming_a_corrupt_manifest_refuses_to_guess() { let shard = two_generations(); // CURRENT still names 4 and still validates. Generation 4's own bytes are // damaged. shard.corrupt("manifests/4.manifest", 40); - let selection = resolve(&shard).expect("generation 3 must be reachable"); - assert_eq!(selection.generation, 3); - match selection.source { - ManifestSource::Fallback { - reason: ManifestFallbackReason::ReferentCorrupt { generation, .. }, - } => assert_eq!(generation, 4), - ref other => panic!( - "a corrupt referent is a third distinct branch, not the corrupt-pointer \ - one; got {other:?}" - ), - } - assert_agrees_with_a1(&shard); + assert_eq!( + resolve(&shard), + None, + "corrupt generation 4 cannot prove generation 3 authorizes the same tail" + ); } #[test] -fn the_three_branches_report_three_different_reasons() { - // The point of scope 4-A2's "may not be collapsed" is only checkable by - // comparing the three outcomes against each other. +fn only_a_pointer_failure_with_a_highest_valid_manifest_permits_fallback() { let corrupt_pointer = { let shard = two_generations(); shard.corrupt("CURRENT", 12); - resolve(&shard).expect("selected").source + resolve(&shard) }; let missing_referent = { let shard = two_generations(); shard.write_current(9, ROOT_UUID); - resolve(&shard).expect("selected").source + resolve(&shard) }; let corrupt_referent = { let shard = two_generations(); shard.corrupt("manifests/4.manifest", 40); - resolve(&shard).expect("selected").source + resolve(&shard) }; - assert_ne!(corrupt_pointer, missing_referent); - assert_ne!(corrupt_pointer, corrupt_referent); - assert_ne!(missing_referent, corrupt_referent); + assert_eq!( + corrupt_pointer + .expect("highest immutable manifest validates") + .generation, + 4 + ); + assert_eq!(missing_referent, None); + assert_eq!(corrupt_referent, None); } // =========================================================================== @@ -279,7 +251,7 @@ fn the_three_branches_report_three_different_reasons() { // =========================================================================== #[test] -fn a_corrupt_segment_referenced_by_the_active_manifest_forces_a_fallback() { +fn a_corrupt_segment_referenced_by_current_refuses_a_shorter_closure() { let shard = two_generations(); // Generation 4's manifest is intact; the segment it names is truncated to // nothing, which is what a failed link or a partially copied restore leaves @@ -288,19 +260,11 @@ fn a_corrupt_segment_referenced_by_the_active_manifest_forces_a_fallback() { // place A2 and A1 legitimately disagree. std::fs::write(shard.paths.segments().join("4-0-9.seg"), b"").expect("truncate segment"); - let selection = resolve(&shard).expect("generation 3 must be reachable"); - assert_eq!(selection.generation, 3); assert_eq!( - selection.source, - ManifestSource::Fallback { - reason: ManifestFallbackReason::ReferentFileInvalid { - generation: 4, - filename: "4-0-9.seg".into(), - cause: ReferencedFileFault::TooShort, - } - }, - "scope 3.8 step 2 falls back when ANY referenced file fails validation, \ - not only when the manifest itself is corrupt" + resolve(&shard), + None, + "generation 3 names a different immutable segment closure; accepting it \ + could discard acknowledged generation-4 transactions" ); let (a1, _) = segment::load_manifest_with_fallback(&shard.paths, &ROOT_UUID) @@ -314,23 +278,11 @@ fn a_corrupt_segment_referenced_by_the_active_manifest_forces_a_fallback() { } #[test] -fn a_missing_segment_referenced_by_the_active_manifest_forces_a_fallback() { +fn a_missing_segment_referenced_by_current_refuses_a_shorter_closure() { let shard = two_generations(); std::fs::remove_file(shard.paths.segments().join("4-0-9.seg")).expect("remove"); - let selection = resolve(&shard).expect("generation 3 must be reachable"); - assert_eq!(selection.generation, 3); - assert_eq!( - selection.source, - ManifestSource::Fallback { - reason: ManifestFallbackReason::ReferentFileInvalid { - generation: 4, - filename: "4-0-9.seg".into(), - cause: ReferencedFileFault::Missing, - } - } - ); - assert_agrees_with_a1(&shard); + assert_eq!(resolve(&shard), None); } #[test] @@ -377,7 +329,7 @@ fn a_manifest_whose_generation_disagrees_with_its_file_name_is_refused() { // =========================================================================== #[test] -fn the_fallback_scan_is_bounded() { +fn an_invalid_highest_finalized_manifest_is_never_skipped() { let shard = ShardDir::new(); shard.write_placeholder_segment("1-0-9.seg", SEGMENT_LEN); shard.write_manifest(&manifest(1, "1-0-9.seg")); @@ -402,15 +354,18 @@ fn the_fallback_scan_is_bounded() { of reach; startup must refuse rather than hunt linearly" ); - let selection = resolve_manifest( - &shard.paths, - &ROOT_UUID, - &PresenceAndLengthValidator::default(), - 64, - ) - .expect("resolution") - .expect("with a larger cap the good generation is found"); - assert_eq!(selection.generation, 1); + assert_eq!( + resolve_manifest( + &shard.paths, + &ROOT_UUID, + &PresenceAndLengthValidator::default(), + 64, + ) + .expect("resolution"), + None, + "a larger scan bound must not license rollback past a finalized higher \ + generation whose closure cannot be proved" + ); } #[test] diff --git a/crates/levcs-store/tests/recovery_receipts.rs b/crates/levcs-store/tests/recovery_receipts.rs index 67a3eb4..3ef5442 100644 --- a/crates/levcs-store/tests/recovery_receipts.rs +++ b/crates/levcs-store/tests/recovery_receipts.rs @@ -28,6 +28,7 @@ fn receipt( repo_sequence: id as u64, shard_sequence: id as u64, current_authority: ObjectId([0xA1; 32]), + refs: Vec::new(), objects_new: 1, retry_until_micros: retry_until, first_receipt_visibility_micros: first_visible, diff --git a/crates/levcs-store/tests/recovery_step8.rs b/crates/levcs-store/tests/recovery_step8.rs new file mode 100644 index 0000000..b2afbdf --- /dev/null +++ b/crates/levcs-store/tests/recovery_step8.rs @@ -0,0 +1,333 @@ +#![cfg(all(feature = "store-internals", feature = "failpoints"))] + +use std::fs::{File, OpenOptions}; +use std::os::unix::fs::FileExt; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use levcs_store::drive::{faults, ShardDrive}; +use levcs_store::format::{Manifest, TailRange, JOURNAL_HEADER_LEN}; +use levcs_store::journal::Journal; +use levcs_store::segment::{self, RootLayout}; +use levcs_store::{DurabilityCounters, NamespaceId}; + +fn append(drive: &mut ShardDrive, repo_sequence: u64) { + let frame = drive + .build_frame( + NamespaceId([0xA5; 32]), + repo_sequence, + vec![repo_sequence as u8; 64], + ) + .expect("build frame"); + drive + .append_group_and_fence(&[frame]) + .expect("append and fence"); +} + +fn only_active_journal(active: &Path) -> PathBuf { + let mut journals: Vec = std::fs::read_dir(active) + .expect("list active") + .map(|entry| entry.expect("entry").path()) + .filter(|path| { + path.extension() + .is_some_and(|extension| extension == "journal") + }) + .collect(); + assert_eq!( + journals.len(), + 1, + "write readiness requires one active journal" + ); + journals.pop().expect("one journal") +} + +#[test] +fn nonempty_recovery_preserves_evidence_seals_the_exact_prefix_and_opens_fresh() { + let _serial = faults::serial(); + let dir = tempfile::tempdir().expect("tempdir"); + let (root_uuid, original_path, original_bytes) = { + let mut drive = ShardDrive::create(dir.path(), 0, 1).expect("create"); + append(&mut drive, 0); + append(&mut drive, 1); + let path = drive.journal_path().to_path_buf(); + ( + drive.root_uuid(), + path.clone(), + std::fs::read(path).expect("read crash image"), + ) + }; + + let recovered = + ShardDrive::reopen_through_recovery(dir.path(), 0).expect("recover nonempty journal"); + assert!(recovered.report.ready); + assert_eq!(recovered.adopted_shard_sequences, vec![0, 1]); + let evidence = recovered + .report + .preserved_journal + .as_ref() + .expect("recovery must retain the original inode"); + assert_eq!( + std::fs::read(evidence).expect("read evidence"), + original_bytes, + "recovery may not truncate or rewrite the crash image" + ); + assert!( + !original_path.exists(), + "the old active name must be retired" + ); + + let paths = RootLayout::new(dir.path()).shard(0); + let fresh = only_active_journal(&paths.active()); + assert_ne!(fresh, original_path); + let counters = Arc::new(DurabilityCounters::default()); + let (journal, scan) = Journal::open(&fresh, &root_uuid, counters).expect("open fresh"); + assert!(scan.frames.is_empty()); + assert_eq!(scan.stop_offset, JOURNAL_HEADER_LEN as u64); + assert_eq!(journal.next_shard_sequence(), 2); + assert!( + std::fs::read_dir(paths.segments()) + .expect("segments") + .all(|entry| !entry + .expect("entry") + .path() + .extension() + .is_some_and(|extension| extension == "prefix")), + "a successful recovery may not leak a construction prefix" + ); + + let manifest_generation = recovered.recovered.manifest_generation; + drop(recovered); + let second = + ShardDrive::reopen_through_recovery(dir.path(), 0).expect("idempotent re-recovery"); + assert_eq!(second.recovered.manifest_generation, manifest_generation); + assert!(second.report.preserved_journal.is_none()); + assert_eq!(only_active_journal(&paths.active()), fresh); +} + +#[test] +fn torn_first_frame_is_quarantined_and_replaced_without_an_empty_segment() { + let _serial = faults::serial(); + let dir = tempfile::tempdir().expect("tempdir"); + let (root_uuid, original_path, original_bytes) = { + let mut drive = ShardDrive::create(dir.path(), 0, 1).expect("create"); + let frame = drive + .build_frame(NamespaceId([0xB6; 32]), 0, vec![0x7E; 256]) + .expect("build"); + let encoded = frame.encode().expect("encode"); + let path = drive.journal_path().to_path_buf(); + let file = OpenOptions::new().write(true).open(&path).expect("open"); + file.write_at(&encoded[..encoded.len() / 2], JOURNAL_HEADER_LEN as u64) + .expect("write torn first frame"); + file.sync_data().expect("fence torn bytes"); + ( + drive.root_uuid(), + path.clone(), + std::fs::read(path).expect("read crash image"), + ) + }; + + let recovered = + ShardDrive::reopen_through_recovery(dir.path(), 0).expect("recover torn first frame"); + assert!(recovered.report.ready); + assert!(recovered.adopted_shard_sequences.is_empty()); + assert!(recovered.report.quarantined_bytes > 0); + let evidence = recovered + .report + .preserved_journal + .as_ref() + .expect("damaged zero-frame journal must be retained"); + assert_eq!(std::fs::read(evidence).expect("evidence"), original_bytes); + assert_ne!( + std::fs::read(&original_path).expect("fresh journal at reused sequence name"), + original_bytes, + "the sequence-zero pathname may be reused, but it must name a fresh inode" + ); + + let paths = RootLayout::new(dir.path()).shard(0); + assert!( + std::fs::read_dir(paths.segments()) + .expect("segments") + .all(|entry| entry + .expect("entry") + .path() + .extension() + .and_then(|extension| extension.to_str()) + != Some("seg")), + "an empty validated prefix is not a segment" + ); + let fresh = only_active_journal(&paths.active()); + let counters = Arc::new(DurabilityCounters::default()); + let (journal, scan) = Journal::open(&fresh, &root_uuid, counters).expect("open fresh"); + assert!(scan.frames.is_empty()); + assert_eq!(journal.next_shard_sequence(), 0); +} + +#[test] +fn healthy_empty_active_journal_is_already_write_ready_and_is_not_replaced() { + let _serial = faults::serial(); + let dir = tempfile::tempdir().expect("tempdir"); + let (root_uuid, original) = { + let drive = ShardDrive::create(dir.path(), 0, 1).expect("create"); + (drive.root_uuid(), drive.journal_path().to_path_buf()) + }; + + let recovered = + ShardDrive::reopen_through_recovery(dir.path(), 0).expect("recover healthy empty"); + assert!(recovered.report.ready); + assert!(recovered.report.preserved_journal.is_none()); + assert_eq!(recovered.recovered.manifest_generation, None); + let paths = RootLayout::new(dir.path()).shard(0); + assert_eq!(only_active_journal(&paths.active()), original); + let (journal, scan) = Journal::open( + &original, + &root_uuid, + Arc::new(DurabilityCounters::default()), + ) + .expect("open"); + assert!(scan.frames.is_empty()); + assert_eq!(journal.next_shard_sequence(), 0); +} + +#[test] +fn rejected_newer_generation_forces_recovery_artifacts_above_every_immutable_name() { + let _serial = faults::serial(); + let dir = tempfile::tempdir().expect("tempdir"); + { + let mut drive = ShardDrive::create(dir.path(), 0, 1).expect("create"); + append(&mut drive, 0); + drive.seal_and_install().expect("generation one"); + append(&mut drive, 1); + + let paths = drive.shard_paths(); + let selected = segment::read_manifest(paths, 1, &drive.root_uuid()).expect("manifest one"); + let rejected = Manifest { + root_uuid: drive.root_uuid(), + generation: 100, + base_generation: 0, + retained_tail_ranges: selected.retained_tail_ranges.clone(), + index_runs: vec![(99, "missing.idx".into())], + checkpoints: Vec::new(), + committed_shard_sequence: selected.committed_shard_sequence, + }; + segment::install_manifest(paths, &rejected, 2, &DurabilityCounters::default()) + .expect("publish rejected newer manifest"); + } + + let recovered = + ShardDrive::reopen_through_recovery(dir.path(), 0).expect("fallback and repair"); + assert_eq!(recovered.recovered.manifest_generation, Some(101)); + assert_eq!(recovered.adopted_shard_sequences, vec![0, 1]); +} + +#[test] +fn journal_creation_reuses_one_target_scoped_temp_after_an_interrupted_attempt() { + let dir = tempfile::tempdir().expect("tempdir"); + let active = dir.path().join("active"); + std::fs::create_dir(&active).expect("active"); + std::fs::write(active.join(".0.journal.tmp"), [0xA5; 31]).expect("partial temp"); + let counters = Arc::new(DurabilityCounters::default()); + let journal = Journal::create( + &active, + [7; 16], + 0, + 0, + [9; 16], + 1 << 20, + 1, + Arc::clone(&counters), + ) + .expect("resume partial creation"); + assert_eq!(journal.next_shard_sequence(), 0); + assert!(!active.join(".0.journal.tmp").exists()); + assert_eq!(only_active_journal(&active), active.join("0.journal")); + + let reopened = Journal::create(&active, [8; 16], 0, 0, [9; 16], 1 << 20, 2, counters) + .expect("resume after final rename"); + assert_eq!(reopened.journal_id(), [7; 16]); + assert_eq!( + std::fs::read_dir(active).expect("active entries").count(), + 1 + ); +} + +#[test] +fn recovered_prefix_resumes_one_deterministic_partial_copy() { + let _serial = faults::serial(); + let dir = tempfile::tempdir().expect("tempdir"); + let (root_uuid, journal_path, paths) = { + let mut drive = ShardDrive::create(dir.path(), 0, 1).expect("create"); + append(&mut drive, 0); + ( + drive.root_uuid(), + drive.journal_path().to_path_buf(), + drive.shard_paths().clone(), + ) + }; + let original = std::fs::read(&journal_path).expect("original"); + let counters = Arc::new(DurabilityCounters::default()); + let (journal, scan) = + Journal::open(&journal_path, &root_uuid, Arc::clone(&counters)).expect("open journal"); + let artifact = paths.segments().join(format!( + ".recovery-{}-1.prefix", + hex::encode(journal.journal_id()) + )); + std::fs::write(&artifact, &original[..137]).expect("partial prefix"); + + let installed = segment::seal_recovered_prefix( + journal.file(), + journal.header(), + &scan, + &paths, + 1, + &counters, + ) + .expect("resume and seal"); + assert!(installed.exists()); + assert!(!artifact.exists()); + assert_eq!( + std::fs::read(journal_path).expect("source after seal"), + original, + "the source journal must remain byte-for-byte unchanged" + ); +} + +#[test] +fn manifest_install_resumes_exact_temp_and_exact_final_but_rejects_difference() { + let dir = tempfile::tempdir().expect("tempdir"); + let layout = RootLayout::new(dir.path()); + let counters = DurabilityCounters::default(); + segment::initialize_root(&layout, 1, [0xC7; 16], 1, &counters).expect("initialize"); + let paths = layout.shard(0); + let manifest = Manifest { + root_uuid: [0xC7; 16], + generation: 1, + base_generation: 0, + retained_tail_ranges: vec![TailRange { + generation: 1, + first_shard_sequence: 0, + last_shard_sequence: 0, + filename: "1-0-0.seg".into(), + }], + index_runs: Vec::new(), + checkpoints: Vec::new(), + committed_shard_sequence: 0, + }; + let encoded = manifest.encode().expect("encode"); + let temp = paths.manifests().join("1.manifest.tmp"); + std::fs::write(&temp, &encoded).expect("exact temp"); + File::open(&temp) + .expect("open temp") + .sync_data() + .expect("fence"); + segment::install_manifest(&paths, &manifest, 2, &counters).expect("resume exact temp"); + + std::fs::write(&temp, &encoded).expect("temp after final"); + segment::install_manifest(&paths, &manifest, 2, &counters) + .expect("resume after final publication"); + assert!(!temp.exists()); + + std::fs::write(&temp, [0xDD; 64]).expect("different temp"); + let error = segment::install_manifest(&paths, &manifest, 2, &counters) + .expect_err("different temp must not be overwritten"); + assert!(error.to_string().contains("different bytes")); +} diff --git a/doc/instance-throughput-rewrite-plan.md b/doc/instance-throughput-rewrite-plan.md index e66fc58..b416a32 100644 --- a/doc/instance-throughput-rewrite-plan.md +++ b/doc/instance-throughput-rewrite-plan.md @@ -890,6 +890,89 @@ This is the second frozen Phase 0 classification found to be physically wrong. B found by asking what state the device is actually in, rather than by checking the model against itself. +##### Contract review 2026-07-27-A + +Wave B's D0-B publication freeze requires nine amendments to Wave A frozen or +signature-frozen surfaces. They land as one reviewed interface change before B1, B3, or B4 +is dispatched: + +- `lib.rs` declares and re-exports the immutable publication roots and runtime-agnostic + completion primitive. +- The workspace and `levcs-store` manifests take `im`; `Cargo.lock` records the resolved + graph. Decision 9.7 applies structural sharing to repositories, terminal + receipts/tombstones, typed refs, and transient operation statuses. Canonically iterated + refs use `OrdMap`; unordered hot lookup maps use the HAMT-backed `HashMap`. +- `recovery.rs` exposes one production recovery path returning `RecoveredShard`, including + the complete layered index, catalog, refs, exact receipts, sequences, report, staging + resolutions, and live retained segment/index/checkpoint/tail ownership. A root-wide + `RecoverySession` holds `LOCK` continuously across all shard recoveries and is retained by + the engine; the one-shot drive wrapper uses that same session. `drive.rs` now retains the + recovered state instead of projecting it down to diagnostics. +- `options.rs` gains the status-root and projection-staging session/principal/global + count/object/byte/file/age/rate/debt ceilings. Startup checks all non-zero and nesting + constraints and, with checked ceiling arithmetic, refuses a configuration in which one + maximal projection cannot finish before the session horizon. +- `types.rs` makes the exact shared completion outcome cloneable. `StoreError::Io` carries + `Arc` and a handwritten `From` preserves existing `?` + call sites without reconstructing the error; `StoreError::Overloaded` is the typed + status-capacity refusal from decision 9.8. +- `transaction.rs` and `staging.rs` freeze the opaque projection-adoption handle. The handle + is the pin, exposes only read-only resolution and committed-root reference proof, and must + end as adopted, definitively failed before append, or transferred to recovery. Recovery's + committed/proved-absent notification is part of the same seam, and cleanup may run only + after recovery resolves the shard. A staging-owned recovery resolver turns every + authoritative committed descriptor into namespace-scoped index layers and live artifact + pins before readiness; notification alone is insufficient. + +The resolver exposed one Wave A index assumption that inline-only tests could not exercise. +`IndexLocation` names the complete certified storage record, not naked object bytes. That +record is a transaction frame for inline objects and a canonical digest-bound stage chunk +for an adopted projection. The packed storage-version-1 fields do not change: the captured +committed root maps `segment_generation` to a retained source kind and selects the +corresponding decoder. Generation collisions across source kinds are corruption. This +amends the `index.rs` contract without changing its bytes. + +Integration found one additional frozen-format defect. A canonical transaction frame +contains the exact applied refs returned in `CommitReceipt`, but Wave A's checkpoint +`ReceiptRecord` omitted them. After the replay horizon moved past the frame, current ref +state could not reconstruct old values, deletions, `force`, or transaction membership, so +the same committed retry could return a different receipt after reopen. + +`checkpoint.rs` therefore adds a bounded applied-ref vector to each retained receipt and +sets authenticated checkpoint capability flag bit 0. Production recovery and the drive +seam populate it only from the canonical committed transaction. A storage-version-1 +checkpoint without the flag remains readable if it has no retained receipts; if it has any, +the generation is rejected as `ReceiptRefsUnavailable` and the existing fallback/offline +rebuild policy applies. Older readers already reject the new non-zero flag. The derived +checkpoint format therefore fails closed in both directions without a global +`STORAGE_VERSION` bump; journal and segment authority bytes are unchanged. + +Production-path integration also found that Wave A's recovery stopped after logical replay: +it did not perform normative step 8, so a damaged active journal could be reported ready +without sealing its validated prefix and installing a fresh active journal. Recovery now +preserves crash evidence, constructs the recovered segment and fresh journal through +deterministic resumable names, validates any pre-existing construction artifact byte for +byte, and publishes readiness only after the repaired physical state is complete. + +The same amendment tightens manifest authority. Checkpoints are derived and may fall back +only among checkpoint rows retained by the selected authoritative manifest; their failure +does not authorize a shorter transaction tail. Missing or corrupt authoritative segment +bytes refuse readiness. If `CURRENT` is corrupt or missing, recovery may choose the highest +valid finalized manifest only when no higher invalid finalized manifest makes the closure +ambiguous. Manifest tuple validation includes segment generation/sequence coverage, index +generation, and checkpoint sequence. Generation allocation scans immutable names and moves +above rejected artifacts within a configured bound, so repair never reuses an ambiguous +name. The drive checkpoint path now seals and publishes its checkpoint reference in one +manifest installation, keeping the test seam subject to the production authority model. + +The new `roots.rs` and `completion.rs` files are not amendments to Wave A, but their +interfaces freeze with this review. The completion state stores exactly +`Result` behind one mutex shared by outcome and the full waiter +set. Publication wakes outside the mutex, a dropped waiter is not a publication failure, +and every waiter receives one owned clone of the same result. The committed root uses pure, +idempotent subtree merge and carries addressable live pins for every retained artifact so a +captured read cannot race reclamation. + ### Phase 1 — storage engine spine Lead first defines sealed transaction/frame/snapshot interfaces and file ownership. That deliverable (D0) landed on 2026-07-24 as `crates/levcs-store`: the frozen public API compiling against `StoreError::NotImplemented`, the file-ownership split, strict configuration validation, the single durability syscall funnel with its counters and fault hooks, the failpoint registry in enforced one-to-one correspondence with `oracle::AppendFailpoint`, and the journal-level drive seam that lets the crash harness run in Wave A. The enforced gate is `scripts/check-phase1.sh`, which runs `check-phase0.sh` first so the Phase 0 freeze stays enforced. That work is scoped in `doc/phase1-storage-spine-scope.md`, which realizes this section as a file-ownership matrix, a frozen `levcs-store` API, a physical format and durability/recovery specification, per-package deliverables and acceptance criteria, the Wave A adversarial review charter, and the capacity analysis for P2 on the frozen reference hardware. This plan remains authoritative; that document is the Phase 1 realization of it and lists the decisions that must be resolved before Wave A starts. diff --git a/doc/phase1-storage-spine-scope.md b/doc/phase1-storage-spine-scope.md index 7406db6..c44507d 100644 --- a/doc/phase1-storage-spine-scope.md +++ b/doc/phase1-storage-spine-scope.md @@ -5,11 +5,13 @@ The rewrite plan remains authoritative. Where this document is more specific, it lead's Phase 1 realization of the plan; where it appears to contradict the plan, the plan wins and this document is defective. -Status: **Wave A frozen 2026-07-26 at `5ee9c6b`; Wave B scoped, D0-B not yet started.** +Status: **Wave A frozen 2026-07-26 at `5ee9c6b`; Wave B scoped, D0-B implemented and +gate-green 2026-07-27.** Sections 9.1-9.6 were resolved on 2026-07-24 and are recorded there with their conditions; 9.7 and 9.8 were ruled on 2026-07-26 and are recorded in §6.9, and contract review -2026-07-26-A resolved the `EvidenceHandoffFailure` classification (§6.3). D0-B is unblocked; -no open decision remains. Contract review 2026-07-24-B +2026-07-26-A resolved the `EvidenceHandoffFailure` classification (§6.3). D0-B is complete, +its frozen-surface implementation amendments are recorded by contract review 2026-07-27-A, +and B1/B3/B4 dispatch is unblocked; no open decision remains. Contract review 2026-07-24-B (sections 9.1 and 9.2) has been applied to `bench/result-schema.json` and `bench/reference-hardware.toml` and recorded in the plan document. `crates/levcs-store` exists with the frozen API, the ownership split, the durability funnel, the failpoint @@ -323,6 +325,7 @@ D0 lands; `Cargo.lock` is a reviewed artifact per §10. |---|---|---| | `arc-swap` | §5.1 mandates `ArcSwap`, explicitly not `RwLock>` | — | | `crossbeam-channel` | §7 "bounded crossbeam-style channels" to shard threads | `std::sync::mpsc` is unbounded/single-consumer | +| `im` | §5.3 requires structurally shared hot publication maps; decision 9.7 applies it to repositories, receipts/tombstones, typed refs, and transient statuses | `Arc` clone-on-write is O(entries); a hand-rolled HAMT puts correctness risk in every captured read root | | `rustix` | `renameat2(RENAME_NOREPLACE)`, `O_TMPFILE`, `fdatasync`, `fallocate`, `pwritev`, `flock` without hand-rolled `libc` unsafe | raw `libc` (more unsafe surface), `nix` (heavier) | | `memmap2` | §5.3 memory-mapped immutable index runs | read-based runs; revisit if mmap SIGBUS-on-truncate handling proves worse | | `hdrhistogram` | §10 HDR-style histograms in the result bundle | — | @@ -669,10 +672,13 @@ Normative rules attached to this sequence: ```text 1 acquire LOCK; validate FORMAT and root_uuid - 2 read CURRENT -> manifests/.manifest; if CURRENT is missing/corrupt or - any referenced file fails validation, fall back to the newest valid manifest - generation whose referenced files all validate - 3 load the newest of >=2 validated checkpoints. Three outcomes, kept distinct: + 2 read CURRENT -> manifests/.manifest. If CURRENT is missing/corrupt, + select the highest valid finalized manifest only when no higher finalized manifest is + invalid. Never move to a shorter committed tail because a newer authoritative segment + is missing/corrupt; that is RecoveryRequired, not fallback. Derived checkpoint failure + is handled by step 3 and does not invalidate the transaction-authority manifest. + 3 from the selected manifest's retained checkpoint rows, load the newest of >=2 + validated checkpoints. Three outcomes, kept distinct: no checkpoint directory entries at all -> a fresh root; replay from the start at least one validates -> load the newest entries exist and all fail validation -> explicit offline rebuild mode @@ -700,6 +706,21 @@ failed" would license a replay from sequence zero on a *corrupt* store, which is failure the rule exists to prevent. Emptiness and total corruption must be distinguishable by the type recovery returns, not by a comment. +Contract review 2026-07-27-A narrows the original step-2 phrase "any referenced file." It +was unsafe as written. If manifest 2 authorizes sealed frames 4–5 and checkpoint 5 is +corrupt, falling back to manifest 1 may authorize only frames 0–3 while the active journal +correctly begins at 6. Treating the shorter state as recovered manufactures acknowledged +loss. Checkpoints are derived: keep manifest 2's transaction tail, load retained checkpoint +3 from manifest 2, then replay sealed frames 4–5 and active frame 6. If every retained +checkpoint fails, enter explicit offline rebuild. + +The same asymmetry is stricter for authority bytes. A missing or corrupt segment cannot be +made safe by selecting an older manifest that omits it; recovery must refuse readiness. A +corrupt or missing `CURRENT` may still select the highest valid immutable manifest in the +directory, including one installed just before a crash prevented the pointer swap, but it +may not skip a higher invalid finalized manifest and guess that the lower committed closure +is complete. + Two rules in there carry most of the risk and must be individually asserted, not asserted in aggregate: @@ -791,16 +812,20 @@ Acceptance: for each of the following, a dedicated test, not a shared one — to frame; complete frame after a torn frame (must be discarded); zeroed tail; stale preallocated content in the tail; `EIO` on tail read; corrupt checkpoint (one generation, then both); corrupt segment referenced by the active manifest; **corrupt `CURRENT` with a -valid predecessor manifest**; **valid `CURRENT` naming a missing manifest generation**; -**valid `CURRENT` naming a manifest that fails its own checksum**; wrong `journal_id`; -wrong `root_uuid`; shard-sequence duplicate; repo-sequence gap; `previous_event_digest` -mismatch. +highest valid immutable manifest and no higher invalid finalized manifest**; **valid +`CURRENT` naming a missing manifest generation**; **valid `CURRENT` naming a manifest that +fails its own checksum**; wrong `journal_id`; wrong `root_uuid`; shard-sequence duplicate; +repo-sequence gap; `previous_event_digest` mismatch. The three manifest cases are distinct branches of recovery step 2 and may not be collapsed. -Corrupt-`CURRENT` exercises "the pointer does not validate"; the two naming cases exercise -"the pointer validates but its referent does not" — a failure the single-`CURRENT` layout -could not express and which the pointer design introduces. All three must fall back to the -newest valid manifest generation rather than to an error. +A corrupt pointer may select the highest valid immutable manifest only when no higher +invalid finalized manifest makes the committed closure ambiguous. A valid pointer naming a +missing or corrupt manifest, or an authoritative manifest naming a missing or corrupt +segment, must refuse readiness rather than silently shorten the acknowledged transaction +tail. Checkpoint corruption is different because checkpoints are derived: recovery keeps +the selected manifest, falls back among that manifest's retained checkpoint rows, and +replays its remaining authoritative segments. These are the safe-fallback constraints of +contract review 2026-07-27-A. ### A3 — StoreHarness @@ -1164,7 +1189,7 @@ frozen surface: `format.rs`, `journal.rs`, `segment.rs`, `index.rs`, `checkpoint golden corpus. **No Wave B package may edit any of them.** A change any package believes it needs is an interface request to the lead, arbitrated and — if granted — recorded as a contract review in `doc/instance-throughput-rewrite-plan.md`. D0-B already exercises this: -seven of its nine items amend a frozen or signature-frozen file, and each is a recorded +nine of its eleven items amend a frozen or signature-frozen file, and each is a recorded amendment rather than an edit. The frozen surface is the **library**. Wave A's harness — `src/bin/store-crash-driver.rs`, @@ -1185,7 +1210,7 @@ What Wave A already delivers, so no package rebuilds it: | Object index | `index::{IndexDelta, IndexRunBuilder, IndexRun}` | Namespace-scoped keys, Bloom-filtered sealed runs. | | Namespace catalog | `index::NamespaceCatalog` | `bind`, `advance`, `set_lifecycle`. | | Durable ref/receipt tables | `checkpoint::{RefRecord, ReceiptRecord, Checkpoint}` | Install and prune have production callers as of the freeze. | -| Recovery | `recovery::*` | The algorithms are complete. The production **entry point** returning a `RecoveredShard` does not exist yet and is D0-B item 5; `ShardDrive::reopen_through_recovery` is a `store-internals` test seam, not it. | +| Recovery | `recovery::*` | Wave A supplies parsing, validation, and logical replay. The production **entry point** returning a `RecoveredShard` is D0-B item 5; the physically complete step-8 repair and manifest-authority corrections are D0-B item 11. `ShardDrive::reopen_through_recovery` is a `store-internals` test seam, not the engine path. | | Retention arithmetic | `oracle::{retained_terminal_status, recovered_receipt_visibility}` | Frozen; compute from these, never restate. | ### 6.1 Ownership matrix @@ -1226,11 +1251,28 @@ work, and every row touching a frozen Wave A file is a **contract amendment reco | 2 | The completion primitive behind `async fn submit` | `completion.rs` *(new)* | no | | 3 | `pub mod roots;` and `pub mod completion;` plus re-exports | `lib.rs` | **yes** | | 4 | `im` dependency (decision 9.7). **No dependency for item 2** — see below | `Cargo.toml`, workspace `Cargo.toml`, `Cargo.lock` | **yes** | -| 5 | `RecoveredShard` and a non-feature-gated production recovery entry point | `recovery.rs` | **yes** | +| 5 | `RecoverySession`, `RecoveredShard`, and a non-feature-gated production recovery entry point | `recovery.rs` | **yes** | | 6 | `ShardDrive::reopen_through_recovery` re-pointed onto item 5 | `drive.rs` | **yes** | | 7 | Staging and status-root limits (§6.5, decision 9.8) | `options.rs` | **yes** | -| 8 | `StoreError::Overloaded { limit, retry_after_micros }` | `types.rs` | **yes** | -| 9 | The adoption seam: `ProjectionAdoption` handle (resolve, pin, three-way outcome, reference proof) and `ValidatedTransactionBuilder::adopt_projection(StagedProjectionInstallV1, ProjectionAdoption)` | `transaction.rs`, `staging.rs` | signature frozen | +| 8 | Cloneable exact completion outcomes: `StoreError: Clone`, `Io(Arc)` with lossless `From`, and `Overloaded { limit, retry_after_micros }` | `types.rs` (plus mechanical construction call sites) | **yes** | +| 9 | The adoption seam: `ProjectionAdoption` handle (resolve, pin, three-way outcome, reference proof), `ValidatedTransactionBuilder::adopt_projection(StagedProjectionInstallV1, ProjectionAdoption)`, and recovery resolution of committed staged artifacts into the same index/generation ownership model | `transaction.rs`, `staging.rs`, `roots.rs`, `recovery.rs`, `index.rs` | **yes** | +| 10 | Retain exact `CommitReceipt.refs` in checkpoint receipt rows, populated from the canonical frame by production recovery and the drive seam | `checkpoint.rs`, `recovery.rs`, `drive.rs` | **yes** | +| 11 | Complete normative recovery step 8 and manifest authority: deterministically preserve and seal the validated active prefix, install a fresh active journal before readiness, validate authoritative manifest tuples, select only its retained checkpoints, allocate generations above rejected immutable artifacts, and publish drive checkpoints through the manifest | `journal.rs`, `segment.rs`, `recovery.rs`, `drive.rs`, recovery tests | **yes** | + +**Item 10 is a D0-B integration finding, not B1 work.** A replayed canonical frame carries +the exact applied refs in `committed.transaction.refs`, but Wave A's `ReceiptRecord` omitted +them. Once a checkpoint moves the replay horizon past that frame, the current ref table +cannot recover old values, deleted refs, `force`, or which refs belonged to that +transaction. Returning an empty projection would make a committed retry differ across +reopen. + +New checkpoints therefore set an authenticated checkpoint capability flag and encode the +complete bounded applied-ref vector in every retained receipt row. A storage-version-1 +checkpoint without that flag remains readable when it has no retained receipts. If it has +any, it is rejected as `ReceiptRefsUnavailable` and follows the existing explicit offline +rebuild path; inventing empty receipt refs is forbidden. Readers predating the amendment +already reject the non-zero flag, so compatibility fails closed in both directions without +a global storage-version bump. Contract review 2026-07-27-A records the amendment. **Item 5 is a blocker discovered in review and is the reason this section was rewritten.** The first draft required `StoreEngine::open` to recover through @@ -1247,6 +1289,13 @@ the engine recover identically by construction rather than by review. If the two again, it must be because somebody changed the shared function, not because a caller quietly grew its own. +`LOCK` is root-wide, so the production entry point is owned by a `RecoverySession` that +holds it continuously while every shard is recovered and then moves into `StoreEngine` for +the engine's lifetime. A free one-shard wrapper exists for the drive seam and is implemented +by creating that same session. Acquiring and dropping one lock per shard would leave a +second process able to enter between shards or immediately after recovery but before +readiness, invalidating the recovered root before its first read. + **`RecoveredShard` must carry everything `CommittedRoot` needs, which is more than the replayed tail.** A first draft listed "an index delta" and that is a defect: an object whose only index entry lives in a sealed `IndexRun` or a checkpointed generation would be present @@ -1256,7 +1305,10 @@ pinned against reclamation. The contents are therefore: - the namespace catalog, ref state, and receipt table; - the **complete layered index** — the replayed delta *over* the ordered set of sealed `IndexRun` references the selected manifest and checkpoint generation retain, in the - lookup order `CommittedRoot` will use, not the delta alone; + lookup order `CommittedRoot` will use, not the delta alone. For a committed + `StagedProjectionInstallV1`, the staging-owned recovery resolver must supply its exact + namespace-scoped membership and live artifact/index pins before readiness; a notification + without incorporating those objects is not complete recovery; - **every retained generation reference** — segments, index runs, and checkpoint generations — so constructing the root transfers ownership of the things that keep those files alive rather than merely naming them; @@ -2056,7 +2108,7 @@ D0 lead skeleton, frozen API, sys/failpoint shims, deps, decisions 9.1-9.5 Wave A frozen 2026-07-26 at 5ee9c6b | D0-B lead: roots.rs, completion.rs, RecoveredShard entry point, adoption handle, - lib/deps/options/error amendments (9 items, 7 touching frozen files) + lib/deps/options/error/recovery amendments (11 items, 9 touching frozen files) | +-- B1 NamespaceTxn ---+ +-- B3 StagingSessions ---+--> Wave B freeze gate + adversarial review @@ -2083,7 +2135,8 @@ matrix that can express the class of defect Wave A shipped. Sections 9.1-9.6 are resolved, so Wave A was unblocked once D0 landed; 9.7 and 9.8 were ruled on 2026-07-26 in §6.9, and contract review 2026-07-26-A closed the `EvidenceHandoffFailure` -conflict, so D0-B is unblocked. The item that was sequenced ahead of A3 — +conflict. D0-B is implemented and gate-green, so B1, B3, and B4 dispatch is unblocked. The +item that was sequenced ahead of A3 — **contract review 2026-07-24-B**, the `result-schema.json` per-flag conditional with its re-pin requirement and the top-level `promotable`, the added `workload.generator` field, and `reference-hardware.toml`'s `store_directory_attributes` —