//! `StoreEngine`: shard routing, startup and recovery, lifecycle, and the //! `ArcSwap` committed/status roots. //! //! **Shared file.** Lead owns the signatures (D0); **B1 NamespaceTxn** fills //! the bodies (scope 2.1, 6-B1). Same pattern as `format.rs` and `drive.rs`: //! the public API must compile in D0 so A3 can write against real signatures //! in Wave A rather than retrofitting them in Wave B. //! //! # What is implemented here and what is not //! //! This is the first vertical slice: one retained [`RecoverySession`] across //! every shard, one shard writer per shard wired //! `GroupBuilder` -> `append_group_and_fence` -> [`ShardSubtree`] -> committed //! root CAS -> completion, and the publication ordering of scope 6.3. Startup //! states 1 and 2 are implemented — an absent or empty root is initialized //! under the root lock, and a valid `FORMAT` opens through production //! recovery. States 3 and 4, the signer pool and its suffix repair, //! coalescing, terminal //! retention, rotation/sealing, and `RepoSnapshot` are later B1 assignments //! and refuse with a [`StoreError::NotImplemented`] naming themselves. None of //! them returns a partial result or a plausible default; a caller that reaches //! one gets told which deliverable it is waiting on. //! //! # The ordering that matters most in this file //! //! Scope 6.3 steps 7, 8, and 9, in that order and never another. The committed //! root is published first, the status entries are removed second, and waiters //! are woken third. Removing a status entry before the receipt is visible //! manufactures `Unknown` for a transaction that is already durable — the //! exact race the mandatory committed-root B read exists to close, reintroduced //! from the write side where no read can fix it. Waking before publication //! would let a caller act on a receipt no other reader can see yet. use std::collections::BTreeMap; use std::sync::atomic::{fence, Ordering}; use std::sync::Arc; use std::thread::JoinHandle; use std::time::{SystemTime, UNIX_EPOCH}; use arc_swap::ArcSwap; use crossbeam_channel::{Receiver, RecvTimeoutError, Sender, TrySendError}; use im::Vector; use levcs_core::{ObjectId, ObjectType}; use levcs_protocol::oracle::{self, AppendDeadlinePhase, DeadlineDecision}; use levcs_protocol::v2::{ self as protocol, AppliedRefV1, CommittedTransactionV1, DurabilityResultV1, RefMutation, RefStateV1, SignedCommittedTransactionV1, }; use crate::checkpoint::{ReceiptRecord, RefRecord}; use crate::completion::SharedCompletion; use crate::failpoints::{self, Failpoint, FailpointAction}; use crate::format::{ frame_total_len, object_type_code, Frame, FrameHeader, FrameObjectV1, FrameObjectsV1, FrameReceiptFieldsV1, RepositoryCreateV1, TransactionFramePayloadV1, }; use crate::index::{ delta_pressure, DeltaPressure, IndexDelta, IndexKey, IndexLocation, IndexRun, IndexRunBuilder, NamespaceLifecycle, NamespaceRecord, NamespaceStorageMode, }; use crate::journal::{GroupBuilder, Journal}; use crate::options::StoreOptions; use crate::recovery::{RecoveredShard, RecoveryConfig, RecoverySession}; use crate::roots::{ CommittedRoot, GenerationId, GenerationMap, LayeredObjectIndex, OperationKey, OperationStatusMetricSnapshot, OperationStatusMetrics, OperationStatusRoot, RepoMap, RepoState, RetainedGeneration, RetainedIndexRun, RetainedReceipt, ShardSequenceMap, ShardSubtree, StatusEntry, StatusReservation, TerminalStatusEntry, TerminalStatusMap, TypedRefMap, }; use crate::segment::{self, RootLayout}; use crate::snapshot::RepoSnapshot; use crate::staging::ProjectionStaging; use crate::transaction::ValidatedTransaction; use crate::types::{ CommitEvidenceSigner, CommitReceipt, DurabilityCounterSnapshot, DurabilityCounters, NamespaceId, OperationId, PendingPhase, StoreError, TransactionStatus, }; /// The durable transaction service. /// /// Owns one `ArcSwap` and one bounded /// `ArcSwap`; neither uses `RwLock>` (plan §5.1). /// The committed-root swap is the visibility boundary. pub struct StoreEngine { shared: Arc, shards: Vec, } /// A held checkpoint export lease. Pins the referenced generation's segments, /// indexes, and checkpoints against reclamation until dropped. pub struct CheckpointLease { _private: (), } impl std::fmt::Debug for StoreEngine { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter .debug_struct("StoreEngine") .field("root", &self.shared.options.root) .field("shard_count", &self.shared.shard_count) .finish_non_exhaustive() } } /// Everything a shard writer and a reader share. /// /// The [`RecoverySession`] is in here rather than in `StoreEngine` for one /// reason: it holds the root-wide `LOCK`, and it must stay held for exactly as /// long as anything in this process can write to the root. Tying its lifetime /// to the shared state rather than to the handle makes "the lock outlives every /// writer" a fact about ownership instead of a rule about drop order. struct EngineShared { options: StoreOptions, signer: Arc, shard_count: u16, root_uuid: [u8; 16], committed: ArcSwap, status: ArcSwap, status_metrics: OperationStatusMetrics, /// The one `ProjectionStaging` for this root. /// /// Constructed exactly once, by [`StoreEngine::open`], under the already /// held [`RecoverySession`], and kept alive for exactly as long as that /// session holds `LOCK`. Staging's ceilings are root-global, and a second /// handle built outside the lock would carry its own independent accounting /// — two handles, or two processes, each admitting up to the full global /// limit. Ownership here is what makes "root-global" true rather than /// asserted. _staging: Arc, _session: RecoverySession, /// What startup state 1 actually cost, or `None` when this open did not /// initialize. /// /// Charter item 7: "state 1 created the tree and fsynced it" is otherwise /// a claim about [`initialize_root_state_1`] rather than an observation. A /// test that opens an absent root can read the directory fsyncs the /// initialization performed instead of inferring them from the tree it /// left behind — and, just as importantly, an open of state 2 can assert /// this is `None`, which is the only direct evidence that reopening a /// formatted root writes nothing. /// /// `#[cfg(test)]` because exposing it is a frozen-surface change; see the /// interface request in this deliverable's report. #[cfg(test)] initialization: Option, /// What recovery concluded about each shard during this open, in shard /// order. /// /// The engine consumes these reports and keeps only their effects, which /// is right for production and leaves nothing to compare against the drive /// seam — `DriveRecovery` retains its report precisely so the two can be /// compared, and the engine side had no counterpart. `#[cfg(test)]` for /// the same reason as [`EngineShared::initialization`]. #[cfg(test)] recovery_reports: Vec, /// Observed committed-root CAS contention. /// /// Charter item 7: the re-merge path is otherwise only reachable as a race, /// so a test that claims to have exercised it has to be able to count it. #[cfg(test)] root_cas_retries: std::sync::atomic::AtomicU64, } struct ShardHandle { /// `Option` so `Drop` can close the channel before joining. A shard thread /// exits on disconnect, so dropping the sender is the shutdown signal. submissions: Option>, thread: Option>, /// **Not reachable from outside the crate yet.** The counters are the only /// witness for "one fence per group", and B4's harness will need them /// through the public API; adding a public accessor is a frozen-surface /// change and therefore an interface request, not an edit. In-crate tests /// read them today. #[cfg_attr(not(test), allow(dead_code))] counters: Arc, } struct Submission { transaction: ValidatedTransaction, completion: SharedCompletion, } impl StoreEngine { /// Open or initialize a store root. /// /// Handles plan §5.2's four startup states explicitly and in order, with /// no inference: absent-or-empty initializes; a valid `FORMAT` opens /// through production recovery; a recognized non-empty legacy layout /// without `FORMAT` is refused with the exact `migrate-store` command; and /// every other non-empty unrecognized layout is refused without /// modification. It never infers legacy state merely from a missing /// marker. /// /// **This slice implements states 1 and 2**, including state 1's /// crash-resume shape: a root whose initialization was interrupted is /// recognized by the marker this path installs before it builds anything /// and is finished rather than refused /// ([`RootStartupState::InterruptedInitialization`]). States 3 and 4 are /// still refused by name below, after a read-only classification and before /// anything is written, so a caller cannot mistake "not built yet" for /// "your root is broken". See [`classify_root`] for why the classification /// itself creates nothing. /// /// Recovery runs through **one** [`RecoverySession`] for every shard, and /// that session then moves into the engine. `LOCK` is root-wide: taking /// and dropping one lock per shard would leave a window between two shard /// recoveries, and another between the last recovery and readiness, in /// which a second process could enter and invalidate the root this one is /// about to publish. pub fn open(options: StoreOptions) -> Result { // Configuration is validated first: an invalid configuration must fail // startup regardless of what is on disk, and validating after touching // the root would make the refusal depend on the root's state. options.validate()?; // Hoisted above every root access on purpose. This used to run after // the `FORMAT` probe, which was harmless while state 1 refused — but // now that an absent root is *initialized*, a signerless configuration // reaching this point would have already created a tree, written // `FORMAT`, and fsynced the parent before failing. A configuration // error must not be able to leave a root behind. let signer = options.signer.clone().ok_or_else(|| { StoreError::InvalidConfiguration( "a CommitEvidenceSigner must be registered by instance composition before the \ engine can sequence a transaction" .into(), ) })?; let layout = RootLayout::new(&options.root); #[cfg_attr(not(test), allow(unused_variables))] let initialization = match classify_root(&layout)? { // Both reach the same code because they are the same state: a root // that has never finished initializing. The second one only carries // residue this code path wrote, and `initialize_root_state_1` // re-decides which it is under the lock before it writes anything. RootStartupState::AbsentOrEmpty | RootStartupState::InterruptedInitialization => { Some(initialize_root_state_1(&layout, &options)?) } RootStartupState::Formatted => None, // States 3 and 4 share one refusal because telling them apart is // the legacy recognizer, and returning `UnrecognizedLayout` for a // root that is in fact a legacy instance would tell an operator // holding real data "refusing to modify" instead of the migration // command. Nothing here has written, probed, or inferred. RootStartupState::NonEmptyWithoutFormat => return Err(StoreError::NotImplemented( "StoreEngine::open startup states 3 and 4 (LegacyLayout carrying the exact \ migrate-store command, UnrecognizedLayout) — B1 NamespaceTxn, scope 6-B1 \ deliverable 1. The root is non-empty and carries no FORMAT; it has not been \ modified", )), }; let session = RecoverySession::open(&options.root)?; if session.shard_count() != options.shard_count { // Scope 2.5: the topology is frozen into FORMAT for the life of // the root. A silent reroute would move repositories between // shards, and per-repository sequence ownership is only sound // while a repository's shard assignment never moves. return Err(StoreError::FormatMismatch(format!( "root was initialized with {} shards but this process is configured for {}", session.shard_count(), options.shard_count ))); } // The single `ProjectionStaging` for this root, built here and nowhere // else. It is constructed *after* the session has taken `LOCK` and // *before* any shard is recovered, because it is recovery's projection // resolver: a committed staged-install frame cannot become ready // without it, and a resolver constructed after recovery would be a // resolver recovery never had. It then lives in `EngineShared` for // exactly as long as the session does. let staging_durability = Arc::new(DurabilityCounters::default()); let staging = ProjectionStaging::open(&session, options.clone(), Arc::clone(&staging_durability))?; let config = RecoveryConfig::from_store_options(&options) .with_projection_recovery_resolver(staging.as_ref()); let mut recovered = Vec::with_capacity(session.shard_count() as usize); for shard in 0..session.shard_count() { recovered.push(session.recover_shard(shard, &config)?); } drop(config); // Scope 3.8 step 12. Readiness is a computed field of each shard's // report; a shard that is not ready is not a shard this engine may // publish a root from. for shard in &recovered { if !shard.report.ready { return Err(StoreError::RecoveryRequired); } } let root = initial_committed_root(&recovered)?; let shared = Arc::new(EngineShared { shard_count: session.shard_count(), root_uuid: session.root_uuid(), signer, committed: ArcSwap::from(Arc::new(root)), status: ArcSwap::from(Arc::new(OperationStatusRoot::new( options.max_status_entries, ))), status_metrics: OperationStatusMetrics::default(), options, _staging: staging, _session: session, #[cfg(test)] initialization, #[cfg(test)] recovery_reports: recovered.iter().map(|shard| shard.report.clone()).collect(), #[cfg(test)] root_cas_retries: std::sync::atomic::AtomicU64::new(0), }); let mut shards = Vec::with_capacity(recovered.len()); for shard in recovered { shards.push(spawn_shard_writer(Arc::clone(&shared), shard)?); } Ok(Self { shared, shards }) } /// Acquire exactly one committed root and derive a repository view from /// it. Readers never observe an index entry newer than their captured /// root (plan §5.3). pub fn snapshot(&self, _repo: NamespaceId) -> Result { Err(StoreError::NotImplemented( "StoreEngine::snapshot — B1 NamespaceTxn, scope 6-B1 deliverable 8", )) } /// Submit a validated transaction for sequencing, group append, fence, and /// publication. /// /// Returns only after the durable sequence is fenced and the committed /// root published. A success response means every reachable object and ref /// in the receipt survives immediate power loss (plan §4 transaction /// invariant 3). pub async fn submit(&self, txn: ValidatedTransaction) -> Result { let shard = StoreOptions::shard_of(&txn.namespace, self.shared.shard_count); let handle = self .shards .get(shard as usize) .ok_or_else(|| StoreError::Corruption(format!("shard {shard} has no writer")))?; let sender = handle.submissions.as_ref().ok_or(StoreError::NotReady)?; let completion = SharedCompletion::new(); let waiter = completion.subscribe(); let submission = Submission { transaction: txn, completion, }; // The queue bound is backpressure, not a wait. Blocking the caller's // task inside a crate that starts no runtime would hand the store a // scheduling decision that belongs to its embedder (plan §7). match sender.try_send(submission) { Ok(()) => {} Err(TrySendError::Full(_)) => { return Err(StoreError::Overloaded { limit: "shard_submission_queue", retry_after_micros: self .shared .options .max_group_idle .as_micros() .try_into() .unwrap_or(u64::MAX), }) } // The writer thread is gone, which in this slice means the process // is shutting down or the thread panicked inside the poison // window. Either way nothing further can be appended. Err(TrySendError::Disconnected(_)) => return Err(StoreError::NotReady), } let receipt = waiter.await?; // Scope 6.3 step 10. A failure here is not a failure of the // transaction: the fence succeeded and the root published, so the // receipt stays queryable through `transaction_status` whatever this // returns. The failpoint is evaluated only on the success path, // because it names the moment a receipt is handed back and a request // that already failed has no receipt to lose. match failpoints::hit(Failpoint::BeforeResponse) { FailpointAction::Continue => Ok(receipt), FailpointAction::Fail => Err(StoreError::Io(Arc::new(std::io::Error::other( "failpoint BeforeResponse: the response was lost after publication; the \ transaction is committed and its receipt remains queryable", )))), FailpointAction::Panic => { panic!("failpoint BeforeResponse panicked the responding task") } FailpointAction::HardExit => { unreachable!("HardExit calls _exit(3) inside failpoints::hit and never returns") } } } /// Linearizable status lookup across the two roots. /// /// Plan §5.1: load committed root A and return any receipt or expired /// tombstone; otherwise load the status root and return any pending or /// resolving entry; if no status entry exists, load committed root B, /// return any terminal entry found there, and otherwise `Unknown`. The /// mandatory B read closes the old-committed/new-empty-status race without /// requiring A and B to share a generation. /// /// The three loads are `Acquire` (`ArcSwap::load`), and the writer publishes /// the committed root before it removes a status entry. So a reader that /// observes the status root *without* an entry has synchronized with the /// store that removed it, and therefore cannot then read a committed root /// older than the publication that preceded that removal. That is what /// makes the B read a proof rather than a retry. /// /// Time-based promotion of a retained receipt to `Expired` and then to /// `Unknown` (`oracle::retained_terminal_status`) is B1 deliverable 7; this /// read reports the terminal row the root actually holds. pub fn transaction_status( &self, repo: NamespaceId, operation: OperationId, ) -> Result { let key = OperationKey::new(repo, operation); Ok(two_root_status_read( self.shared.committed.load().terminal_status(&key).cloned(), self.shared.status.load().get(&key).copied(), // Evaluated eagerly and deliberately: the B load must happen after // the status load in program order, and a lazily evaluated // argument would let a future refactor skip it. self.shared.committed.load().terminal_status(&key).cloned(), )) } /// Force a checkpoint and hold a lease on the resulting generation. pub fn checkpoint(&self) -> Result { Err(StoreError::NotImplemented( "StoreEngine::checkpoint — B1 NamespaceTxn, scope 6-B1", )) } /// Durability counters for one shard, so "one fence per group" and "no /// per-object fsync" are assertions against observed syscalls rather than /// claims about this file (scope 2.3, charter item 7). /// /// Returns a **snapshot**, never the live `Arc`. The /// counter object is a set of mutable atomics: handing it out would let a /// consumer — including the harness whose whole job is to measure this /// store — add to the very numbers it reports, and a durability claim that /// its own auditor can write to is not evidence. `None` names a shard index /// this engine has no writer for, which is a caller error rather than a /// zeroed reading. pub fn durability_counters(&self, shard: u16) -> Option { self.shards .get(shard as usize) .map(|handle| handle.counters.snapshot()) } /// Status-root occupancy and refusal counts, root-wide. /// /// Read-only for the same reason as [`StoreEngine::durability_counters`]. /// Occupancy is the size of the status root as of the last successful CAS /// by any writer, and `rejections` counts operations refused by the /// decision-9.8 capacity bound. pub fn operation_status_metrics(&self) -> OperationStatusMetricSnapshot { self.shared.status_metrics.snapshot() } /// What index maintenance has done and what it still owes, root-wide. /// /// Both fields come from **one** captured `CommittedRoot`. Reading them from /// two loads would let a caller see a run count from one publication beside a /// backlog from another — and the pair is only meaningful together, because /// the whole claim of a seal is that runs went up as the backlog went down. /// Two loads could show both rising, which never happens in any single root. /// /// A snapshot of immutable counts, not a counter handle, for the reason /// [`StoreEngine::durability_counters`] gives: the harness that reports these /// numbers must not be able to write to them. pub fn index_maintenance(&self) -> IndexMaintenanceSnapshot { let root = self.shared.committed.load(); IndexMaintenanceSnapshot { sealed_runs: root.index().sealed_run_count() as u64, unsealed_delta_layers: root.index().delta_layer_count() as u64, } } } /// Sealed runs and unsealed backlog, as of one committed root. /// /// `unsealed_delta_layers` is the count of delta layers no sealed run covers /// yet — the backlog a seal removes. Zero after a seal that covered everything /// published; nonzero whenever groups have committed since. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub struct IndexMaintenanceSnapshot { pub sealed_runs: u64, pub unsealed_delta_layers: u64, } /// Plan §5.1's two-root read, written once so both the engine and its test can /// only ever agree with `oracle::two_root_status_read`. fn two_root_status_read( committed_a: Option, status: Option, committed_b: Option, ) -> TransactionStatus { if let Some(entry) = committed_a { return entry.transaction_status(); } if let Some(entry) = status { return entry.transaction_status(); } match committed_b { Some(entry) => entry.transaction_status(), None => TransactionStatus::Unknown, } } impl Drop for StoreEngine { fn drop(&mut self) { // Close every channel first, then join. A shard thread publishes its // open group on disconnect, so a transaction already accepted by a // writer is fenced and published before this returns rather than // abandoned with its waiter attached. for shard in &mut self.shards { shard.submissions = None; } for shard in &mut self.shards { if let Some(thread) = shard.thread.take() { let _ = thread.join(); } } } } // =========================================================================== // Startup: classifying the root (plan §5.2, scope 3.1) // =========================================================================== /// Which of plan §5.2's startup states a root is in, decided by reading and /// nothing else. /// /// States 3 and 4 share a variant because this pass does not implement the /// legacy recognizer that separates them. That is a deliberate under- /// classification, not a catch-all: the variant is named for exactly the /// condition it holds — non-empty, no `FORMAT` — and `open` refuses it by /// naming both unbuilt states. Charter item 6 forbids a `_ =>` arm; it does /// not require inventing a distinction whose recognizer has not been written. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum RootStartupState { /// §5.2 state 1. The path does not exist, or it is a directory holding /// nothing that this store did not create before it had decided anything. AbsentOrEmpty, /// §5.2 state 2. A `FORMAT` entry exists. Whether it *validates* is /// deliberately not decided here; see [`classify_root`]. Formatted, /// §5.2 state 1, interrupted. The root carries this store's own /// initialization marker and nothing outside the skeleton that /// [`segment::initialize_root`] builds, so it is a root that started /// initializing and never finished. /// /// Not a fifth plan state: it is state 1 observed part-way through, and it /// resolves to the same action state 1 takes. It exists as its own variant /// because the *evidence* is different — an empty root proves nothing has /// been written, this one proves that what was written was written by this /// code path — and because charter item 6 requires the condition that /// authorizes a write to be named rather than folded into a neighbour. InterruptedInitialization, /// §5.2 states 3 and 4, undivided this pass. NonEmptyWithoutFormat, } // --------------------------------------------------------------------------- // The initialization marker (plan §5.2 state 1, crash-resume) // --------------------------------------------------------------------------- /// The name of the marker that says "this root is mid-initialization". /// /// Computed from `layout.root` here rather than added to [`RootLayout`], /// because `RootLayout` is A1's frozen surface. Giving the name one definition /// there is filed as an interface request; until it moves, this is the only /// place in the crate that spells it. const INITIALIZING_NAME: &str = "INITIALIZING"; /// The staging name the marker is installed from, so the marker itself is /// never observed torn. const INITIALIZING_TMP_NAME: &str = "INITIALIZING.tmp"; /// First bytes of the marker. The whole point of the classification is that /// this cannot be produced by accident: a directory that merely *looks* like an /// abandoned root has no reason to contain a file whose first sixteen bytes are /// this string. const INITIALIZING_MAGIC: &[u8; 16] = b"levcs-store-init"; /// 16 magic + 1 version + 2 shard_count + 8 created_at_micros. const INITIALIZING_MARKER_LEN: usize = 27; /// The exact bytes of the marker for one initialization attempt. /// /// `shard_count` and `created_at_micros` are **evidence, not input**. A resume /// restarts from the configuration the resuming process was given, not from the /// one recorded here — the interrupted attempt published nothing, so it has no /// claim on the topology. They are recorded so that an operator looking at /// residue can see what the interrupted attempt intended, which is the one /// question the file's existence alone cannot answer. fn initializing_marker_bytes(shard_count: u16, created_at_micros: i64) -> [u8; 27] { let mut bytes = [0u8; INITIALIZING_MARKER_LEN]; bytes[..16].copy_from_slice(INITIALIZING_MAGIC); bytes[16] = 1; bytes[17..19].copy_from_slice(&shard_count.to_le_bytes()); bytes[19..27].copy_from_slice(&created_at_micros.to_le_bytes()); bytes } /// Open `path` if and only if it is a **regular file holding exactly this code /// path's marker**, and return the open descriptor so a caller that then wants /// to act on those bytes acts on the object it validated rather than on the /// name a second time. /// /// Four conditions, all necessary, none inferred from the name: /// /// 1. **Regular file, not a symlink** — [`crate::sys::open_regular_nofollow`] /// opens with `O_NOFOLLOW` and checks the type through `fstat` on the /// descriptor it got. A symlink here is not "our marker whose bytes live /// elsewhere", it is somebody else's name, and following it would let a link /// planted in the root decide what this process reads and (see /// [`install_initializing_marker`]) writes. /// 2. **Exactly [`INITIALIZING_MARKER_LEN`] bytes** — one byte more is read /// than the marker occupies, so a longer file is rejected by the same /// comparison that rejects a shorter one. /// 3. **[`INITIALIZING_MAGIC`]**, which nothing else in this crate and nothing /// in a legacy layout writes. /// 4. **A version this build knows.** /// /// `Ok(None)` for every failure, including absence: the caller has already seen /// the name in a `read_dir` that may be a moment stale, and a marker that /// vanished between the scan and this read is a root with no marker. Every one /// of these answers means the same thing to every caller — *these are not our /// bytes* — so they are one variant rather than a distinction no caller acts /// on. fn open_exact_marker(path: &std::path::Path) -> Result, StoreError> { let file = match crate::sys::open_regular_nofollow(path)? { Some(file) => file, None => return Ok(None), }; // One byte more than the marker, so a longer file is rejected by the same // length comparison that rejects a shorter one. let mut bytes = [0u8; INITIALIZING_MARKER_LEN + 1]; let read = crate::sys::pread(&file, 0, &mut bytes)?; if read == INITIALIZING_MARKER_LEN && &bytes[..16] == INITIALIZING_MAGIC && bytes[16] == 1 { Ok(Some(file)) } else { Ok(None) } } /// Whether `/INITIALIZING` is this code path's marker. /// /// Judged on content and file type, never on the name alone. A file of any /// other length, with any other first sixteen bytes, or that is not a regular /// file at all, is somebody else's and is treated as unrecognized content — the /// direction that wastes an operator's time rather than the one that destroys /// their data. fn initializing_marker_is_ours(layout: &RootLayout) -> Result { Ok(open_exact_marker(&layout.root.join(INITIALIZING_NAME))?.is_some()) } /// Decide the startup state of `layout.root` **without creating, modifying, or /// removing anything**. /// /// One `read_dir` and a name comparison. No `create_dir_all`, no `File::open` /// with `create`, no lock — a probe that creates a directory has already /// broken state 4's byte-identity guarantee before the refusal it exists to /// support is even reached, and there would be no test that could tell. /// /// # The names this function has to rule on /// /// - **`FORMAT` present.** State 2, on the strength of the *name* alone. A /// `FORMAT` that fails to decode is still `FORMAT`: it is not evidence of an /// empty root, and treating it as one would let a corrupt marker authorize /// `initialize_root` to build a fresh tree over a populated one. Validation /// is [`segment::read_format`]'s job inside [`RecoverySession::open`], where /// failing is a refusal and never a rebuild. /// - **`LOCK`, present as a regular file.** Ignored. It is not store data and /// it is not evidence that anything has been decided: `LOCK` is a zero-length /// file that [`segment::lock_root`] creates if absent, as a side /// effect of *asking* whether the root is busy, so any process that merely /// probed the root leaves one. Treating it as state 4 would permanently wedge /// a root that has no bytes to lose. /// - **`INITIALIZING.tmp`, present as a regular file holding exactly the /// marker.** Ignored, on the same reasoning and on nothing weaker. It is the /// staging name the marker below is installed from, and it normally exists /// only between a fenced write and the `rename_noreplace` that consumes it. /// - **`INITIALIZING` present as a regular file, and every other entry inside /// the initialization skeleton.** /// [`RootStartupState::InterruptedInitialization`] — see below. /// - **Anything else.** State 3 or 4, refused without modification. Including /// `FORMAT.tmp` on its own: it is residue this code path writes, but by /// itself it is residue from *after* the marker was installed and then /// removed, which cannot happen — a `FORMAT.tmp` with no marker beside it is /// somebody else's file. /// /// # Why a name is never enough, and why the *type* of every entry is checked /// /// Contract review finding P1 (2026-07-29). This function used to ignore /// `INITIALIZING.tmp` **whatever it was**, on the argument that the name can /// only be this code path's residue by construction — and /// [`build_root_under_lock`] then opened it with `create` + `truncate`. The /// argument is not available here, twice over. It is the very claim the /// classification is running in order to establish, and before the store owns /// the root it has no standing to assume anything about what is in it. An /// operator's regular file of that name in an otherwise-empty configured root /// was destroyed silently; a **symlink** of that name was followed, so the /// truncation landed on a file outside the root entirely, and the link was then /// renamed away and unlinked so the evidence went with it. /// /// So every entry is judged by `DirEntry::file_type`, which reports the type of /// the directory entry itself and never of a symlink's target, and the two /// names whose contents can authorize anything are additionally opened with /// `O_NOFOLLOW` and validated ([`open_exact_marker`]). A symlink at `LOCK`, /// `INITIALIZING`, `INITIALIZING.tmp`, or `FORMAT.tmp`, and a directory or fifo /// at any of them, is a foreign entry and disqualifies the root. `FORMAT` is /// still decided on its name alone, which stays safe for the opposite reason: /// nothing on the state-2 path ever creates or truncates it — `initialize_root` /// installs it only by `rename_noreplace`, which refuses an occupied name /// including a symlink — so the worst a planted `FORMAT` can do is be read and /// fail to decode, which is a refusal. /// /// # The cost of this, stated rather than discovered /// /// A torn or partially written `INITIALIZING.tmp` — a crash between its /// creation and its fence — is now **refused** rather than overwritten, and /// that refusal is permanent until an operator removes the file. That is the /// intended trade and it is the same asymmetry the rest of this classification /// is built on: refusing a resumable root costs an operator time, and /// overwriting a real file costs them data. There is no test that can tell a /// torn marker of ours from a stranger's truncated file, because there is no /// difference between them on disk. /// /// # Why an interrupted initialization is recognized, and how it cannot be /// confused with a root that merely looks abandoned /// /// [`segment::initialize_root`] creates `quarantine/`, `staging/`, `shards/` /// and the per-shard tree *before* it installs `FORMAT`. A crash anywhere in /// that window used to leave a non-empty root with no `FORMAT`, which this /// function classified as states 3/4 and `open` refused — permanently, with a /// message about legacy layouts and migration, for a root that had never held /// a byte of anyone's data. That is why [`initialize_root_state_1`] installs /// `INITIALIZING` as its first durable act and removes it as its last. /// /// Two independent conditions must both hold before this returns /// `InterruptedInitialization`, and each alone is enough to exclude a genuine /// unrecognized layout: /// /// 1. **`INITIALIZING` decodes as this code path's marker** — a regular file of /// exactly [`INITIALIZING_MARKER_LEN`] bytes, opened `O_NOFOLLOW`, opening /// with [`INITIALIZING_MAGIC`] and a version this build knows. Nothing else /// in this crate, and nothing in a legacy layout, writes that file. A /// human's note that happens to be named `INITIALIZING` fails on content. /// 2. **Every entry in the root is inside the initialization skeleton, with the /// type this store creates it as** — `LOCK`, `INITIALIZING`, /// `INITIALIZING.tmp` and `FORMAT.tmp` as regular files, `quarantine`, /// `staging` and `shards` as directories, and nothing else. A single foreign /// name — a legacy `<64-hex>/` repository directory, a `notes.txt`, anything /// an operator put there — disqualifies the root however convincing the /// marker is, and so does a right name of the wrong type. /// /// So mistaking a real root for an abandoned one requires an operator to have /// placed a byte-exact copy of this marker into a directory that otherwise /// contains only names this store invented. The asymmetry is deliberate: the /// failure this can produce is a refusal that wastes time, and the failure it /// cannot produce is initializing over somebody's data. /// /// The ordering makes the converse hole impossible too. The marker is fenced, /// and the root directory fsynced, *before* the first directory of the tree is /// created — so a durable partial tree always has a durable marker beside it, /// and there is no crash point that leaves tree residue this function would /// refuse. /// /// A missing directory and an empty directory are the same state and are /// deliberately not distinguished. That is now true because /// [`initialize_root_state_1`] fences every directory entry it creates on the /// way down to the root, not because `initialize_root` fsyncs `root.parent()`: /// that parent fsync covers exactly one link, and a root reached through /// several newly created ancestors has several. fn classify_root(layout: &RootLayout) -> Result { let entries = match std::fs::read_dir(&layout.root) { Ok(entries) => entries, Err(error) if error.kind() == std::io::ErrorKind::NotFound => { return Ok(RootStartupState::AbsentOrEmpty) } // Every other failure — `ENOTDIR` because the configured root is a // regular file, `EACCES`, `ELOOP` — is reported as itself. A root this // process cannot read is not a root it may conclude is empty, and the // refusal has modified nothing. Err(error) => return Err(StoreError::from(error)), }; let mut has_format = false; let mut has_marker = false; // Anything that is neither `FORMAT` nor an ignorable probe artifact. // Non-empty in the sense states 3 and 4 mean it. let mut has_content = false; // Anything outside the closed set of names an initialization can leave. let mut has_foreign = false; for entry in entries { let entry = entry?; let name = entry.file_name(); // `DirEntry::file_type` is the type of the entry itself: it is `lstat`, // never `stat`, so a symlink reports as a symlink and this loop cannot // be made to describe an object outside the root. let kind = entry.file_type()?; if name == std::ffi::OsStr::new("FORMAT") { has_format = true; continue; } // Probe residue, ignorable only in the exact shape the probe leaves. if name == std::ffi::OsStr::new("LOCK") { if kind.is_file() { continue; } has_content = true; has_foreign = true; continue; } // The staging name, ignorable only when it is a regular file carrying // this code path's marker byte for byte. Anything else at that name — // a symlink, a directory, a stranger's file, a torn write — is foreign, // because the alternative is deciding it is ours and then writing // through it. if name == std::ffi::OsStr::new(INITIALIZING_TMP_NAME) { if kind.is_file() && open_exact_marker(&layout.root.join(INITIALIZING_TMP_NAME))?.is_some() { continue; } has_content = true; has_foreign = true; continue; } has_content = true; if name == std::ffi::OsStr::new(INITIALIZING_NAME) { // Content is checked below, once, and only if nothing foreign was // found; the type is checked here because a non-regular entry at // this name is foreign whatever it would have read as. if kind.is_file() { has_marker = true; } else { has_foreign = true; } continue; } // The rest of what `segment::initialize_root` creates directly in the // root, each with the type it creates it as. Named exhaustively rather // than pattern-matched: a name this list does not contain must // disqualify the root, and a wildcard here is the one edit that would // silently stop it doing so. The type is part of the name's meaning — // a symlink called `FORMAT.tmp` is not residue, it is an instruction to // write outside the root. `segment::write_fenced` refuses one directly // now as well; this list is why the root is disqualified rather than // initialized-then-refused, which is a different and better answer. if !matches!( (name.to_str(), kind.is_file(), kind.is_dir()), (Some("FORMAT.tmp"), true, _) | (Some("quarantine"), _, true) | (Some("staging"), _, true) | (Some("shards"), _, true) ) { has_foreign = true; } } // Order is plan §5.2's order. `FORMAT` wins over everything below it // because state 2 is decided before any other state; the interrupted shape // is decided next because it is state 1 and its evidence is positive; and // "empty" is evaluated against the same scan rather than a second one, so // no interleaved writer can make the two disagree. if has_format { return Ok(RootStartupState::Formatted); } if has_marker && !has_foreign && initializing_marker_is_ours(layout)? { return Ok(RootStartupState::InterruptedInitialization); } Ok(if has_content { RootStartupState::NonEmptyWithoutFormat } else { RootStartupState::AbsentOrEmpty }) } /// Plan §5.2 startup state 1: build the v2 tree under the root lock. /// /// # Why the lock, and why it is taken here rather than inherited /// /// Initializing a root is a mutation, and two processes configured for the /// same absent path will otherwise both classify it `AbsentOrEmpty` and both /// run [`segment::initialize_root`]. The loser's `rename_noreplace` of /// `FORMAT` would fail, but only after it had already created a shard tree and /// fsynced it into a root the winner had begun to consider its own. So the /// classification is repeated **under the lock**, and it is the second answer /// that authorizes the write. That is the whole race: `FORMAT` is installed by /// a no-replace rename as the last durable act of initialization, so if the /// second classification still says `AbsentOrEmpty` while this process holds /// `LOCK`, no other process can be mid-initialization. /// /// The lock is a [`segment::RootLock`] and is released by dropping it, which /// issues an explicit `LOCK_UN` rather than relying on a descriptor close a /// concurrently forked child can extend (commit `e050b6d`). /// /// The same rule governs the resume path. Finding an `INITIALIZING` marker /// before the lock authorizes nothing: another process may be building that /// tree *right now*, in which case this one is refused `AlreadyLocked` and never /// reaches the second classification. It is the classification taken **while /// holding `LOCK`** that authorizes finishing someone else's interrupted /// initialization, and by then the only process that could have been mid-build /// has released the lock and is gone. /// /// # Why the lock is then released and immediately retaken /// /// [`RecoverySession::open`] takes `LOCK` itself, and there is no constructor /// that adopts an already-held one — adding it is a change to A2's frozen /// surface and is filed as an interface request, not made here. The gap is /// safe but it is a real gap and worth stating exactly: an initialized root is /// complete and durable the moment this returns, so a second process entering /// the window can only take state 2, and if it does, this process's /// `RecoverySession::open` is refused `AlreadyLocked` — the correct answer for /// a root another process owns, not a corruption. What the window cannot /// produce is two initializations, because the second process no longer /// classifies the root as empty. fn initialize_root_state_1( layout: &RootLayout, options: &StoreOptions, ) -> Result { let counters = DurabilityCounters::default(); // `LOCK` lives inside the root, so the directory has to exist before it // can be locked. This is the one write that precedes the lock, and it is // the write state 1 is defined to perform: creating the path is // idempotent, two racing processes both succeed, and neither has yet // written a byte either could lose. It is reached only after the read-only // classification said the root has never been initialized, so it never runs // against states 2, 3, or 4. create_and_fence_root_path(&layout.root, &counters)?; let lock = segment::lock_root(layout)?; match classify_root(layout)? { // Both are state 1. The second says the marker is already on disk, so // `build_root_under_lock` re-fences the name it finds instead of // installing a second one. RootStartupState::AbsentOrEmpty => { build_root_under_lock(layout, options, false, &counters)? } RootStartupState::InterruptedInitialization => { build_root_under_lock(layout, options, true, &counters)? } // Another process initialized this root between the first // classification and this lock. Its tree is complete and fsynced, so // there is nothing to do and nothing to repair; the caller falls // straight through to state 2. RootStartupState::Formatted => {} // Non-empty with no `FORMAT` and no marker, discovered only under the // lock. The refusal is the same one state 3/4 gets, and this path has // written nothing but the directories it was told were absent. RootStartupState::NonEmptyWithoutFormat => { return Err(StoreError::NotImplemented( "StoreEngine::open startup states 3 and 4 (LegacyLayout carrying the exact \ migrate-store command, UnrecognizedLayout) — B1 NamespaceTxn, scope 6-B1 \ deliverable 1. The root became non-empty without a FORMAT between \ classification and the root lock; it has not been modified", )) } } drop(lock); Ok(counters.snapshot()) } /// Create every missing directory on the way to `root`, and fence each new /// directory entry — deepest first. /// /// # Why this is not `create_dir_all` followed by `initialize_root` /// /// `create_dir_all` creates *every* missing ancestor, and /// [`segment::initialize_root`] fences exactly one link: `root.parent()`. So a /// configured root of `.../not/yet/a-root` under an existing directory left the /// entries naming `not` and `yet` unfenced, while `open` returned success. A /// power loss could then take an ancestor and the whole initialized root with /// it — a store that reported itself initialized, reachable only through /// directory entries no one had made durable. /// /// # Why fencing rather than refusing an absent parent /// /// Refusing would have been less code and is genuinely safe. It was rejected /// because it moves a durability defect onto the operator as a configuration /// restriction: a root path under a per-instance directory the store is /// expected to create is the ordinary deployment shape, and `StoreEngine::open` /// already promises to build an absent root. "Absent" would then have had to /// mean "absent by exactly one component", which is a rule nothing on disk /// explains and which every caller would have to learn by being refused. /// /// # The ordering, which is the whole content of the fix /// /// Only the directories *this call* creates are fenced, and a directory entry /// lives in the parent that names it — so fencing directory `D` is fsyncing /// `parent(D)`. They are fenced deepest first: a child's entry must be durable /// before the entry naming its parent, because the reverse order can leave a /// crash image in which a fenced parent names a child that was never fenced. /// For `base/a/b/c` with `base` already present, that is fsync(`b`), fsync(`a`), /// fsync(`base`) — one fsync per created directory. /// /// `c`'s own contents are not this function's concern: it is empty here, and /// [`segment::initialize_root`] fsyncs the root itself once it has built the /// tree inside it. /// /// Routed through `sys::fsync_dir` because that module is the crate's sole /// durability funnel; a raw `File::open(..).sync_all()` here would be invisible /// to the counters that make this fix testable at all. fn create_and_fence_root_path( root: &std::path::Path, counters: &DurabilityCounters, ) -> Result<(), StoreError> { // Deepest first. `try_exists` rather than `exists` so that a path this // process cannot stat is an error instead of silently "missing" — creating // over an `EACCES` would be a refusal turned into a mutation. let mut missing: Vec<&std::path::Path> = Vec::new(); let mut cursor = Some(root); while let Some(path) = cursor { if path.try_exists()? { break; } missing.push(path); cursor = path .parent() .filter(|parent| !parent.as_os_str().is_empty()); } if missing.is_empty() { return Ok(()); } // Shallowest first: a directory cannot be created before the one that names // it. `create_dir` rather than `create_dir_all` so that what this call // created is exactly what it fences — `create_dir_all` would silently // absorb an ancestor another process created in the same instant and leave // this one believing it had made the link. for path in missing.iter().rev() { match std::fs::create_dir(path) { Ok(()) => {} // A racing initializer of the same path got there first. Its own // fence covers the link either way, and fencing it twice is // harmless; refusing here would make two correct processes fight. Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} Err(error) => return Err(StoreError::from(error)), } } for path in &missing { let parent = path .parent() .filter(|parent| !parent.as_os_str().is_empty()) .unwrap_or_else(|| std::path::Path::new(".")); crate::sys::fsync_dir(parent, counters)?; } Ok(()) } /// Install the initialization marker, build the tree, and remove the marker. /// /// The order is the contract [`classify_root`] reads back: /// /// 1. `INITIALIZING.tmp` is created with `O_EXCL`, written and fenced /// ([`install_initializing_marker`]), renamed onto `INITIALIZING` with /// `rename_noreplace`, and the root directory fsynced. The marker is /// therefore never observable torn under its installed name, and it is /// durable **before** any part of the tree exists. /// 2. [`segment::initialize_root`] builds the tree and installs `FORMAT`. /// 3. The marker is unlinked and the removal fenced. /// /// A crash in step 1 leaves at most `LOCK` and `INITIALIZING.tmp`. A *whole* /// `INITIALIZING.tmp` classifies as an empty root and is adopted by the next /// attempt; a torn one is refused, which is the trade /// [`install_initializing_marker`] states. A crash in step 2 leaves the marker and a partial /// skeleton, which classify as `InterruptedInitialization` and land back here. /// A crash in step 3 leaves a complete, valid root, which classifies as state 2 /// — the marker is then inert litter that `FORMAT` outranks, and removing it on /// a state-2 open is rejected precisely because it would make reopening a /// formatted root a mutation. /// /// `resuming` says the marker is already there. It is passed rather than /// re-derived because `rename_noreplace` onto an existing `INITIALIZING` would /// fail, and because re-deciding it here would be a third classification that /// could disagree with the one taken under the lock. /// /// A resume restarts the build from the *current* configuration rather than the /// one the marker records: the interrupted attempt published no `FORMAT`, so it /// froze no topology and has no claim on this one. If the two disagree on /// `shard_count`, the surplus shard directories from the wider attempt are left /// behind — empty, referenced by no `FORMAT`, and read by nothing, since every /// consumer iterates `0..FORMAT.shard_count`. fn build_root_under_lock( layout: &RootLayout, options: &StoreOptions, resuming: bool, counters: &DurabilityCounters, ) -> Result<(), StoreError> { let marker = layout.root.join(INITIALIZING_NAME); if resuming { // The marker exists but its *name* may not be durable: the crash could // have landed between the rename and the fsync below. Fencing it again // before touching the tree is what keeps "a durable partial tree always // has a durable marker beside it" true across a second interruption. crate::sys::fsync_dir(&layout.root, counters)?; } else { let tmp = layout.root.join(INITIALIZING_TMP_NAME); let bytes = initializing_marker_bytes(options.shard_count, now_micros()); install_initializing_marker(&tmp, &bytes, counters)?; crate::sys::rename_noreplace(&tmp, &marker)?; crate::sys::fsync_dir(&layout.root, counters)?; } segment::initialize_root( layout, options.shard_count, fresh_root_uuid(), now_micros(), counters, )?; // Last, and after `FORMAT` is durable. Removing it earlier would open a // window in which the tree exists, `FORMAT` does not, and nothing says the // residue is ours — which is the exact defect this marker was added to // close. crate::sys::unlink(&marker)?; crate::sys::fsync_dir(&layout.root, counters)?; Ok(()) } /// Put the marker's bytes at the staging name and fence them, **without ever /// writing through a name this process has not established is its own**. The /// name is not durable when this returns; the caller renames and fsyncs the /// directory. /// /// # The two branches, and why neither of them can truncate /// /// The ordinary case creates the name: [`crate::sys::create_new_nofollow`] is /// `O_CREAT | O_EXCL | O_NOFOLLOW`, so it either brings a brand-new regular /// file into existence in *this* directory or reports that the name is taken. /// `O_CREAT | O_EXCL` is specified to fail on a symlink whether or not the link /// resolves, so there is no arrangement of the root that can redirect this /// write. /// /// The name being taken is the interesting case, and this is where contract /// review finding P1 was. What used to happen was `create` + `truncate`: an /// operator's file at that name was destroyed, and a *symlink* at that name was /// followed so the destruction landed on a file outside the root, after which /// the link was renamed and unlinked and the evidence went with it. What /// happens now is that the occupant is validated by [`open_exact_marker`] — /// regular file, exact length, exact magic, known version — and only *then* /// adopted, in place, by fencing the bytes that are already there. Adoption /// writes nothing: the file already holds exactly what this call was about to /// write, so the fence is the whole remaining act. /// /// Adopting the occupant's bytes rather than replacing them also keeps the /// resume rule consistent. `shard_count` and `created_at_micros` in the marker /// are evidence of the interrupted attempt, not input to this one /// ([`initializing_marker_bytes`]), and the resume-from-`INITIALIZING` path /// already carries the earlier attempt's bytes forward untouched. This does the /// same thing one step earlier. /// /// # What is refused, and what that costs /// /// Everything else at that name: a stranger's file, a symlink, a directory, and /// a **torn or partially written marker** — a crash between the create and the /// fence. The torn case is the one that costs something real: a root that was /// previously resumable now needs an operator to remove one file before it will /// initialize. That is the intended trade, and it is not close. Refusing a /// resumable root costs an operator time; overwriting a real file costs them /// their data. And nothing on disk distinguishes our torn 12-byte marker from a /// stranger's truncated 12-byte file, so a rule that recovers the first /// necessarily destroys the second. /// /// # Standing disclosure /// /// This began as a second copy of `segment::write_fenced`, which is private to /// A1's file, and the duplication was filed as an interface request. Neither can /// follow a symlink any more — contract review 2026-07-29-B routed /// `write_fenced` through the same funnel — so what remains is a real difference /// in refusal rule rather than in safety: this one refuses an existing regular /// file, because a marker that already exists is somebody's data and adopting it /// would be a decision about content; `write_fenced` adopts and empties one, /// because a `.tmp` name is residue from an interrupted attempt at that exact /// write and reusing it is how the retry works. Both are correct for their name. /// Any hoisting therefore has to keep two rules, not pick one. fn install_initializing_marker( tmp: &std::path::Path, bytes: &[u8], counters: &DurabilityCounters, ) -> Result<(), StoreError> { match crate::sys::create_new_nofollow(tmp)? { Some(mut file) => { crate::sys::write_vectored_all(&mut file, &[std::io::IoSlice::new(bytes)], counters)?; crate::sys::fdatasync(&file, counters)?; Ok(()) } // Reachable in normal operation when a previous attempt crashed between // the create and the rename, and reachable adversarially at any time, // since the classification that ran a moment ago under `LOCK` is a // statement about the past. Both are handled by looking, not by // assuming. None => match open_exact_marker(tmp)? { Some(file) => { crate::sys::fdatasync(&file, counters)?; Ok(()) } None => Err(StoreError::UnrecognizedLayout(format!( "{}: the initialization staging name is occupied by something that is not this \ store's marker — a foreign file, a symlink, a directory, or a torn marker from \ an interrupted attempt. Refusing to write through it; nothing has been modified. \ Remove that entry to let the root initialize", tmp.display() ))), }, } } /// A fresh `root_uuid` for a root being initialized. /// /// `FORMAT.root_uuid` binds every `CURRENT`, manifest, journal, segment, index /// run, and checkpoint in the root (scope 3.1), and its whole job is to make a /// file copied in from another instance detectable. So it must not be a /// function of anything two roots can share — the path in particular, since a /// root deleted and recreated at the same path is exactly the case this is /// meant to catch. /// /// **Duplication, disclosed rather than hidden:** `drive.rs` has a private /// `fresh_id` doing the same thing for journal and root identifiers. This is a /// second implementation of "derive 16 bytes with enough entropy to be unique /// per process and per call", and it is here because `drive.rs` is A1's file /// and is `#[cfg(feature = "store-internals")]`, so neither borrowing it nor /// editing it is available to this deliverable. Hoisting one into `sys` is /// filed as an interface request. fn fresh_root_uuid() -> [u8; 16] { use std::sync::atomic::AtomicU64; static COUNTER: AtomicU64 = AtomicU64::new(0); let stack = 0u8; let mut hasher = blake3::Hasher::new(); hasher.update(b"levcs-store-root-uuid/v1\0"); hasher.update(&now_micros().to_le_bytes()); hasher.update(&COUNTER.fetch_add(1, Ordering::Relaxed).to_le_bytes()); hasher.update(&std::process::id().to_le_bytes()); hasher.update(&(&stack as *const u8 as usize).to_le_bytes()); let mut id = [0u8; 16]; id.copy_from_slice(&hasher.finalize().as_bytes()[..16]); id } // =========================================================================== // Startup: the recovered state becomes the first committed root // =========================================================================== fn initial_committed_root(shards: &[RecoveredShard]) -> Result { let mut repositories = RepoMap::new(); let mut terminal_statuses = TerminalStatusMap::new(); let mut shard_committed_sequences = ShardSequenceMap::new(); let mut retained_generations = GenerationMap::new(); let mut delta_layers_newest_first = Vector::new(); let mut sealed_runs_newest_first = Vector::new(); for shard in shards { for (namespace, record) in shard.catalog.iter() { repositories.insert( *namespace, Arc::new(repo_state_from_recovery(record, &shard.refs)?), ); } for receipt in &shard.receipts { terminal_statuses.insert( OperationKey::new(receipt.namespace, receipt.operation_id), TerminalStatusEntry::Committed(retained_receipt(receipt)?), ); } if let Some(sequence) = shard.committed_shard_sequence { shard_committed_sequences.insert(shard.shard_index, sequence); } // The generation, not its name: recovery opened these artifacts, and // handing over a path instead of the live reference would leave a // window in which nothing holds them and reclamation is legal. retained_generations.insert( shard.retained_generation.id, Arc::clone(&shard.retained_generation), ); for layer in shard.index.delta_layers() { delta_layers_newest_first.push_back(layer.clone()); } for run in shard.index.sealed_runs() { sealed_runs_newest_first.push_back(Arc::clone(run)); } } Ok(CommittedRoot::new( repositories, LayeredObjectIndex::new(delta_layers_newest_first, sealed_runs_newest_first), terminal_statuses, shard_committed_sequences, retained_generations, )) } fn repo_state_from_recovery( record: &NamespaceRecord, refs: &[RefRecord], ) -> Result { let mut typed_refs = TypedRefMap::new(); for reference in refs.iter().filter(|r| r.namespace == record.namespace) { // Contract review 2026-07-28-A item 4: the ref-kind table is stated // once, in `checkpoint.rs`, in each direction. This file used to carry // a third copy that agreed with it today. typed_refs.insert(reference.target()?, reference.target); } Ok(RepoState { repo_sequence: record.repo_sequence, current_authority: record.current_authority, genesis_authority: record.genesis_authority, refs: typed_refs, lifecycle: record.lifecycle, storage_mode: record.storage_mode, previous_event_digest: record.previous_event_digest, }) } fn retained_receipt(record: &ReceiptRecord) -> Result { // Recovery step 10 has already promoted every row, so `None` here means a // receipt reached the root without passing through the promotion that // decides its retention. That is a store-consistency failure, not a value // to substitute. let first_visible_at_micros = record.first_receipt_visibility_micros.ok_or_else(|| { StoreError::Corruption(format!( "recovered receipt for operation {} has no first-visibility instant", record.operation_id.to_hex() )) })?; Ok(RetainedReceipt { operation_digest: record.operation_digest, receipt: CommitReceipt { operation_id: record.operation_id, repo_sequence: record.repo_sequence, current_authority: record.current_authority, refs: record.refs.clone(), objects_new: record.objects_new, }, shard_sequence: record.shard_sequence, retry_until_micros: record.retry_until_micros, first_visible_at_micros, receipt_visible_until_micros: record.receipt_visible_until_micros, }) } fn spawn_shard_writer( shared: Arc, recovered: RecoveredShard, ) -> Result { let shard_index = recovered.shard_index; let tail = recovered.tail.as_ref().ok_or_else(|| { StoreError::Corruption(format!( "shard {shard_index} reported ready with no active journal" )) })?; let counters = Arc::new(DurabilityCounters::default()); let (journal, _scan) = Journal::open(&tail.path, &shared.root_uuid, Arc::clone(&counters))?; // The journal recovery installed is empty, so its next sequence is the one // after the committed tail. Checking rather than assuming: a mismatch here // would silently append into a sequence range recovery already adopted. let expected_next = recovered .committed_shard_sequence .map(|sequence| sequence.saturating_add(1)) .unwrap_or(0); if journal.next_shard_sequence() != expected_next { return Err(StoreError::Corruption(format!( "shard {shard_index} recovered through sequence {expected_next} but its active \ journal resumes at {}", journal.next_shard_sequence() ))); } let mut repositories = BTreeMap::new(); for (namespace, record) in recovered.catalog.iter() { repositories.insert( *namespace, repo_state_from_recovery(record, &recovered.refs)?, ); } let writer = ShardWriter { builder: GroupBuilder::from_options(&shared.options), pending: Vec::new(), waiters: Vec::new(), group_base: BTreeMap::new(), in_flight_reservation: None, repositories, tail_generation: tail.logical_generation, journal, shard_index, poison: None, shared, }; let (submissions, receiver) = crossbeam_channel::bounded(submission_queue_depth(&writer)); let thread = std::thread::Builder::new() .name(format!("levcs-store-shard-{shard_index}")) .spawn(move || run_shard_writer(writer, receiver))?; Ok(ShardHandle { submissions: Some(submissions), thread: Some(thread), counters, }) } /// Two maximal groups' worth of queue. /// /// One would make every group boundary a backpressure event even when the /// device is keeping up; unbounded would replace a typed `Overloaded` refusal /// with unbounded memory, which decision 9.8 rules out for the status root for /// the same reason. fn submission_queue_depth(writer: &ShardWriter) -> usize { (writer.shared.options.max_group_transactions as usize).saturating_mul(2) } // =========================================================================== // The shard writer // =========================================================================== /// One shard's sequencer and journal writer. /// /// Everything in here is owned by exactly one thread. `repo_sequence`, the /// per-repository event chain, and `shard_sequence` are assigned with no /// cross-shard coordination, which is sound only because a repository's shard /// assignment never moves (scope 2.5). struct ShardWriter { shared: Arc, shard_index: u16, journal: Journal, builder: GroupBuilder, /// Parallel to the frames held by `builder`, in the same order. pending: Vec, /// Every caller this thread owes an outcome to, oldest first. /// /// Deliberately *not* stored inside [`Prepared`]. A panic anywhere in the /// sequencer or the publication window unwinds this thread, and a /// completion that unwound with the frame it belonged to is a request that /// never resolves — indistinguishable from a store that hung. Keeping the /// waiters on the writer means the panic handler can still reach them, and /// each one carries the publication phase it is actually in so that the /// handler resolves it as that phase rather than as the worst one. waiters: Vec, /// Speculative state: every earlier accepted transaction in the pending /// group is already applied here, which is what scope 6.3 step 1 requires /// the precondition checks to see. repositories: BTreeMap, /// The repository state each namespace held *before* the open group /// touched it, so the whole group can be re-derived from a clean base when /// a member is dropped by the pre-mark deadline recheck. /// /// `None` records "this namespace did not exist yet", which is a different /// fact from "it existed with default state" and is the difference between /// restoring a repository and inventing one. group_base: BTreeMap>, /// The status-root reservation held by a transaction that is inside /// `prepare` and has not yet joined `pending`. /// /// Scope 6.3 steps 1-3 are pre-append and definitively absent on failure, /// including on a panic. Without this, a panic between `reserve` and the /// push into `pending` leaves a `Pending` entry that nothing owns and that /// no recovery resolves, because recovery only ever sees what reached the /// device — and nothing did. in_flight_reservation: Option, /// Logical index generation of the active journal, so an index location /// built here resolves to the same pinned file after a reopen. tail_generation: u64, poison: Option, } /// One shard's unsealed index backlog, summed from one captured root. #[derive(Copy, Clone, Debug, Default)] struct UnsealedBacklog { layers: usize, entries: u64, encoded_bytes: u64, /// The highest shard sequence any of those layers covers, which is what a /// seal built from them may claim to cover. through_shard_sequence: u64, } /// Which of scope 6.3's phases a caller is actually in. /// /// The fate of a request on an abnormal exit is a function of this and nothing /// else. Treating every panic as a publication panic is what lets a transaction /// that never reached step 4 be reported as possibly durable, and a transaction /// that is already committed be reported as poisoned. enum WaiterPhase { /// Steps 1-3. Nothing is appended and nothing can be; a failure here is /// definitively absent and releases the reservation. PreAppend, /// Steps 4-8, the poison window. A failure here leaves the operation /// `Resolving` and the shard read-only until recovery. Publishing, /// Steps 9-10. The fence succeeded and the root published, so the only /// honest outcome is the receipt. Scope 6.3: failure here is not a failure. Committed(CommitReceipt), } struct Waiter { completion: SharedCompletion, phase: WaiterPhase, /// The status-root reservation this waiter owns while it is `PreAppend`. /// Taken over by step 8 once the group publishes. reservation: Option, } impl Waiter { fn new(completion: SharedCompletion) -> Self { Self { completion, phase: WaiterPhase::PreAppend, reservation: None, } } } /// One sequenced transaction, complete except for where its frame landed. struct Prepared { key: OperationKey, operation_digest: ObjectId, retry_until_micros: i64, first_visible_at_micros: i64, shard_sequence: u64, /// The repository state *after* this transaction. repo_state: Arc, objects: Vec<(ObjectId, u8)>, receipt: CommitReceipt, /// Retained so the whole group can be re-sequenced, re-chained, and /// re-signed when the pre-mark deadline recheck drops a member. /// /// Signing covers `SignedCommittedTransactionV1::signing_digest`, which /// commits to the chained `previous_event_digest`; dropping a member /// therefore invalidates every signature after it. Keeping only the derived /// frame would make a partial repair the only available repair, and a /// partial repair is a durable, correctly fenced frame carrying a signature /// that verifies against nothing. transaction: Arc, } fn run_shard_writer(mut writer: ShardWriter, submissions: Receiver) { let unwound = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { loop { // The idle bound is a receive deadline, not a timer thread: an // open group closes on time even when nothing else arrives, and a // shard with no open group blocks instead of spinning. let message = match writer.builder.time_to_close() { Some(remaining) => submissions.recv_timeout(remaining), None => submissions .recv() .map_err(|_| RecvTimeoutError::Disconnected), }; match message { Ok(submission) => writer.accept(submission), Err(RecvTimeoutError::Timeout) => writer.publish_open_group(), Err(RecvTimeoutError::Disconnected) => { // A transaction already accepted is already sequenced. // Closing the engine publishes it rather than dropping it // with its waiter attached. writer.publish_open_group(); break; } } } })); if let Err(payload) = unwound { // A writer-thread panic still resolves every caller this thread owes an // outcome to — a completion that unwound with its frame is a request // that never resolves. What each caller is told is decided by the phase // it was in, not by where the catch happens to be: this boundary sees // pre-append requests and published-but-unwoken requests as well as the // poison window, and reporting all three as `ShardPoisoned` would claim // that a transaction which never reached step 4 might be durable. let poison = writer.poison_error("the writer thread panicked inside the publication window".into()); let mut poisoned_any = false; for waiter in std::mem::take(&mut writer.waiters) { match waiter.phase { WaiterPhase::PreAppend => { if let Some(key) = waiter.reservation { writer.release_reservation(&key); } waiter.completion.complete(Err(pre_append_panic_error())); } WaiterPhase::Publishing => { poisoned_any = true; waiter.completion.complete(Err(poison.clone())); } WaiterPhase::Committed(receipt) => { waiter.completion.complete(Ok(receipt)); } } } if let Some(key) = writer.in_flight_reservation.take() { // A panic strictly inside `prepare`, before the transaction had a // waiter phase to be in. writer.release_reservation(&key); } if poisoned_any { writer.poison.get_or_insert(poison); } std::panic::resume_unwind(payload); } } /// The outcome of a panic in scope 6.3 steps 1-3. /// /// Definitive, not poisoning: no sequence was consumed, no byte was written, /// and the reservation is released, so the operation reads back `Unknown` and /// the caller may retry. `Conflict` is the taxonomy's definitive-rejection /// variant and is what the ordinary pre-append refusals in this file already /// return. fn pre_append_panic_error() -> StoreError { StoreError::Conflict( "the sequencer panicked before this transaction was marked Resolving; scope 6.3 steps \ 1-3 are pre-append, so nothing was appended and nothing can be" .into(), ) } impl ShardWriter { fn accept(&mut self, submission: Submission) { let Submission { transaction, completion, } = submission; // Registered before anything that can fail or panic. Until this // transaction joins `pending`, it is the last waiter in the list and // `fail_newest_waiter` is what resolves it. self.waiters.push(Waiter::new(completion)); if let Some(poison) = self.poison.clone() { self.fail_newest_waiter(poison); return; } let transaction = Arc::new(transaction); let (prepared, frame) = match self.prepare_caught(&transaction) { Ok(pair) => pair, Err(error) => { self.fail_newest_waiter(error); return; } }; if let Err(frame) = self.builder.push(frame) { // The group is full. The new request is rolled back completely — // reservation released, speculative state restored — *before* the // open group publishes, and re-prepared afterwards. // // The alternative, publishing while this request is prepared but // not yet in `pending`, is what made a publication panic report a // trailing request as poisoned though it never appended, and leave // its reservation pending. It also made its `shard_sequence` a // prediction about how many members the group about to publish // would keep, which the pre-mark deadline recheck can change. drop(frame); self.roll_back_prepared(&prepared); self.publish_open_group(); if let Some(poison) = self.poison.clone() { self.fail_newest_waiter(poison); return; } let (reprepared, frame) = match self.prepare_caught(&transaction) { Ok(pair) => pair, Err(error) => { self.fail_newest_waiter(error); return; } }; self.builder .push(frame) .expect("an empty group admits any frame"); self.attach_newest_reservation(reprepared.key); self.pending.push(reprepared); } else { self.attach_newest_reservation(prepared.key); self.pending.push(prepared); } if self.builder.is_closed() { self.publish_open_group(); } } /// Scope 6.3 steps 1-3 with their own unwind boundary. /// /// A panic in here is a pre-append failure by definition, so it must not /// take the shard with it: `later_append_allowed_before_recovery` is true /// for every step 1-3 row of the frozen oracle, and a dead writer thread /// makes that false no matter what the error says. fn prepare_caught( &mut self, transaction: &Arc, ) -> Result<(Prepared, Frame), StoreError> { let sequence = self.journal.next_shard_sequence() + self.pending.len() as u64; match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { self.prepare(transaction, sequence) })) { Ok(result) => result, Err(_) => { if let Some(key) = self.in_flight_reservation.take() { self.release_reservation(&key); } Err(pre_append_panic_error()) } } } /// Hand the reservation the newest waiter is about to depend on to that /// waiter, so a panic before publication releases exactly one reservation /// per unresolved caller. fn attach_newest_reservation(&mut self, key: OperationKey) { self.in_flight_reservation = None; if let Some(waiter) = self.waiters.last_mut() { waiter.reservation = Some(key); } } /// Undo a prepared-but-unpublished transaction completely: release its /// status reservation and restore the speculative repository state it /// advanced. No sequence was consumed, because sequences are assigned at /// publication. fn roll_back_prepared(&mut self, prepared: &Prepared) { self.release_reservation(&prepared.key); self.in_flight_reservation = None; let namespace = prepared.key.namespace; // The group base still holds this namespace's pre-image only if this // transaction is the one that recorded it; otherwise an earlier member // of the open group did, and its post-state is what must be restored. let restored = self .pending .iter() .rev() .find(|earlier| earlier.key.namespace == namespace) .map(|earlier| (*earlier.repo_state).clone()); match restored { Some(state) => { self.repositories.insert(namespace, state); } None => match self.group_base.remove(&namespace) { Some(Some(before)) => { self.repositories.insert(namespace, before); } Some(None) => { self.repositories.remove(&namespace); } None => {} }, } } /// Resolve the most recently registered waiter, which is the transaction /// currently being accepted. fn fail_newest_waiter(&mut self, error: StoreError) { if let Some(waiter) = self.waiters.pop() { if let Some(key) = waiter.reservation { self.release_reservation(&key); } waiter.completion.complete(Err(error)); } } /// Admission, reservation, and scope 6.3 steps 1 to 3 for one new /// submission. /// /// Ordering inside this function is load-bearing. Every refusal that can /// happen must happen before the reservation is turned into a group /// member. `shard_sequence` is *not* consumed here: it is the tentative /// position `sequence` this transaction would occupy, and the journal's /// counter advances only at publication, once the pre-mark deadline /// recheck has decided which members survive. Consuming it here would make /// a dropped member leave a sequence hole, which poisons the shard. fn prepare( &mut self, transaction: &Arc, sequence: u64, ) -> Result<(Prepared, Frame), StoreError> { let namespace = transaction.namespace; let key = OperationKey::new(namespace, transaction.operation_id); let now = now_micros(); // The configured per-transaction ceilings, checked where a consumer's // input actually arrives: `submit` is the only entry point that takes // a caller-shaped transaction, and the codec's static maxima are a // hostile-decode bound, not an operator's configuration. Refused before // anything is reserved or sequenced. let objects = transaction.objects.len() as u64; let allowed_objects = u64::from(self.shared.options.max_objects_per_transaction); if objects > allowed_objects { return Err(StoreError::LimitExceeded { limit: "max_objects_per_transaction", observed: objects, allowed: allowed_objects, }); } let refs = transaction.refs.len() as u64; let allowed_refs = u64::from(self.shared.options.max_refs_per_transaction); if refs > allowed_refs { return Err(StoreError::LimitExceeded { limit: "max_refs_per_transaction", observed: refs, allowed: allowed_refs, }); } // Decision 9.8 ordering, first rule: a committed or expired operation // is answered from the committed root and never touches the status // root. The answer itself — returning the durable receipt to a // retrying caller — is deliverable 7. if self.shared.committed.load().terminal_status(&key).is_some() { return Err(StoreError::NotImplemented( "submit idempotency against an already committed operation — B1 NamespaceTxn, \ scope 6-B1 deliverable 7", )); } // Plan §5.3: at a hard ceiling the shard seals synchronously **before // admitting more work**. Here, and not after publishing, because this is // the last point at which a refusal has reserved and sequenced nothing — // a seal that fails partway through has to poison, and poisoning a shard // that has just accepted a transaction owes that caller an answer it can // no longer give. self.seal_index_if_required()?; // The reservation is what makes two concurrent submissions of one // operation ID resolvable at all. It happens before sequencing, so a // refusal here has consumed nothing. self.reserve(key, transaction, now)?; // Everything from here on releases the reservation on failure. Scope // 6.3 steps 1-3 fail definitively and pre-append, and leaving a // `Pending` entry behind would report `Pending` forever for an // operation the oracle calls definitively absent. match self.sequence_into_frame(transaction, sequence, now) { Ok(pair) => Ok(pair), Err(error) => { self.release_reservation(&key); Err(error) } } } /// Scope 6.3 steps 1 to 3 and the frame they feed, for a transaction whose /// reservation already exists. /// /// Called both by `prepare` for a new submission and by `reform_group` for /// a member being re-sequenced after an earlier member was dropped. There /// is exactly one of these because the repair path must produce bytes /// indistinguishable from the ones the first pass would have produced at /// the same position; two implementations of "sequence, chain, sign, encode" /// is precisely how a repaired suffix acquires a signature over a chain /// that no longer exists. fn sequence_into_frame( &mut self, transaction: &Arc, shard_sequence: u64, now: i64, ) -> Result<(Prepared, Frame), StoreError> { let namespace = transaction.namespace; let key = OperationKey::new(namespace, transaction.operation_id); let state = self.speculative_state(transaction)?; let sequenced = self.sequence_and_sign(transaction, &state, now)?; let payload = sequenced.payload.encode_canonical()?; let payload_len = payload.len() as u64; let total_len = frame_total_len(payload_len); // Rotation and sealing are not in this slice. Refusing a group that no // longer fits the preallocated journal here — with the exact length in // hand, before anything is marked `Resolving` — keeps the missing // deliverable a clean pre-append refusal rather than a `LimitExceeded` // discovered between the `Resolving` mark and the fence. if !self .journal .fits(self.builder.bytes().saturating_add(total_len)) { return Err(StoreError::NotImplemented( "active journal rotation and sealing — B1 NamespaceTxn, scope 6-B1 \ deliverable 1 and scope 3.4", )); } let frame = Frame { header: FrameHeader { flags: 0, total_len, journal_id: self.journal.journal_id(), shard_sequence, repo_sequence: sequenced.receipt.repo_sequence, namespace: *key.namespace.as_bytes(), operation_id: *key.operation_id.as_bytes(), operation_digest: transaction.operation_digest, payload_len, // Derived by `Frame::encode`; carried here so the value this // writer holds is the value that reaches the device. payload_digest: crate::format::digest( crate::format::FRAME_PAYLOAD_DIGEST_DOMAIN, &payload, ), }, payload, }; // The speculative state of scope 6.3 step 1 advances here, not at // publication: every later member of the forming group must check its // preconditions against every earlier accepted one. // // The pre-image is recorded the first time the open group touches a // namespace, and it is the *only* way back: the pre-mark deadline // recheck can drop any member, and every retained member after it has // to be re-derived from the state that member saw, not from the state // the dropped member left. self.group_base .entry(namespace) .or_insert_with(|| self.repositories.get(&namespace).cloned()); self.repositories .insert(namespace, (*sequenced.repo_state).clone()); Ok(( Prepared { key, operation_digest: sequenced.operation_digest, retry_until_micros: sequenced.retry_until_micros, first_visible_at_micros: sequenced.first_visible_at_micros, shard_sequence, repo_state: sequenced.repo_state, objects: sequenced.objects, receipt: sequenced.receipt, transaction: Arc::clone(transaction), }, frame, )) } fn speculative_state( &self, transaction: &ValidatedTransaction, ) -> Result { let namespace = transaction.namespace; let existing = self.repositories.get(&namespace); if let Some(genesis) = transaction.create_genesis_authority { if let Some(existing) = existing { // Plan §4 identity invariant 2: the binding is permanent. return Err(StoreError::Conflict(format!( "namespace {} is already bound to genesis {}; refusing to rebind to {}", namespace.to_hex(), hex::encode(existing.genesis_authority.0), hex::encode(genesis.0) ))); } return Ok(RepoState { repo_sequence: 0, // Zero, not the genesis: `old_authority` in the event and // `expected_authority` in the frame are the same fact written // twice, and a create transaction's expected authority is // nothing. Seeding this with the genesis would make the two // copies disagree in every init frame the store ever writes. current_authority: ObjectId([0u8; 32]), genesis_authority: genesis, refs: TypedRefMap::new(), lifecycle: NamespaceLifecycle::Active, storage_mode: NamespaceStorageMode::Full, previous_event_digest: ObjectId([0u8; 32]), }); } let existing = existing.ok_or_else(|| { StoreError::Conflict(format!( "namespace {} is not bound; a transaction that does not create a repository \ cannot be the first one in it", namespace.to_hex() )) })?; if !matches!(existing.lifecycle, NamespaceLifecycle::Active) { return Err(StoreError::Conflict(format!( "namespace {} does not admit mutation in lifecycle {:?}", namespace.to_hex(), existing.lifecycle ))); } Ok(existing.clone()) } /// Scope 6.3 steps 2 and 3, and the mutable-precondition half of step 1. /// /// The signer is called inline and in repository-sequence order, so no /// speculative suffix can exist to repair: each member is chained, signed, /// and sequenced completely before the next is looked at. Moving signing /// onto its own bounded pool (deliverable 5) is what introduces /// out-of-order completion, and with it the suffix re-chaining that /// deliverable owns. fn sequence_and_sign( &mut self, transaction: &ValidatedTransaction, before: &RepoState, now: i64, ) -> Result { let namespace = transaction.namespace; let creates = transaction.create_genesis_authority.is_some(); // Mutable preconditions only. Nothing here re-parses, re-hashes, // re-verifies a signature, or re-evaluates policy: those ran before // the transaction was validated, and repeating them inside the // mutation lane is what the sequencer exists to avoid. let mut refs = before.refs.clone(); let mut applied = Vec::with_capacity(transaction.refs.len()); for update in &transaction.refs { let observed = refs.get(&update.target).copied(); if observed != update.expected { return Err(StoreError::Conflict(format!( "typed ref CAS for {:?} in namespace {} expected {:?} but observed {:?}", update.target, namespace.to_hex(), update.expected.map(|id| hex::encode(id.0)), observed.map(|id| hex::encode(id.0)) ))); } let new = match update.mutation { RefMutation::Set(target) => { refs.insert(update.target.clone(), target); Some(target) } RefMutation::Delete => { refs.remove(&update.target); None } }; applied.push(AppliedRefV1 { target: update.target.clone(), old: observed, new, force: update.force, }); } let expected_authority = transaction .expected_authority .unwrap_or(ObjectId([0u8; 32])); if !creates && expected_authority != before.current_authority { return Err(StoreError::Conflict(format!( "authority CAS for namespace {} expected {} but observed {}", namespace.to_hex(), hex::encode(expected_authority.0), hex::encode(before.current_authority.0) ))); } let new_authority = transaction .new_authority .unwrap_or(before.current_authority); if creates && Some(before.genesis_authority) != transaction.new_authority { // The catalog binds `current_authority` from the event's // `new_authority` and `genesis_authority` from the create record. // A create that moves the first to something other than the second // would bind a repository to an authority it never held. return Err(StoreError::Conflict( "a repository-create transaction must set its new authority to the genesis \ authority it binds" .into(), )); } // Scope 6.3 step 3, stated once (`deadline_permits_append`) and // delegated to the frozen oracle rather than restated. This evaluation // is the one at sequencing time; the binding one runs again immediately // before the group is marked `Resolving`. deadline_permits_append(now, transaction.retry_until_micros) .map_err(|reason| reason.into_error(transaction.operation_id))?; let repo_sequence = if creates { 0 } else { before.repo_sequence.checked_add(1).ok_or_else(|| { StoreError::Corruption(format!( "repo_sequence overflow in namespace {}", namespace.to_hex() )) })? }; let mut object_ids: Vec = transaction.objects.iter().map(|o| o.id).collect(); object_ids.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes())); let commit_ids: Vec = transaction .objects .iter() .filter(|object| matches!(object.object_type, ObjectType::Commit)) .map(|object| object.id) .collect(); let mut ref_states: Vec = refs .iter() .map(|(target, object)| RefStateV1 { target: target.clone(), object: *object, }) .collect(); ref_states.sort(); let resulting_state_digest = protocol::ref_state_digest(&ref_states) .map_err(|e| StoreError::Corruption(format!("resulting state digest: {e}")))?; let committed = CommittedTransactionV1 { repo_id: ObjectId(*namespace.as_bytes()), repo_sequence, previous_event_digest: before.previous_event_digest, sequenced_at_micros: now, source_kind: transaction.evidence.source_kind(), // The evidence, never this instance's signing key. Frozen mirror // validation requires `destination_event.actor == source_instance` // for all four mirror kinds, and the administrative signing digests // bind `actor` directly, so `actor` is a property of the evidence // that this event restates. Filling it from the destination signer // produces an event that signs, fences and publishes, and only then // fails evidence verification — at a mirror, on another instance, // with the transaction already durable and unwithdrawable. actor: transaction.evidence.actor(), operation_id: *transaction.operation_id.as_bytes(), operation_digest: transaction.operation_digest, signed_evidence_digest: transaction .evidence .evidence_digest() .map_err(|e| StoreError::Corruption(format!("evidence digest: {e}")))?, old_authority: before.current_authority, new_authority, refs: applied.clone(), object_ids, commit_ids, resulting_state_digest, }; let event_digest = committed .event_digest() .map_err(|e| StoreError::Corruption(format!("event digest: {e}")))?; // Scope 6.3 step 2. The handoff is the last thing that can fail before // a sequence is consumed. match failpoints::hit(Failpoint::EvidenceHandoffFailure) { FailpointAction::Continue => {} FailpointAction::Fail => { return Err(StoreError::Signer(crate::types::SignerError::Unavailable)) } FailpointAction::Panic => { panic!("failpoint EvidenceHandoffFailure panicked the sequencer") } FailpointAction::HardExit => { unreachable!("HardExit calls _exit(3) inside failpoints::hit and never returns") } } let source_key_epoch = self.shared.signer.key_epoch(); let durability_result = DurabilityResultV1::LocallyDurable; // The signature must be the one `SignedCommittedTransactionV1::verify` // checks, so it covers that type's signing digest — which binds the // key epoch and the durability result around the canonical event — // rather than the bare event digest. A frame carrying a signature over // anything else would be durable, correctly fenced, and verifiable // against nothing. let signing_digest = SignedCommittedTransactionV1::signing_digest( &committed, source_key_epoch, &durability_result, ) .map_err(|e| StoreError::Corruption(format!("event signing digest: {e}")))?; let source_signature = self.shared.signer.sign_event(&ObjectId(signing_digest))?; let objects_new = transaction.objects.len() as u64; let receipt = CommitReceipt { operation_id: transaction.operation_id, repo_sequence, current_authority: new_authority, refs: applied, objects_new, }; Ok(Sequenced { operation_digest: transaction.operation_digest, retry_until_micros: transaction.retry_until_micros, first_visible_at_micros: now, repo_state: Arc::new(RepoState { repo_sequence, current_authority: new_authority, genesis_authority: before.genesis_authority, refs, lifecycle: before.lifecycle, storage_mode: before.storage_mode, previous_event_digest: event_digest, }), objects: transaction .objects .iter() .map(|object| (object.id, object_type_code(object.object_type))) .collect(), receipt, payload: TransactionFramePayloadV1 { repository_create: repository_create(transaction)?, objects: FrameObjectsV1::Inline( transaction .objects .iter() .map(|object| FrameObjectV1 { object_type: object.object_type, object_id: object.id, raw: object.raw.clone(), }) .collect(), ), ref_cas: transaction.refs.clone(), expected_authority, new_authority, evidence: transaction.evidence.clone(), committed: SignedCommittedTransactionV1 { transaction: committed, source_key_epoch, durability_result, source_signature, }, receipt: FrameReceiptFieldsV1 { objects_new, retry_until_micros: transaction.retry_until_micros, first_receipt_visibility_micros: now, }, }, }) } // ----------------------------------------------------------------------- // Publication: scope 6.3 steps 4 to 9 // ----------------------------------------------------------------------- fn publish_open_group(&mut self) { if self.builder.is_empty() { debug_assert!(self.pending.is_empty()); return; } // Scope 6.3 step 3, in the position the ordering actually requires: // immediately before the group is marked `Resolving`, not back at // sequencing time. Between the two sit the signer handoff and the whole // idle delay that closes an unfull group, either of which can be longer // than what remains of a caller's signed deadline. let (frames, pending) = self.close_group(); if frames.is_empty() { debug_assert!(pending.is_empty()); return; } debug_assert_eq!(frames.len(), pending.len()); // The oldest `pending.len()` waiters are this group's. A transaction // being accepted when a full group forced this publication is newer // than all of them and stays registered. debug_assert!(self.waiters.len() >= pending.len()); // Steps 4-8 are the poison window and the waiters say so *before* it is // entered, so a panic inside is classified by the phase the request is // in rather than by where the catch happens to sit. for waiter in self.waiters.iter_mut().take(pending.len()) { waiter.phase = WaiterPhase::Publishing; } let keys: Vec = pending.iter().map(|prepared| prepared.key).collect(); let outcome = self.publish_window(&frames, &pending, &keys); match outcome { Ok(()) => { // Step 8 removed the status entries, so the reservations are no // longer this waiter's to release, and the receipt is the only // outcome that remains truthful. for (prepared, waiter) in pending.iter().zip(self.waiters.iter_mut()) { waiter.reservation = None; waiter.phase = WaiterPhase::Committed(prepared.receipt.clone()); } self.wake_committed_group(pending.len()); } Err(error) => { // Scope 3.7: steps 4 to 8 are the poison window. The shard is // read-only from here, the group's status entries stay // `Resolving` because step 8 never ran, and nothing is // appended, published, or acknowledged until a close and // reopen through recovery. self.poison = Some(error.clone()); for waiter in self.waiters.drain(..pending.len()) { waiter.completion.complete(Err(error.clone())); } } } } /// Scope 6.3 steps 9 and 10, which cannot poison. /// /// The transaction is committed: the fence succeeded and the root /// published. So this has its own unwind boundary — a panic here must not /// kill the writer, because the frozen oracle says /// `later_append_allowed_before_recovery` for this row and a dead writer /// thread makes that false whatever the error says. The waiters are still /// owned by `self` while the failpoint fires, so a caught panic can still /// hand every one of them the receipt it is owed. fn wake_committed_group(&mut self, members: usize) { let hung = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { match failpoints::hit(Failpoint::AfterRootCasBeforeWaiterWake) { // Deliberately a hung request and not an error: scope 6.3 says // a waiter that never wakes is a hung request, never an absent // transaction, and the receipt stays queryable either way. FailpointAction::Fail => true, FailpointAction::Continue => false, FailpointAction::Panic => { panic!("failpoint AfterRootCasBeforeWaiterWake panicked the writer thread") } FailpointAction::HardExit => { unreachable!("HardExit calls _exit(3) inside failpoints::hit and never returns") } } })); // Charter item 6: every outcome named, no catch-all. `hung` being // `Err` is a caught panic, which is a *harder* reason to deliver the // receipt than an ordinary return, not a reason to skip it. let deliver = !matches!(hung, Ok(true)); for waiter in self.waiters.drain(..members) { match waiter.phase { WaiterPhase::Committed(receipt) => { if deliver { waiter.completion.complete(Ok(receipt)); } } WaiterPhase::PreAppend | WaiterPhase::Publishing => unreachable!( "every member of a published group was marked Committed before the wake" ), } } } /// Close the open group, applying scope 6.3 step 3 to every member. /// /// A member past its signed deadline appends nothing. Because signing /// covers `SignedCommittedTransactionV1::signing_digest`, which commits to /// the chained `previous_event_digest`, dropping one member invalidates /// every signature after it — so the retained suffix is not patched, it is /// re-derived: restored to the state the group started from and then /// re-sequenced, re-chained, re-signed, and re-encoded in order. A partial /// repair would produce a durable, correctly fenced frame carrying a /// signature that verifies against nothing, which no later recovery can /// detect. fn close_group(&mut self) -> (Vec, Vec) { let frames = self.builder.take(); let pending = std::mem::take(&mut self.pending); let base = std::mem::take(&mut self.group_base); let now = now_micros(); if pending .iter() .all(|prepared| deadline_permits_append(now, prepared.retry_until_micros).is_ok()) { return (frames, pending); } // Back to the state the group started from, so every retained member is // re-derived against exactly what it would have seen had the dropped // members never been accepted. for (namespace, before) in base { match before { Some(state) => { self.repositories.insert(namespace, state); } None => { self.repositories.remove(&namespace); } } } let base_sequence = self.journal.next_shard_sequence(); let mut new_frames = Vec::with_capacity(pending.len()); let mut new_pending: Vec = Vec::with_capacity(pending.len()); // Index into `self.waiters` of the member currently being re-derived. // The group's waiters are its oldest `pending.len()`, in order, and a // dropped member's waiter is removed at this cursor so the survivors // stay aligned with `new_pending`. let mut cursor = 0usize; for prepared in pending { let key = prepared.key; let transaction = Arc::clone(&prepared.transaction); let next_sequence = base_sequence + new_pending.len() as u64; let outcome = deadline_permits_append(now, prepared.retry_until_micros) .map_err(|reason| reason.into_error(key.operation_id)) .and_then(|()| self.sequence_into_frame(&transaction, next_sequence, now)); match outcome { Ok((reprepared, frame)) => { new_frames.push(frame); new_pending.push(reprepared); cursor += 1; } Err(error) => { // Pre-append and definitive: no sequence was consumed, the // reservation goes, and the caller learns its transaction // is absent rather than waiting on a frame that will never // be written. self.release_reservation(&key); let waiter = self.waiters.remove(cursor); waiter.completion.complete(Err(error)); } } } // `sequence_into_frame` repopulated the base for a group that is // closing right now; the next group starts from a clean one. self.group_base.clear(); (new_frames, new_pending) } /// Scope 6.3 steps 4 through 8. Every failure inside poisons the shard. fn publish_window( &mut self, frames: &[Frame], pending: &[Prepared], keys: &[OperationKey], ) -> Result<(), StoreError> { // The journal's sequence counter advances here and nowhere else. // `prepare` computes a member's position but consumes nothing, because // the pre-mark deadline recheck above may drop members and a consumed // sequence that never appends is a hole that poisons the shard. By this // point the retained set is final, so consuming is safe and the // assertion below is what proves the two agree. for frame in frames { let assigned = self.journal.assign_shard_sequence(); if assigned != frame.header.shard_sequence { return Err(self.poison_error(format!( "the journal assigned shard_sequence {assigned} where the closed group \ carried {}", frame.header.shard_sequence ))); } } // --- step 4: mark every operation in the group Resolving ----------- self.mark_resolving(pending)?; fire_in_poison_window(self.shard_index, Failpoint::AfterMarkedResolving)?; // --- step 5: exact reserved frames, one fence ---------------------- let sequences = self.journal.append_group_and_fence(frames)?; if sequences.len() != pending.len() { return Err(self.poison_error(format!( "append returned {} sequences for a group of {}", sequences.len(), pending.len() ))); } // --- step 6: build the immutable shard subtree --------------------- fire_in_poison_window(self.shard_index, Failpoint::DuringCommittedRootBuild)?; let subtree = self.build_subtree(pending)?; fire_in_poison_window( self.shard_index, Failpoint::AllocationFailureBeforePublication, )?; // --- step 7: CAS-publish the committed root ------------------------ fire_in_poison_window(self.shard_index, Failpoint::BeforeRootCas)?; self.publish_subtree(&subtree)?; #[cfg(test)] probe::fire(&self.shared, keys); // --- step 8: remove the group's status entries --------------------- // // Strictly after step 7 and never before. `ArcSwap`'s store is a // release operation and its load is an acquire, so a reader that // observes the status root without these entries has synchronized // with this removal and cannot then read a committed root older than // the publication above. The explicit fence makes the dependency // between the two swaps a statement rather than an inherited property // of the crate that implements them. fence(Ordering::Release); self.remove_status_entries(keys); Ok(()) } fn mark_resolving(&self, pending: &[Prepared]) -> Result<(), StoreError> { loop { let current = self.shared.status.load(); let mut next = OperationStatusRoot::clone(¤t); for prepared in pending { next = next .replace_existing( prepared.key, StatusEntry::resolving( prepared.operation_digest, prepared.retry_until_micros, Some(prepared.shard_sequence), ), ) .ok_or_else(|| { // The reservation was made by this thread before // sequencing and nothing else may remove or rebind it. self.poison_error(format!( "operation {} lost its status reservation before append", prepared.key.operation_id.to_hex() )) })?; } let next = Arc::new(next); // The root that was just published, never the one it replaced. // Recording `current` makes occupancy lag every insertion and every // removal by one publication, which is a metric that is wrong in // exactly the direction that hides an approach to the ceiling. let previous = self .shared .status .compare_and_swap(¤t, Arc::clone(&next)); if Arc::ptr_eq(&previous, ¤t) { self.shared.status_metrics.record_published(&next); return Ok(()); } } } fn build_subtree(&self, pending: &[Prepared]) -> Result { let mut delta = IndexDelta::from_options(&self.shared.options); // The device offsets the fence just made durable, read back from the // journal rather than predicted, so an index location can only ever // name bytes that are actually there. let index = self.journal.frame_index(); let appended = &index[index.len() - pending.len()..]; for (prepared, (sequence, offset, len)) in pending.iter().zip(appended) { if *sequence != prepared.shard_sequence { return Err(self.poison_error(format!( "journal recorded sequence {sequence} where the group carried {}", prepared.shard_sequence ))); } let frame_len = u32::try_from(*len).map_err(|_| StoreError::LimitExceeded { limit: "frame_len", observed: *len, allowed: u64::from(u32::MAX), })?; for (object, object_type) in &prepared.objects { delta.insert( IndexKey::new(prepared.key.namespace, *object), IndexLocation { segment_generation: self.tail_generation, frame_offset: *offset, frame_len, object_type: *object_type, shard_sequence: *sequence, }, )?; } } let mut repositories = RepoMap::new(); let mut terminal_statuses = TerminalStatusMap::new(); for prepared in pending { // In group order, so a group touching one repository twice leaves // the later state. repositories.insert(prepared.key.namespace, Arc::clone(&prepared.repo_state)); terminal_statuses.insert( prepared.key, TerminalStatusEntry::Committed(RetainedReceipt { operation_digest: prepared.operation_digest, receipt: prepared.receipt.clone(), shard_sequence: prepared.shard_sequence, retry_until_micros: prepared.retry_until_micros, first_visible_at_micros: prepared.first_visible_at_micros, // Computed by the frozen protocol function, never // restated: the store and the wire must not be able to // disagree about when a receipt stops being visible. receipt_visible_until_micros: protocol::receipt_visible_until( prepared.retry_until_micros, prepared.first_visible_at_micros, self.shared.options.terminal_status_grace_micros, ) .map_err(|e| StoreError::Corruption(format!("receipt visibility: {e}")))?, }), ); } let last = pending .last() .map(|prepared| prepared.shard_sequence) .ok_or_else(|| self.poison_error("published an empty group".into()))?; Ok(ShardSubtree::new( self.shard_index, last, Arc::new(delta), None, Vector::new(), repositories, terminal_statuses, // No new generation: the active journal these frames landed in is // already pinned by the generation recovery installed. GenerationMap::new(), )) } /// This shard's unsealed index backlog, as one captured root reports it. fn unsealed_backlog(&self, root: &CommittedRoot) -> UnsealedBacklog { let mut backlog = UnsealedBacklog::default(); for layer in root.index().delta_layers() { if layer.shard_index != self.shard_index { continue; } backlog.layers += 1; backlog.entries += layer.delta.len(); // Summed rather than deduplicated: two layers holding the same key // are counted twice, so the total is an over-estimate and the // decision it feeds can only ever seal *early*. Deduplicating would // mean merging every layer on every admission. backlog.encoded_bytes += layer.delta.encoded_bytes(); backlog.through_shard_sequence = backlog .through_shard_sequence .max(layer.through_shard_sequence); } backlog } /// Index entries a reopen of this shard would have to rebuild. /// /// Sealed runs plus the unsealed backlog, because with the committed prefix /// unadvanced every frame is still replayed. Counted from the shard's own /// retained generations rather than from the root's run vector, which is not /// tagged by shard and would charge one shard for another's runs. fn replayable_index_entries(&self, root: &CommittedRoot, backlog: &UnsealedBacklog) -> u64 { let sealed: u64 = root .retained_generations() .values() .filter(|generation| generation.id.shard_index == self.shard_index) .flat_map(|generation| generation.index_runs.iter()) .map(|retained| retained.run().entry_count()) .sum(); sealed + backlog.entries } /// Seal when the accumulated delta has reached a hard ceiling (scope 3.6). /// /// Two ceilings, because the backlog has two costs. `delta_pressure` answers /// for entries and bytes — the memory the unsealed delta occupies — through /// the same function `IndexDelta::pressure` uses, so the watermark has one /// implementation. The layer *count* is the other cost: every layer is a map /// a lookup consults before it reaches any run, so a million single-object /// groups would leave lookup fan-out unbounded while entry pressure stayed /// low. `max_index_runs` bounds it, which is the ceiling the refusal this /// replaced already used for exactly that reason. fn seal_index_if_required(&mut self) -> Result<(), StoreError> { let root = self.shared.committed.load(); let backlog = self.unsealed_backlog(&root); if backlog.layers == 0 { return Ok(()); } let entries_or_bytes = delta_pressure( backlog.entries, backlog.encoded_bytes, self.shared.options.max_active_index_entries, self.shared.options.max_active_index_bytes, ) == DeltaPressure::SealRequired; let fan_out = backlog.layers as u64 >= u64::from(self.shared.options.max_index_runs); if entries_or_bytes || fan_out { self.seal_index(&root, backlog.through_shard_sequence)?; } // What a reopen would have to rebuild, checked **after** the seal so a // seal that was due still happens. // // Sealing moves entries out of the delta layers and into a run, but it // does not move a single *frame* out of `active/`. The manifest's // `committed_shard_sequence` advances only when a checkpoint or a segment // rotation makes a prefix durable elsewhere, and neither happens in this // slice — so recovery replays every frame this shard ever wrote, into one // delta bounded by `max_active_index_entries`. A shard that kept // accepting work past that point would be writing a store it could not // reopen, and would find out at the next open. // // Refusing makes that a refusal instead of a corrupt outcome, and it is // not conservative padding: recovery's delta holds at most // `max_active_index_entries`, and every admitted transaction adds at // least one object. // // The consequence is worth stating plainly, because it bounds what this // slice delivers: a seal triggered by **entry pressure** lands the shard // exactly on this ceiling, so the next admission is refused. Only a seal // triggered by the **fan-out** ceiling leaves the shard able to continue. // Entry-pressure sealing becomes useful when `StoreEngine::checkpoint` // can advance the committed prefix; until then it converts an // unreopenable store into an honest refusal, which is all it can do. let backlog = self.unsealed_backlog(&self.shared.committed.load()); let replayable = self.replayable_index_entries(&self.shared.committed.load(), &backlog); if replayable >= self.shared.options.max_active_index_entries { return Err(StoreError::LimitExceeded { limit: "max_active_index_entries", observed: replayable + 1, allowed: self.shared.options.max_active_index_entries, }); } Ok(()) } /// Seal this shard's covered delta layers into one durable run and publish /// it, discarding exactly what the run covers in the same CAS. /// /// # The poison boundary /// /// Everything before [`segment::install_index_run`] is pre-durable: the /// ceiling checks, the generation lookup, the merge and the encode all fail /// without having changed a byte on the device, so they are ordinary errors /// and the shard stays usable. From that call onward the shard poisons on any /// failure, including a failure the call may have made *before* writing /// anything — the caller cannot tell those apart, and the conservative /// direction is the one that refuses to keep writing against a root that may /// no longer describe the device. A false poison costs an operator a /// recovery; a missed one lets the writer continue on an ambiguous root, /// which is the failure this boundary exists to prevent. fn seal_index(&mut self, root: &CommittedRoot, covered_through: u64) -> Result<(), StoreError> { // Both ceilings, before anything is durable, and against the *same* // numbers recovery enforces when it reopens the manifest (scope 3.8). A // seal that passed here can always be reopened; one that bypassed them // would install a manifest this store could never open again. let projected = root.index().sealed_run_count() as u64 + 1; for (limit, allowed) in [ ("max_index_runs", self.shared.options.max_index_runs), ( "max_open_index_runs", self.shared.options.max_open_index_runs, ), ] { if projected > u64::from(allowed) { return Err(StoreError::LimitExceeded { limit, observed: projected, allowed: u64::from(allowed), }); } } // Newest-first, skipping any key a newer layer already answered: that is // `LayeredObjectIndex::lookup`'s first-hit-wins rule, so the run answers // every covered key exactly as the layers did. Merging oldest-first would // instead meet `IndexDelta::insert`'s conflict check the moment one object // had been rewritten at a new offset — a legitimate history that would // then fail to seal. // // The merge target's ceilings are relaxed on purpose. Its contents are // already resident in the layers being merged, so the bound that matters // was applied when those were admitted; measuring the accumulation // against the configured ceiling *here* would refuse the very seal that // relieves it. let mut merged = IndexDelta::new(u64::MAX, u64::MAX); for layer in root.index().delta_layers() { if layer.shard_index != self.shard_index || layer.through_shard_sequence > covered_through { continue; } for (key, location) in layer.delta.iter() { if merged.get(&key).is_none() { merged.insert(key, location)?; } } } let current_generation = root .retained_generations() .keys() .filter(|id| id.shard_index == self.shard_index) .map(|id| id.manifest_generation) .max() .ok_or_else(|| { // Not a poison: nothing has been written, and wedging the shard // for a condition that only recovery can have produced would // turn a diagnosable state into an outage. StoreError::Corruption(format!( "shard {} has no retained manifest generation to seal an index run against", self.shard_index )) })?; let next_generation = current_generation + 1; let root_uuid = self.shared.root_uuid; let run_bytes = IndexRunBuilder::new(root_uuid, next_generation, next_generation) .build(&merged) .map_err(|error| { StoreError::Corruption(format!("encoding index run {next_generation}: {error}")) })?; let paths = RootLayout::new(&self.shared.options.root).shard(self.shard_index); let manifest_retain = self.shared.options.manifest_retain; let counters = Arc::clone(self.journal.counters_handle()); // --- durable from here: every failure below poisons ---------------- let sealed = segment::install_index_run( &paths, &root_uuid, current_generation, next_generation, &run_bytes, manifest_retain, &counters, ); let sealed = match sealed { Ok(sealed) => sealed, Err(error) => return Err(self.poison_now(format!("installing index run: {error}"))), }; // Reopened from the device rather than kept from the bytes just encoded. // The run's checksum, header and `root_uuid` are validated by the same // code recovery will use, so a run that cannot be reopened is found now — // while the shard can still poison — instead of at the next open. let run = match IndexRun::open(&sealed.path, &root_uuid) { Ok(run) => Arc::new(run), Err(error) => return Err(self.poison_now(format!("reopening a sealed run: {error}"))), }; // The successor generation pins what the predecessor pinned plus the new // run, and the predecessor's pin is released in the same publication. // Copying rather than referencing keeps `object_source`'s rule satisfied: // one logical generation may appear in several manifest generations, but // every occurrence must name the same path. let previous = root .retained_generations() .get(&GenerationId::new(self.shard_index, current_generation)) .map(Arc::clone); let previous = match previous { Some(previous) => previous, None => { return Err(self.poison_now(format!( "generation {current_generation} vanished from the root while sealing" ))) } }; let mut index_runs = previous.index_runs.to_vec(); index_runs.push(RetainedIndexRun::new(sealed.path.clone(), Arc::clone(&run))); let successor = Arc::new(RetainedGeneration::new( GenerationId::new(self.shard_index, next_generation), Arc::clone(&previous.segments), index_runs.into(), Arc::clone(&previous.checkpoints), Arc::clone(&previous.active_tails), Arc::clone(&previous.projection_artifacts), )); let mut generations = GenerationMap::new(); generations.insert(successor.id, successor); let mut sealed_runs = Vector::new(); sealed_runs.push_back(run); let mut subtree = ShardSubtree::new( self.shard_index, // No frame was appended, so the shard's committed sequence is // unchanged. `CommittedRoot::merge` recognizes a maintenance // publication and does not treat the unchanged sequence as a // duplicate group (contract review 2026-07-29-C, amendment 3). root.shard_committed_sequence(self.shard_index) .unwrap_or(covered_through), Arc::new(IndexDelta::from_options(&self.shared.options)), Some(covered_through), sealed_runs, RepoMap::new(), TerminalStatusMap::new(), generations, ); subtree .retained_generation_removals .push_back(GenerationId::new(self.shard_index, current_generation)); if let Err(error) = self.publish_subtree(&subtree) { return Err(self.poison_now(format!("publishing a sealed index run: {error}"))); } Ok(()) } fn publish_subtree(&self, subtree: &ShardSubtree) -> Result<(), StoreError> { loop { let current = self.shared.committed.load(); #[cfg(test)] probe::fire_cas_contention(&self.shared); let merged = Arc::new(current.merge(subtree)); let previous = self.shared.committed.compare_and_swap(¤t, merged); if Arc::ptr_eq(&previous, ¤t) { return Ok(()); } // Another shard published between the load and the swap. Merge // again against the newer root; the subtree is unchanged and // `merge` is pure, so this is a re-merge and never a lost update. #[cfg(test)] self.shared .root_cas_retries .fetch_add(1, std::sync::atomic::Ordering::Relaxed); fire_in_poison_window(self.shard_index, Failpoint::DuringRootCasRetry)?; } } fn remove_status_entries(&self, keys: &[OperationKey]) { loop { let current = self.shared.status.load(); let next = Arc::new(current.without_all(keys)); let previous = self .shared .status .compare_and_swap(¤t, Arc::clone(&next)); if Arc::ptr_eq(&previous, ¤t) { self.shared.status_metrics.record_published(&next); return; } } } // ----------------------------------------------------------------------- // Status-root reservations // ----------------------------------------------------------------------- fn reserve( &mut self, key: OperationKey, transaction: &ValidatedTransaction, now: i64, ) -> Result<(), StoreError> { let entry = StatusEntry::pending( transaction.operation_digest, transaction.retry_until_micros, PendingPhase::Sequenced, ); loop { let current = self.shared.status.load(); // Decision 9.8: identity is examined before capacity, and the // bound is enforced atomically with the insert by retrying this // whole reconsideration on CAS failure. match current.reserve(key, entry) { StatusReservation::Attached(_) => { return Err(StoreError::NotImplemented( "same-ID/same-digest coalescing onto an in-flight leader — B1 \ NamespaceTxn, scope 6-B1 deliverable 7", )) } StatusReservation::OperationIdMismatch { .. } => { return Err(StoreError::Conflict(format!( "operation {} is already in flight with a different stable digest", key.operation_id.to_hex() ))) } StatusReservation::AtCapacity { .. } => { // Decision 9.8: the bound is enforced by refusal, never by // eviction. Evicting a `Pending` entry would silently turn // it into `Unknown`, which is a correctness change wearing // a capacity policy's clothes. self.shared.status_metrics.record_rejection(); return Err(StoreError::Overloaded { limit: "max_status_entries", retry_after_micros: retry_after_micros( transaction.retry_until_micros, now, &self.shared.options, ), }); } StatusReservation::Inserted(next) => { let next = Arc::new(next); let previous = self .shared .status .compare_and_swap(¤t, Arc::clone(&next)); if Arc::ptr_eq(&previous, ¤t) { self.shared.status_metrics.record_published(&next); // Owned by the writer until it is handed to a waiter or // released, so a panic anywhere in steps 1-3 has exactly // one place to look for a reservation to drop. self.in_flight_reservation = Some(key); return Ok(()); } } } } } /// Drop a reservation for a transaction that will definitively never /// append. Only legal before step 5; after it, a status entry is removed /// exactly once, by step 8. fn release_reservation(&mut self, key: &OperationKey) { loop { let current = self.shared.status.load(); let next = Arc::new(current.without(key)); let previous = self .shared .status .compare_and_swap(¤t, Arc::clone(&next)); if Arc::ptr_eq(&previous, ¤t) { self.shared.status_metrics.record_published(&next); if self.in_flight_reservation == Some(*key) { self.in_flight_reservation = None; } return; } } } fn poison_error(&self, cause: String) -> StoreError { StoreError::ShardPoisoned { shard: self.shard_index, cause, } } /// Poison the shard **and** latch it, for a failure outside the publication /// window that reaches this file's other durable sequence. /// /// `poison_error` only builds the error. That is right inside /// `publish_window`, whose caller latches `self.poison` for the whole group; /// it is wrong for an index seal, which runs at admission and whose error /// otherwise returns to one caller and leaves the writer accepting the next /// transaction against a root that may no longer describe the device. fn poison_now(&mut self, cause: String) -> StoreError { let error = self.poison_error(cause); self.poison = Some(error.clone()); error } } /// A [`Prepared`] that still carries its undecoded payload. struct Sequenced { operation_digest: ObjectId, retry_until_micros: i64, first_visible_at_micros: i64, repo_state: Arc, objects: Vec<(ObjectId, u8)>, receipt: CommitReceipt, payload: TransactionFramePayloadV1, } /// Why scope 6.3 step 3 refused an append. /// /// Named rather than inlined because the rule is evaluated at two points — at /// sequencing, and again immediately before the group is marked `Resolving` — /// and the two must be the same rule. Two copies of a deadline test are two /// opinions about when a signed receipt expires. enum DeadlineRefusal { Expired, AlreadyAppending, } impl DeadlineRefusal { fn into_error(self, operation: OperationId) -> StoreError { match self { Self::Expired => StoreError::Conflict(format!( "operation {} passed its signed deadline before append", operation.to_hex() )), Self::AlreadyAppending => StoreError::Corruption( "pre-append deadline check reported an append that had already started".into(), ), } } } /// Scope 6.3 step 3, delegated to the frozen oracle rather than restated. /// /// Plan §7 stage 9: no operation may first append after its signed deadline. fn deadline_permits_append(now: i64, retry_until_micros: i64) -> Result<(), DeadlineRefusal> { let deadline = oracle::deadline_expectation(now, retry_until_micros, AppendDeadlinePhase::PreAppend); match deadline.decision { DeadlineDecision::ContinuePreAppend => Ok(()), DeadlineDecision::RejectReceiptExpired => Err(DeadlineRefusal::Expired), DeadlineDecision::RemainResolving => Err(DeadlineRefusal::AlreadyAppending), } } /// Fire a failpoint inside the poison window. Every action is enumerated, so a /// new one cannot be added without a decision here. fn fire_in_poison_window(shard: u16, point: Failpoint) -> Result<(), StoreError> { match failpoints::hit(point) { FailpointAction::Continue => Ok(()), FailpointAction::Fail => Err(StoreError::ShardPoisoned { shard, cause: format!("failpoint {} inside the publication window", point.name()), }), FailpointAction::Panic => { panic!("failpoint {} panicked the writer thread", point.name()) } FailpointAction::HardExit => { unreachable!("HardExit calls _exit(3) inside failpoints::hit and never returns") } } } fn repository_create( transaction: &ValidatedTransaction, ) -> Result, StoreError> { let Some(genesis) = transaction.create_genesis_authority else { return Ok(None); }; // The builder already refused a create whose genesis object is absent, so // this is the second half of that check rather than a new one: the exact // length and hash come from the bytes the transaction carries. let object = transaction .objects .iter() .find(|object| object.id == genesis) .ok_or_else(|| { StoreError::Corruption( "a repository-create transaction reached the sequencer without its genesis \ authority object" .into(), ) })?; Ok(Some(RepositoryCreateV1 { genesis_authority: genesis, genesis_len: object.raw.len() as u64, genesis_hash: object.id, projection: protocol::ProjectionMode::Full, })) } /// How long an overloaded caller should wait. /// /// Bounded by what remains of the caller's own signed deadline: telling a /// caller to retry after its receipt expires is telling it to stop. fn retry_after_micros(retry_until_micros: i64, now: i64, options: &StoreOptions) -> u64 { let idle = options .max_group_idle .as_micros() .try_into() .unwrap_or(i64::MAX); let remaining = retry_until_micros.saturating_sub(now).max(0); u64::try_from(idle.min(remaining.max(0))).unwrap_or(0) } fn now_micros() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) .map(|elapsed| elapsed.as_micros() as i64) .unwrap_or(0) } /// A hook fired at exactly one place: after the committed root is published /// and before the status entries are removed. /// /// It exists because the scope 6.3 step 7/8 ordering is otherwise only /// observable as a race. A concurrent reader can prove the property /// statistically, and a test over a helper would prove nothing about the path /// that runs. Firing here lets a test assert, deterministically and on the /// real writer, that at this instant the receipt is already in the committed /// root *and* the status entry is still present. Moving the removal before the /// publication fails the first half; moving the publication after the removal /// fails the second. /// /// `cfg(test)` only: the shipping writer has no hook, no atomic load, and no /// indirect call here. #[cfg(test)] mod probe { use std::sync::{Arc, Mutex, MutexGuard}; use super::{EngineShared, OperationKey}; type Hook = Arc; type CasHook = Arc; static SERIAL: Mutex<()> = Mutex::new(()); static INSTALLED: Mutex> = Mutex::new(None); static CAS_CONTENTION: Mutex> = Mutex::new(None); /// Exclusive use of the process-global hook, cleared on acquire. /// /// Takes the same shape as `sys::FaultSerial` and for the same reason: a /// lock a test *may* take is a lock a new test will not take, and a test /// that panics cannot run its own cleanup. pub(super) struct ProbeSerial(#[allow(dead_code)] MutexGuard<'static, ()>); pub(super) fn serial() -> ProbeSerial { let guard = SERIAL .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); *INSTALLED .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) = None; *CAS_CONTENTION .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) = None; ProbeSerial(guard) } /// Install a competing publisher that runs between the committed root's /// load and its compare-and-swap. /// /// Contention on that CAS is otherwise only reachable as a race, and a test /// that waits for one either flakes or asserts nothing. This runs inside /// the real `publish_subtree` loop — the production path — and only the /// source of the contention is synthetic. pub(super) fn install_cas_contention( _serial: &ProbeSerial, hook: impl Fn(&EngineShared) + Send + Sync + 'static, ) { *CAS_CONTENTION .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(Arc::new(hook)); } pub(super) fn fire_cas_contention(shared: &EngineShared) { let hook = CAS_CONTENTION .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) .clone(); if let Some(hook) = hook { hook(shared); } } pub(super) fn install( _serial: &ProbeSerial, hook: impl Fn(&EngineShared, &[OperationKey]) + Send + Sync + 'static, ) { *INSTALLED .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(Arc::new(hook)); } pub(super) fn fire(shared: &EngineShared, keys: &[OperationKey]) { let hook = INSTALLED .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) .clone(); if let Some(hook) = hook { hook(shared, keys); } } } #[cfg(test)] impl StoreEngine { /// The published committed root, for assertions the reader API cannot yet /// make: `RepoSnapshot` is B1 deliverable 8. This is the same immutable /// value every reader captures, not a private mirror of it. fn committed_root(&self) -> arc_swap::Guard> { self.shared.committed.load() } } #[cfg(test)] // `WriterSerial` is `crate::sys::FaultSerial` with the `failpoints` feature and // `()` without it, so the token every test binds is a unit value in the default // configuration. Binding it anyway is the point: the same test source must take // the lock in the configuration where the lock exists. #[allow(clippy::let_unit_value)] mod tests { use std::future::Future; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; use std::sync::Mutex; use std::task::{Context, Poll, Wake, Waker}; use std::time::Duration; use levcs_protocol::v2::{RefTarget, TransactionEvidenceV1, TypedRefCas}; use super::*; use crate::index::IndexKey; use crate::segment; use crate::transaction::StagedObject; use crate::types::{PrivilegedConstruction, SignerError}; // ----------------------------------------------------------------------- // Harness // ----------------------------------------------------------------------- /// A deterministic stand-in for the injected signer. /// /// It is not Ed25519, and it does not need to be: the store never verifies /// a signature (plan §5.1 forbids it from deciding who may sign). What the /// tests need is a signature that is a pure function of the digest it was /// asked to cover, so a test can recompute what *should* have been signed /// and prove the writer signed that and not something else. #[derive(Default)] struct TestSigner { calls: AtomicU64, } impl TestSigner { fn expected_signature(digest: &ObjectId) -> [u8; 64] { let mut signature = [0u8; 64]; let first = blake3::hash(digest.as_bytes()); let second = blake3::hash(first.as_bytes()); signature[..32].copy_from_slice(first.as_bytes()); signature[32..].copy_from_slice(second.as_bytes()); signature } } impl CommitEvidenceSigner for TestSigner { fn key_epoch(&self) -> u64 { 7 } fn public_key(&self) -> [u8; 32] { [0x5a; 32] } fn sign_event(&self, event_digest: &ObjectId) -> Result<[u8; 64], SignerError> { self.calls.fetch_add(1, AtomicOrdering::SeqCst); Ok(Self::expected_signature(event_digest)) } } /// Drive one future to completion on this thread. /// /// `levcs-store` starts no runtime, so its tests must not depend on one /// either. pub(super) fn block_on(future: F) -> F::Output { match block_on_until(future, Duration::from_secs(30)) { Some(output) => output, None => panic!("future did not complete within the test deadline"), } } /// Drive one future until it completes or the deadline passes. /// /// `None` is a real answer here rather than a failure: scope 6.3 says a /// failure to wake a waiter leaves a hung request, and a test that asserts /// that has to be able to observe the hang. fn block_on_until(future: F, deadline: Duration) -> Option { struct ThreadWaker(std::thread::Thread); impl Wake for ThreadWaker { fn wake(self: Arc) { self.0.unpark(); } fn wake_by_ref(self: &Arc) { self.0.unpark(); } } let waker = Waker::from(Arc::new(ThreadWaker(std::thread::current()))); let mut context = Context::from_waker(&waker); let mut future = std::pin::pin!(future); let start = std::time::Instant::now(); loop { match future.as_mut().poll(&mut context) { Poll::Ready(output) => return Some(output), Poll::Pending => { let elapsed = start.elapsed(); if elapsed >= deadline { return None; } std::thread::park_timeout(deadline - elapsed); } } } } /// Exclusive use of the process-global failpoint and fault registries. /// /// Every helper that can produce a `StoreOptions` takes one, so a test /// cannot reach the writer without holding it. That is deliberate and it /// is the Wave A `recovery_eio.rs` lesson applied one layer up: the /// registries are one-shot and global, so a test that merely *submits* can /// consume a failpoint another test armed. A comment cannot fail a build; /// a parameter can. #[cfg(feature = "failpoints")] pub(super) type WriterSerial = crate::sys::FaultSerial; #[cfg(not(feature = "failpoints"))] pub(super) type WriterSerial = (); pub(super) fn writer_serial() -> WriterSerial { #[cfg(feature = "failpoints")] { crate::sys::serial() } } fn counters() -> Arc { Arc::new(DurabilityCounters::default()) } pub(super) fn options(_serial: &WriterSerial, root: &Path, shard_count: u16) -> StoreOptions { let mut options = StoreOptions::new(root); options.shard_count = shard_count; options.max_group_transactions = 4; options.max_group_bytes = 64 * 1024; options.max_group_idle = Duration::from_millis(50); options.journal_preallocate_bytes = 4 * 1024 * 1024; options.signer = Some(Arc::new(TestSigner::default())); options } /// Every writer test's root, built by the entry point a consumer calls. /// /// This used to seed the root with `segment::initialize_root` and then /// open it, because `StoreEngine::open` refused startup state 1. That was /// the charter item 8 weakening at the heart of this deliverable: the /// production `submit` was being exercised over a root production could /// not create. With state 1 implemented there is nothing left to seed — /// the first call initializes and every later call on the same path is /// state 2, which is also what makes the reopen tests below reopens rather /// than re-seedings. fn open(serial: &WriterSerial, root: &Path, shard_count: u16) -> StoreEngine { StoreEngine::open(options(serial, root, shard_count)) .expect("open initializes an absent root and reopens a formatted one") } /// The client principal on whose behalf a transaction is submitted. /// /// Deliberately **not** `TestSigner`'s public key. The two were the same /// value, and `CommittedTransactionV1.actor` was being filled from the /// destination signer rather than from the evidence — a defect no assertion /// in this file could see, because every value it could have been was /// `[0x5a; 32]`. Frozen mirror validation requires /// `destination_event.actor == source_instance`, so the substitution /// produces an event that signs, fences, and publishes and only then fails /// verification at a mirror on another instance. const EVIDENCE_ACTOR: [u8; 32] = [0x7e; 32]; fn evidence() -> TransactionEvidenceV1 { TransactionEvidenceV1::AdministrativeV1 { actor: EVIDENCE_ACTOR, actor_key_epoch: 7, command_digest: ObjectId([0x33; 32]), signature: [0x44; 64], } } fn deadline() -> i64 { now_micros() + 600_000_000 } pub(super) fn genesis_object() -> StagedObject { StagedObject { id: ObjectId([0xa1; 32]), object_type: ObjectType::Authority, raw: vec![0xa1; 32], } } pub(super) fn create_transaction( namespace: NamespaceId, operation: u8, ) -> ValidatedTransaction { let genesis = genesis_object(); let id = genesis.id; ValidatedTransaction::builder(PrivilegedConstruction::internal()) .namespace(namespace) .operation( OperationId([operation; 16]), ObjectId([operation; 32]), deadline(), ) .create_repository(id) .objects(vec![genesis]) .refs(Vec::new()) .authority(None, Some(id)) .evidence(evidence()) .build() .expect("a complete create transaction") } /// A push carrying one blob and, optionally, one typed ref update. pub(super) fn push_transaction( namespace: NamespaceId, operation: u8, blob: u8, reference: Option, ) -> ValidatedTransaction { let authority = genesis_object().id; ValidatedTransaction::builder(PrivilegedConstruction::internal()) .namespace(namespace) .operation( OperationId([operation; 16]), ObjectId([operation; 32]), deadline(), ) .objects(vec![StagedObject { id: ObjectId([blob; 32]), object_type: ObjectType::Blob, raw: vec![blob; 64], }]) .refs(reference.into_iter().collect()) .authority(Some(authority), Some(authority)) .evidence(evidence()) .build() .expect("a complete push transaction") } fn set_branch(name: &str, expected: Option, target: ObjectId) -> TypedRefCas { TypedRefCas { target: RefTarget::Branch(name.into()), expected, mutation: RefMutation::Set(target), force: false, } } // ----------------------------------------------------------------------- // Startup // ----------------------------------------------------------------------- /// Deliverable 1's central claim: one session, held from before the first /// shard's recovery until the engine is dropped. /// /// The lock is the observable. If `open` took and released a session per /// shard, or dropped it once recovery finished, a second `open` would /// succeed here — and a second process entering a recovered-but-not-yet- /// ready root is exactly what the single session exists to prevent. #[test] fn one_retained_session_recovers_every_shard_and_holds_the_root_lock() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let engine = open(&serial, temporary.path(), 4); assert!(matches!( StoreEngine::open(options(&serial, temporary.path(), 4)), Err(StoreError::AlreadyLocked) )); // Every shard was recovered, not only the one a transaction happens to // route to: each contributes its committed sequence and its retained // generation to the first published root. let root = engine.committed_root(); assert_eq!(root.retained_generations().len(), 4); drop(engine); StoreEngine::open(options(&serial, temporary.path(), 4)) .expect("the lock is released when the engine is dropped"); } // ----------------------------------------------------------------------- // Plan §5.2's startup states, one test per state // // Charter item 8: every one of these asserts against `StoreEngine::open`, // the entry point a consumer calls, and never against `classify_root` or // `segment::initialize_root` directly. A root that only `initialize_root` // can create is a root the production path has never been shown to build. // ----------------------------------------------------------------------- /// A recursive, order-independent image of a directory tree: relative /// path, whether it is a directory, and the exact bytes of every file. /// /// This is what "byte-identical after the refusal" is asserted with. It /// deliberately captures *contents* rather than mtimes: an mtime can /// change for reasons outside this process, and a test that flakes on it /// would be retried rather than believed. What it must catch is a probe /// that created a directory, truncated a file, or appended a byte, and it /// catches all three including the empty-directory case, which a /// contents-only digest would miss. fn tree_image(root: &Path) -> Vec<(PathBuf, bool, Vec)> { fn walk(base: &Path, dir: &Path, out: &mut Vec<(PathBuf, bool, Vec)>) { let mut entries: Vec<_> = std::fs::read_dir(dir) .expect("read a directory of the image") .map(|entry| entry.expect("a directory entry").path()) .collect(); entries.sort(); for path in entries { let relative = path .strip_prefix(base) .expect("every walked path is under the base") .to_path_buf(); if path.is_dir() { out.push((relative, true, Vec::new())); walk(base, &path, out); } else { out.push((relative, false, std::fs::read(&path).expect("read a file"))); } } } let mut image = Vec::new(); if root.exists() { walk(root, root, &mut image); } image } /// State 1, absent path. The root's *name* does not exist when `open` is /// called, so this also covers the case in which initialization has to /// make a new directory entry durable in the parent. #[test] fn startup_state_1_initializes_an_absent_root_through_open() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let root = temporary.path().join("not").join("yet").join("a-root"); assert!(!root.exists()); let engine = StoreEngine::open(options(&serial, &root, 3)) .expect("startup state 1 initializes an absent root"); // Charter item 7: not "it must have fsynced", but how many times, and // for which entry. The total is stated as its parts so that a change to // any one of them fails here with its name attached. // // `segment::initialize_root` costs six directories per shard, each // fsynced as it is created, plus the shard directory a second time once // its children are durable — seven per shard — and then shards/, // quarantine/, the root, and the root's parent. `staging/` is created // and not separately fsynced, which is correct rather than missing: it // is empty, so the only thing that has to become durable about it is // its name in the root, and the root fsync is what covers that. let inside_initialize_root = 3 * 7 + 4; // The path is `/not/yet/a-root` and none of the three exist, so // `open` creates three directory entries before it can take the root // lock — and fences each one by fsyncing the directory that names it: // `yet` (which names `a-root`), `not` (which names `yet`), and the // tempdir (which names `not`), in that order. // // This is the assertion the P1-1 finding turns on. `initialize_root` // fsyncs `root.parent()` and nothing above it, so before the fix this // number was zero and a power loss could take `not` or `yet` — and the // root with them — from a store that had already reported itself // initialized. let created_ancestors = 3; // The `INITIALIZING` marker: one fsync of the root to make its name // durable before the tree is built, and one more to make its removal // durable after `FORMAT` is installed. let initialization_marker = 2; let initialization = engine .shared .initialization .expect("state 1 records what it cost"); assert_eq!( initialization.fdatasync, 2, "the INITIALIZING marker and FORMAT are each fenced exactly once" ); assert_eq!( initialization.fsync_dir, created_ancestors + initialization_marker + inside_initialize_root, "every created ancestor, the initialization marker's install and removal, \ every created directory, shards/, quarantine/, the root, and its parent" ); assert_eq!(initialization.short_writes, 0); assert!( !root.join("INITIALIZING").exists(), "a finished initialization leaves no marker behind" ); // The root it built is the root the production path then opens: the // frozen topology is the configured one, and it is readable through // `FORMAT` rather than assumed. assert_eq!(engine.shared.shard_count, 3); let marker = segment::read_format(&RootLayout::new(&root)).expect("FORMAT is valid"); assert_eq!(marker.shard_count, 3); assert_eq!(marker.root_uuid, engine.shared.root_uuid); // And it is a *usable* root, not merely a well-formed one. let namespace = NamespaceId([0x51; 32]); block_on(engine.submit(create_transaction(namespace, 1))) .expect("a transaction commits into a root open() created"); } /// P1-1, isolated from every other cost in the open. /// /// The absolute count above is the whole claim, but it is one number made /// of four terms, and a reader has to trust the arithmetic to believe the /// ancestor term is in it. This asserts the ancestor term on its own: two /// otherwise identical initializations whose roots differ only in how many /// directory entries have to be created must differ in directory fsyncs by /// exactly that many. /// /// Before the fix the difference was zero — `create_dir_all` made all the /// entries and `initialize_root` fenced one of them — so this fails without /// it, and it fails with a number rather than an absence. #[test] fn startup_state_1_fences_one_directory_entry_per_created_ancestor() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); // One new entry: the root itself, inside an existing directory. let shallow = temporary.path().join("shallow"); let shallow_cost = StoreEngine::open(options(&serial, &shallow, 1)) .expect("state 1 at depth one") .shared .initialization .expect("state 1 records what it cost") .fsync_dir; // Three new entries: two ancestors and the root. let deep = temporary.path().join("deep").join("er").join("still"); let deep_cost = StoreEngine::open(options(&serial, &deep, 1)) .expect("state 1 at depth three") .shared .initialization .expect("state 1 records what it cost") .fsync_dir; assert_eq!( deep_cost - shallow_cost, 2, "two extra created ancestors must cost two extra directory fences; \ everything else about the two initializations is identical" ); } /// State 1, present but empty. Distinguished from the absent case only by /// who created the directory entry, and required to reach the same place. #[test] fn startup_state_1_initializes_an_existing_empty_root() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); assert_eq!(tree_image(temporary.path()), Vec::new()); let engine = StoreEngine::open(options(&serial, temporary.path(), 1)) .expect("an existing empty directory is startup state 1"); assert!(engine.shared.initialization.is_some()); assert!(temporary.path().join("FORMAT").exists()); } /// State 1, and the deliberate decision inside it: a directory holding /// only `LOCK` is empty. /// /// This is not a hypothetical. `segment::lock_root` creates `LOCK` if it is /// absent, so an initialization that dies between taking the lock /// and the `FORMAT` rename leaves precisely this, and so does any process /// that merely asked whether the root was busy. Classifying it as /// unrecognized would wedge a root with no bytes to lose. #[test] fn startup_state_1_treats_a_root_holding_only_lock_as_empty() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let layout = RootLayout::new(temporary.path()); drop(segment::lock_root(&layout).expect("probing the root creates LOCK")); let image = tree_image(temporary.path()); assert_eq!(image.len(), 1, "exactly LOCK is present: {image:?}"); assert_eq!(image[0].0, PathBuf::from("LOCK")); let engine = StoreEngine::open(options(&serial, temporary.path(), 1)) .expect("a root holding only LOCK is still startup state 1"); assert!(engine.shared.initialization.is_some()); } /// State 1 is a mutation, so it happens under the root lock. /// /// The observable is the refusal: with `LOCK` held elsewhere, `open` on /// what is otherwise a state-1 root must be refused `AlreadyLocked` and /// must leave the root exactly as empty as it found it. An initialization /// that ran before locking, or without locking, would create `FORMAT` here /// — and two processes doing that concurrently is the race the lock /// exists to close. #[test] fn startup_state_1_does_not_initialize_a_root_another_process_holds() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let layout = RootLayout::new(temporary.path()); let held = segment::lock_root(&layout).expect("take the root lock first"); assert!(matches!( StoreEngine::open(options(&serial, temporary.path(), 1)), Err(StoreError::AlreadyLocked) )); let image = tree_image(temporary.path()); assert_eq!(image.len(), 1, "nothing but LOCK was created: {image:?}"); // Releasing it makes the same call succeed, so the refusal was the // lock and not something else about the root. drop(held); StoreEngine::open(options(&serial, temporary.path(), 1)) .expect("the same root initializes once the lock is released"); assert_eq!(segment::RootLock::release_failures(), 0); } /// P1-2. An initialization interrupted after it has left residue is /// **finished** by the next `open` rather than refused for ever. /// /// Before the fix, any crash between `segment::initialize_root`'s first /// `create_dir_all` and its `FORMAT` rename turned a brand-new root into a /// permanently unopenable one: the root was non-empty and carried no /// `FORMAT`, so it was refused as states 3/4 — with a message about legacy /// layouts and migration, for a root that had never held a byte of anyone's /// data. /// /// The crash is simulated, not raced: the image below is written directly, /// so this test has no timing in it at all. The load-bearing part of the /// image — the marker — is produced by *production's own* encoder, so a /// change to the marker format cannot leave this test passing against bytes /// production no longer writes. The companion test below injects the /// interruption into a real `open` instead, which is what proves production /// writes the marker at the moment this image claims it does. #[test] fn startup_state_1_resumes_an_interrupted_initialization() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let root = temporary.path().join("interrupted"); // The crash image: the marker `open` installs as its first durable act, // plus as much of the tree as `segment::initialize_root` had built when // the power went — including a half-built shard directory, which is the // deepest residue the window can produce. std::fs::create_dir_all(root.join("quarantine")).expect("partial tree"); std::fs::create_dir_all(root.join("staging")).expect("partial tree"); std::fs::create_dir_all(root.join("shards").join("00").join("active")) .expect("partial tree"); std::fs::write( root.join("INITIALIZING"), initializing_marker_bytes(2, now_micros()), ) .expect("the marker production writes"); let engine = StoreEngine::open(options(&serial, &root, 2)) .expect("an interrupted initialization is finished, not refused"); assert!( engine.shared.initialization.is_some(), "the resume is an initialization, and is recorded as one" ); assert!(root.join("FORMAT").exists()); assert!( !root.join("INITIALIZING").exists(), "the marker is removed once FORMAT is durable, so the finished root \ classifies as state 2 and not as an interrupted one" ); // Usable, not merely well-formed. A resume that produced a tree nothing // could commit into would satisfy every assertion above. let namespace = NamespaceId([0x62; 32]); block_on(engine.submit(create_transaction(namespace, 1))) .expect("a transaction commits into a resumed root"); } /// P1-2's dangerous direction, asserted rather than argued. /// /// Refusing a resumable root wastes an operator's time; adopting a root /// that is not ours destroys their data. Each of these is one condition /// away from the resumable shape, and each must be refused **and left /// byte-identical**. #[test] fn an_unrecognized_layout_is_never_mistaken_for_an_interrupted_initialization() { let serial = writer_serial(); let refuse = |label: &str, root: &Path| { let before = tree_image(root); match StoreEngine::open(options(&serial, root, 1)) { Err(StoreError::NotImplemented(reason)) => { assert!( reason.contains("startup states 3 and 4"), "{label}: {reason}" ); } other => panic!("{label}: expected a refusal, got {other:?}"), } assert_eq!( tree_image(root), before, "{label}: a refused layout must be byte-identical afterwards" ); }; // The whole initialization skeleton and no marker. This is the shape // the classification must *not* adopt on the strength of the names // alone: those names are ours, but without the marker there is nothing // to say this directory was ever built by this store rather than // arranged to look like it was. let skeleton = tempfile::tempdir().expect("tempdir"); std::fs::create_dir_all(skeleton.path().join("quarantine")).expect("skeleton"); std::fs::create_dir_all(skeleton.path().join("staging")).expect("skeleton"); std::fs::create_dir_all(skeleton.path().join("shards").join("00")).expect("skeleton"); refuse("skeleton without a marker", skeleton.path()); // A byte-exact marker with one foreign name beside it. The marker alone // does not authorize anything: an initialization in progress cannot // have produced a name this store never writes, so the foreign entry // means the marker is in somebody else's directory. let intruded = tempfile::tempdir().expect("tempdir"); std::fs::write( intruded.path().join("INITIALIZING"), initializing_marker_bytes(1, now_micros()), ) .expect("marker"); std::fs::create_dir_all(intruded.path().join("quarantine")).expect("skeleton"); std::fs::write(intruded.path().join("notes.txt"), b"someone's data\n").expect("foreign"); refuse("a valid marker with a foreign entry", intruded.path()); // A file of exactly the marker's length, under exactly the marker's // name, that is not the marker. Length is not evidence; the magic is. let impostor = tempfile::tempdir().expect("tempdir"); std::fs::write(impostor.path().join("INITIALIZING"), [0u8; 27]).expect("impostor"); refuse("a same-length file that is not the marker", impostor.path()); // The magic and nothing after it. A prefix of the marker is not the // marker: accepting short reads would make every truncated file in the // universe of this name a licence to initialize. let truncated = tempfile::tempdir().expect("tempdir"); std::fs::write(truncated.path().join("INITIALIZING"), b"levcs-store-init").expect("short"); refuse("a truncated marker", truncated.path()); // A symlink of the marker's name, pointing at bytes that *are* a valid // marker. The content test passes if it is allowed to run at all, so // this is entirely a test of the type check: our marker is a regular // file in this directory, not a name that resolves to one somewhere // else. let elsewhere = tempfile::tempdir().expect("tempdir"); let real_marker = elsewhere.path().join("borrowed-marker"); std::fs::write(&real_marker, initializing_marker_bytes(1, now_micros())).expect("marker"); let linked = tempfile::tempdir().expect("tempdir"); std::os::unix::fs::symlink(&real_marker, linked.path().join("INITIALIZING")) .expect("symlink"); refuse("a symlink at the marker's name", linked.path()); } /// Contract review finding P1, direction 1: **a regular file at the staging /// name is not this store's residue and is not destroyed.** /// /// `classify_root` used to ignore `INITIALIZING.tmp` whatever it held, /// because the name "can only be this code path's residue by construction" /// — the one claim a classifier running *in order to establish* that is not /// entitled to. `build_root_under_lock` then opened it `create` + /// `truncate`. So an otherwise-empty configured root containing an /// operator's file of that name lost it silently, and reported success. /// /// Every shape below is one that used to be overwritten, including the torn /// marker — refusing that one is the trade `install_initializing_marker` /// states, and it is asserted here so the trade is a fact rather than a /// comment. #[test] fn a_foreign_entry_at_the_initializing_staging_name_is_refused_and_left_intact() { let serial = writer_serial(); let refuse = |label: &str, root: &Path| { let before = tree_image(root); match StoreEngine::open(options(&serial, root, 1)) { Err(StoreError::NotImplemented(reason)) => assert!( reason.contains("startup states 3 and 4"), "{label}: {reason}" ), other => panic!("{label}: expected a refusal, got {other:?}"), } assert_eq!( tree_image(root), before, "{label}: a refused root must be byte-identical afterwards" ); }; // An operator's file that happens to carry the name. This is the whole // finding: nothing else is in the root, so before the fix `open` // returned a working store and this file was gone. let operators = tempfile::tempdir().expect("tempdir"); std::fs::write( operators.path().join("INITIALIZING.tmp"), b"do not delete: staging notes for the migration\n", ) .expect("operator file"); refuse("an operator's file at the staging name", operators.path()); // A prefix of the marker — the exact residue of a crash between the // create and the fence. Refused, deliberately: nothing on disk tells it // apart from a stranger's truncated file, so a rule that adopts this one // destroys that one. let torn = tempfile::tempdir().expect("tempdir"); std::fs::write( torn.path().join("INITIALIZING.tmp"), &initializing_marker_bytes(1, now_micros())[..12], ) .expect("torn marker"); refuse("a torn temporary marker", torn.path()); // The right length and the wrong bytes, and a directory. Both are // "not a regular file holding our marker", and neither is a shape the // classifier may fold into a neighbour. let impostor = tempfile::tempdir().expect("tempdir"); std::fs::write(impostor.path().join("INITIALIZING.tmp"), [0u8; 27]).expect("impostor"); refuse("a same-length file that is not the marker", impostor.path()); let directory = tempfile::tempdir().expect("tempdir"); std::fs::create_dir(directory.path().join("INITIALIZING.tmp")).expect("directory"); refuse("a directory at the staging name", directory.path()); // A fifo, asserted without `tree_image` — deliberately, and the reason // is the point of the case. `tree_image` reads every entry it walks, // and reading a fifo with no writer blocks for ever; using it here // hangs the test before `open` is ever called. That is exactly the // hazard on the production side: anything that opens a name in an // unowned root before establishing its type can be stopped dead by one // `mkfifo`. `classify_root` checks `file_type` first and never opens // this entry, and `sys::open_regular_nofollow` passes `O_NONBLOCK` for // the path that can reach one anyway. This test completes rather than // timing out only because both of those hold. let fifo = tempfile::tempdir().expect("tempdir"); let fifo_name = fifo.path().join("INITIALIZING.tmp"); rustix::fs::mknodat( rustix::fs::CWD, &fifo_name, rustix::fs::FileType::Fifo, rustix::fs::Mode::from_raw_mode(0o644), 0, ) .expect("mkfifo"); match StoreEngine::open(options(&serial, fifo.path(), 1)) { Err(StoreError::NotImplemented(reason)) => { assert!(reason.contains("startup states 3 and 4"), "{reason}") } other => panic!("a fifo at the staging name: expected a refusal, got {other:?}"), } assert!( std::os::unix::fs::FileTypeExt::is_fifo( &fifo_name .symlink_metadata() .expect("the fifo is still there") .file_type() ), "the refusal must not have replaced the fifo with anything" ); assert!(!fifo.path().join("FORMAT").exists()); } /// Contract review finding P1, direction 2 and the worse one: **a symlink /// at the staging name must not be written through.** /// /// `create` + `truncate` follows a final symlink, so the truncation landed /// on a file the store had never owned and that was not even inside the /// configured root — and the link was then renamed onto `INITIALIZING` and /// unlinked, so the only evidence of what had been destroyed went with it. /// /// The assertion is on the *target*, outside the root, because that is /// where the damage was. `O_CREAT | O_EXCL` is what makes it impossible: /// it fails with `EEXIST` on a symlink whether or not the link resolves. #[test] fn a_symlink_at_the_initializing_staging_name_never_truncates_its_target() { let serial = writer_serial(); let outside = tempfile::tempdir().expect("tempdir"); let victim = outside.path().join("payroll.db"); let contents: Vec = (0..4096u32).map(|byte| (byte % 251) as u8).collect(); std::fs::write(&victim, &contents).expect("the file outside the root"); let root = tempfile::tempdir().expect("tempdir"); let link = root.path().join("INITIALIZING.tmp"); std::os::unix::fs::symlink(&victim, &link).expect("symlink"); match StoreEngine::open(options(&serial, root.path(), 1)) { Err(StoreError::NotImplemented(reason)) => { assert!(reason.contains("startup states 3 and 4"), "{reason}") } other => panic!("expected a refusal, got {other:?}"), } assert_eq!( std::fs::read(&victim).expect("the target still exists"), contents, "a symlink in the root must not be able to redirect a truncation at a \ file outside it" ); assert_eq!( std::fs::read_link(&link).expect("the link is still a link"), victim, "the link itself must not have been renamed or unlinked either — that is \ what removed the evidence" ); assert!( !root.path().join("FORMAT").exists(), "the refusal must not have initialized the root" ); } /// The other half of the trade: a **whole** temporary marker still resumes, /// and is *adopted* rather than rewritten. /// /// This is the shape a crash between the fence and the `rename_noreplace` /// leaves, and it must not become collateral damage of the type and content /// checks above. /// /// "Still resumes" alone cannot fail without the fix — the old code reached /// the same working root by truncating the file and writing it again, which /// is precisely the operation that destroyed an operator's file at the same /// name. So the load-bearing assertion is a counter, not an outcome: two /// otherwise identical initializations, one from an empty root and one from /// a root already carrying a whole marker, must differ in durable bytes by /// exactly the marker's length, because the second one does not write it. /// Before the fix that difference was zero. #[test] fn a_whole_temporary_marker_is_adopted_rather_than_rewritten() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); // The control: the same initialization with nothing staged. let fresh = temporary.path().join("fresh"); let fresh_bytes = StoreEngine::open(options(&serial, &fresh, 1)) .expect("state 1 from an empty root") .shared .initialization .expect("state 1 records what it cost") .bytes_written; let staged = temporary.path().join("staged"); std::fs::create_dir(&staged).expect("root"); std::fs::write( staged.join("INITIALIZING.tmp"), initializing_marker_bytes(1, now_micros()), ) .expect("staged marker"); let engine = StoreEngine::open(options(&serial, &staged, 1)) .expect("a whole temporary marker is this store's residue and is resumed from"); let initialization = engine .shared .initialization .expect("the resume is an initialization"); assert_eq!( fresh_bytes - initialization.bytes_written, INITIALIZING_MARKER_LEN as u64, "the staged marker already holds what this attempt would have written, so \ adoption writes nothing and fences what is there; everything else about the \ two initializations is identical" ); assert_eq!( initialization.fdatasync, 2, "adoption still costs exactly the marker's fence and FORMAT's" ); assert!(staged.join("FORMAT").exists()); assert!( !staged.join("INITIALIZING.tmp").exists(), "the staging name is consumed by the rename onto INITIALIZING" ); assert!(!staged.join("INITIALIZING").exists()); let namespace = NamespaceId([0x73; 32]); block_on(engine.submit(create_transaction(namespace, 1))) .expect("a transaction commits into a root resumed from a staged marker"); } /// The same hazard at the other two names this file opens or ignores. /// /// `segment::lock_root` opens `LOCK` and `segment::write_fenced` opens /// `FORMAT.tmp`. Both are in A1's frozen file, so neither open was changed by /// this deliverable; what it changed is that a root carrying a symlink at /// either name never reaches them, because the classification that authorizes /// the write refuses it first. /// /// Both have since been amended by the lead — `lock_root` in contract review /// 2026-07-29-A, `write_fenced` in 2026-07-29-B — because /// `RecoverySession::open`, `drive.rs` and `store-bench` all reach them with /// no classification at all. This test therefore no longer carries either /// guarantee on its own; it asserts that the classifier refuses first, which /// is still the answer an operator wants, and the assertions in `segment.rs` /// are what hold the opens themselves. #[test] fn a_symlink_at_lock_or_format_tmp_is_refused_before_anything_opens_it() { let serial = writer_serial(); let outside = tempfile::tempdir().expect("tempdir"); // `LOCK`, pointing at a name that does not exist. `create(true)` through // this link would bring a file into being outside the root. let absent = outside.path().join("not-there"); let locked = tempfile::tempdir().expect("tempdir"); std::os::unix::fs::symlink(&absent, locked.path().join("LOCK")).expect("symlink"); match StoreEngine::open(options(&serial, locked.path(), 1)) { Err(StoreError::NotImplemented(reason)) => { assert!(reason.contains("startup states 3 and 4"), "{reason}") } other => panic!("expected a refusal for a symlinked LOCK, got {other:?}"), } assert!( !absent.exists(), "a symlinked LOCK must not create a file outside the root" ); // `FORMAT.tmp`, beside a valid marker — the one arrangement that would // otherwise reach `segment::write_fenced`, because a `FORMAT.tmp` on its // own is already refused for having no marker to explain it. let victim = outside.path().join("ledger"); std::fs::write(&victim, b"balances\n").expect("the file outside the root"); let resumable = tempfile::tempdir().expect("tempdir"); std::fs::write( resumable.path().join("INITIALIZING"), initializing_marker_bytes(1, now_micros()), ) .expect("marker"); std::os::unix::fs::symlink(&victim, resumable.path().join("FORMAT.tmp")).expect("symlink"); match StoreEngine::open(options(&serial, resumable.path(), 1)) { Err(StoreError::NotImplemented(reason)) => { assert!(reason.contains("startup states 3 and 4"), "{reason}") } other => panic!("expected a refusal for a symlinked FORMAT.tmp, got {other:?}"), } assert_eq!( std::fs::read(&victim).expect("the target still exists"), b"balances\n", "a symlink at FORMAT.tmp must not become a truncation outside the root" ); } /// P1-2, with the interruption injected into a real `StoreEngine::open` /// instead of written to disk. /// /// One directory fsync is made to fail. On an existing empty root the first /// one `open` issues is the fence that makes the marker's *name* durable, /// so this stops initialization at the earliest instant that can leave /// residue — and proves the ordering the classification depends on: the /// marker is on disk and the tree is not. /// /// Deterministic, not raced: the fault registry is one-shot and positional, /// and this test holds the serial that makes it exclusively its own. #[cfg(feature = "failpoints")] #[test] fn an_interrupted_open_leaves_a_marker_the_next_open_resumes_from() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); crate::sys::arm(&serial, crate::sys::Fault::DirSyncEio); let interrupted = StoreEngine::open(options(&serial, temporary.path(), 1)); crate::sys::disarm(&serial); match interrupted { Err(StoreError::NotImplemented(reason)) => { panic!("an injected I/O failure must not read as an unbuilt state: {reason}") } Err(_) => {} Ok(_) => panic!("the injected directory fsync failure did not stop the open"), } let marker = temporary.path().join("INITIALIZING"); assert_eq!( std::fs::read(&marker) .expect("the interrupted open left its marker") .len(), 27, "the marker is installed by rename, so it is whole or absent and never torn" ); assert!( !temporary.path().join("shards").exists(), "the marker must become durable before the first directory of the tree exists, \ or a crash can leave tree residue with nothing to say whose it is" ); let engine = StoreEngine::open(options(&serial, temporary.path(), 1)) .expect("the next open finishes what the interrupted one started"); assert!(engine.shared.initialization.is_some()); assert!(!marker.exists(), "the finished root carries no marker"); assert_eq!(segment::RootLock::release_failures(), 0); } /// State 2. Reopening a formatted root runs recovery and initializes /// nothing — asserted directly, through the counter, rather than inferred /// from the tree looking unchanged. #[test] fn startup_state_2_opens_a_formatted_root_without_reinitializing_it() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let root = temporary.path().join("root"); let first = StoreEngine::open(options(&serial, &root, 2)).expect("state 1"); let uuid = first.shared.root_uuid; assert!(first.shared.initialization.is_some()); drop(first); let second = StoreEngine::open(options(&serial, &root, 2)).expect("state 2"); assert!( second.shared.initialization.is_none(), "a formatted root must not be initialized again" ); assert_eq!( second.shared.root_uuid, uuid, "the identity FORMAT froze is the identity the reopen carries" ); } /// States 3 and 4, both still refused this pass — and the property that /// survives the refusal either way: **the root is byte-identical**. /// /// Both layouts are checked because the whole hazard of implementing state /// 1 is that classifying "absent or empty" requires looking at a root that /// might be neither, and a probe that creates so much as a directory has /// destroyed this guarantee before the refusal is reached. #[test] fn startup_states_3_and_4_are_refused_by_name_and_leave_the_root_byte_identical() { let serial = writer_serial(); // State 3's shape: the legacy instance signature, `<64-hex>/.levcs/`. let legacy = tempfile::tempdir().expect("tempdir"); let repo = legacy.path().join("a".repeat(64)); std::fs::create_dir_all(repo.join(".levcs").join("objects")).expect("legacy tree"); std::fs::write(repo.join(".levcs").join("HEAD"), b"ref: refs/heads/main\n") .expect("legacy file"); // State 4's shape: non-empty, no `FORMAT`, no legacy signature — // including an empty directory and a zero-byte file, which a // contents-only comparison would not notice being removed. let unrecognized = tempfile::tempdir().expect("tempdir"); std::fs::write(unrecognized.path().join("notes.txt"), b"someone's data\n") .expect("stray file"); std::fs::write(unrecognized.path().join("FORMAT.tmp"), b"\x00\x01\x02").expect("residue"); std::fs::create_dir_all(unrecognized.path().join("empty")).expect("stray directory"); std::fs::write(unrecognized.path().join("empty-file"), b"").expect("stray empty file"); for root in [legacy.path(), unrecognized.path()] { let before = tree_image(root); match StoreEngine::open(options(&serial, root, 1)) { Err(StoreError::NotImplemented(reason)) => { assert!(reason.contains("startup states 3 and 4"), "{reason}"); assert!(reason.contains("LegacyLayout"), "{reason}"); assert!(reason.contains("UnrecognizedLayout"), "{reason}"); assert!(reason.contains("has not been modified"), "{reason}"); } other => panic!("expected an explicit refusal for {root:?}, got {other:?}"), } assert_eq!( tree_image(root), before, "a refused non-empty layout must be byte-identical afterwards: {root:?}" ); } } /// A `FORMAT` that exists but does not decode is state 2 and stays state /// 2. The dangerous misclassification is the other direction: treating an /// unreadable marker as "no marker, therefore empty" would authorize /// `initialize_root` to build a fresh tree over a populated root. #[test] fn a_corrupt_format_is_refused_rather_than_reinitialized_over() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); drop(StoreEngine::open(options(&serial, temporary.path(), 1)).expect("state 1")); let format = RootLayout::new(temporary.path()).format_path(); let original = std::fs::read(&format).expect("read FORMAT"); std::fs::write(&format, vec![0u8; original.len()]).expect("corrupt FORMAT"); let before = tree_image(temporary.path()); let error = StoreEngine::open(options(&serial, temporary.path(), 1)) .expect_err("a corrupt FORMAT is refused"); assert!( !matches!(error, StoreError::NotImplemented(_)), "a corrupt marker is state 2's refusal, not an unbuilt state: {error:?}" ); assert_eq!( tree_image(temporary.path()), before, "the refusal must not have rebuilt the root" ); } /// Copy a whole directory tree, file bytes included. /// /// The equivalence test below needs *two* copies of one crash image /// because recovery is a mutation: it seals the validated prefix, /// quarantines the discarded tail, and opens a fresh journal. Running both /// paths over one directory would compare the first path's recovery /// against the second path's recovery of what the first left behind, which /// would agree for the wrong reason. #[cfg(feature = "store-internals")] fn copy_tree(from: &Path, to: &Path) { std::fs::create_dir_all(to).expect("create the destination"); for entry in std::fs::read_dir(from).expect("read the source") { let entry = entry.expect("a directory entry"); let target = to.join(entry.file_name()); if entry.path().is_dir() { copy_tree(&entry.path(), &target); } else { std::fs::copy(entry.path(), &target).expect("copy a file"); } } } /// Scope 6.4 deliverable 1's second acceptance clause: **the engine and /// the drive seam produce the same recovered state from one crash image.** /// /// The two go through `recovery::recover_shard`, so they agree by /// construction — but "by construction" is exactly the kind of claim that /// survives the construction changing. `ShardDrive::reopen_through_recovery` /// makes one call to that function and `StoreEngine::open` makes one per /// shard; if either ever grows a local decision, this is what notices. /// /// The image is built at production altitude: `StoreEngine::open` /// initializes the root (startup state 1), real transactions are /// submitted, sequenced, signed and fenced through `submit`, and only then /// is the *journal file* damaged. The damage is a byte-level edit and not /// a logic-level one, which is the only kind a crash can produce. /// /// # What is compared, and what is deliberately not /// /// Every field of `ShardRecoveryReport` that is a function of the crash /// image. The two `PathBuf` fields — `quarantined` and `preserved_journal` /// — are excluded because they name files inside two different copies and /// can never be equal; their *contents* are covered by /// `quarantined_bytes`, which is compared. The recovery configurations are /// pinned to the drive's constants so that a disagreement here is a /// disagreement about the algorithm rather than about tuning. #[cfg(feature = "store-internals")] #[test] fn the_engine_and_the_drive_seam_recover_one_crash_image_identically() { use crate::drive::{ShardDrive, DRIVE_PREALLOCATE_BYTES}; let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x9c; 32]); // The drive's recovery configuration is a set of constants rather than // a `StoreOptions`, so the engine's options are pinned to match it. // Anything left unmatched would show up as an algorithmic // disagreement, which is the one thing this test must not be able to // report falsely. let configured = |root: &Path| { let mut configured = options(&serial, root, 1); configured.journal_preallocate_bytes = DRIVE_PREALLOCATE_BYTES; configured.manifest_retain = 2; configured.checkpoint_retain = 2; configured.terminal_status_grace_micros = 900_000_000; configured.max_active_index_entries = 4_000_000; configured.max_active_index_bytes = 512 * 1024 * 1024; configured.max_index_runs = 64; configured.max_open_index_runs = 32; configured }; let image = temporary.path().join("image"); { let engine = StoreEngine::open(configured(&image)).expect("startup state 1"); block_on(engine.submit(create_transaction(namespace, 1))).expect("create"); block_on(engine.submit(push_transaction(namespace, 2, 0xb1, None))).expect("push"); block_on(engine.submit(push_transaction(namespace, 3, 0xb2, None))).expect("push"); } // Where the fenced region ends, asked of the scanner rather than // computed: there is one definition of where a journal ends and this // test does not get to be a second one. let layout = RootLayout::new(&image); let root_uuid = segment::read_format(&layout).expect("FORMAT").root_uuid; let journal_path = std::fs::read_dir(layout.shard(0).active()) .expect("read active/") .map(|entry| entry.expect("entry").path()) .next() .expect("one active journal"); let (stop_offset, last_frame) = { let (_journal, scan) = crate::journal::Journal::open(&journal_path, &root_uuid, counters()) .expect("open the journal"); assert_eq!(scan.frames.len(), 3, "three fenced frames"); // A cleanly closed journal already stops early: the preallocated // remainder is zeros, and zeros are `NotAFrame`. Stated here // because it is why the damage below has to be a *torn frame* and // not merely non-zero residue — the latter would produce the same // classification a clean close does, and the comparison would no // longer be about a crash. assert_eq!(scan.stop, crate::journal::TailStop::NotAFrame); let last = scan.frames.last().expect("a third frame").clone(); (scan.stop_offset, last) }; // The crash residue: a frame whose header reached the device and whose // tail did not — the interrupted append. Recovery must stop at it, // discard it, and quarantine it, and both paths must reach the same // verdict about *which* completeness condition failed. { use std::io::{Read, Seek, SeekFrom, Write}; let mut torn = vec![0u8; last_frame.len as usize]; let mut file = std::fs::File::open(&journal_path).expect("read the journal"); file.seek(SeekFrom::Start(last_frame.offset)).expect("seek"); file.read_exact(&mut torn).expect("read a whole frame"); drop(file); torn.truncate(torn.len() - 8); let mut file = std::fs::OpenOptions::new() .write(true) .open(&journal_path) .expect("open the journal for damage"); file.seek(SeekFrom::Start(stop_offset)).expect("seek"); file.write_all(&torn).expect("write the torn frame"); file.sync_all().expect("fence the damage"); } let through_drive_root = temporary.path().join("through-drive"); let through_engine_root = temporary.path().join("through-engine"); copy_tree(&image, &through_drive_root); copy_tree(&image, &through_engine_root); let drive = ShardDrive::reopen_through_recovery(&through_drive_root, 0) .expect("the drive seam recovers the image"); let engine = StoreEngine::open(configured(&through_engine_root)).expect("the engine recovers it"); let engine_report = engine.shared.recovery_reports[0].clone(); let drive_report = drive.report.clone(); // The image really was a crash image, or the comparison below is a // comparison of two clean opens and proves nothing. assert!( drive_report.quarantined_bytes > 0, "the damaged tail must have been discarded: {drive_report:?}" ); assert!( matches!( drive_report.tail_stop, Some(crate::journal::TailStop::Incomplete(_)) ), "the tail must have been rejected as an incomplete frame: {drive_report:?}" ); assert_eq!(engine_report.shard_index, drive_report.shard_index); assert_eq!( engine_report.adopted_shard_sequences, drive_report.adopted_shard_sequences ); assert_eq!(engine_report.tail_stop, drive_report.tail_stop); assert_eq!( engine_report.quarantined_bytes, drive_report.quarantined_bytes ); assert_eq!(engine_report.active_journal, drive_report.active_journal); assert_eq!( engine_report.manifest_generation, drive_report.manifest_generation ); assert_eq!(engine_report.manifest_source, drive_report.manifest_source); assert_eq!( engine_report.checkpoint_sequence, drive_report.checkpoint_sequence ); assert_eq!( engine_report.offline_rebuild_required, drive_report.offline_rebuild_required ); assert_eq!(engine_report.promotions, drive_report.promotions); assert_eq!(engine_report.ready, drive_report.ready); assert!(engine_report.ready); // And the engine turned that shared recovered state into a committed // root a consumer can read, which is the half the drive seam has no // opinion about: three operations survived the crash image, and the // fourth — never submitted — is `Unknown` rather than invented. for operation in 1u8..=3 { assert!( matches!( engine.transaction_status(namespace, OperationId([operation; 16])), Ok(TransactionStatus::Committed(_)) ), "operation {operation} did not survive the crash image" ); } assert!(matches!( engine.transaction_status(namespace, OperationId([4; 16])), Ok(TransactionStatus::Unknown) )); } /// A configuration error must not be able to leave a root behind. The /// signer check runs before the root is touched at all, so a signerless /// open of an absent path creates nothing — not even the directory. #[test] fn a_configuration_error_refuses_before_startup_state_1_creates_anything() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let root = temporary.path().join("never-created"); let mut without = options(&serial, &root, 1); without.signer = None; assert!(matches!( StoreEngine::open(without), Err(StoreError::InvalidConfiguration(_)) )); assert!(!root.exists(), "a refused configuration created a root"); } #[test] fn a_configured_shard_count_that_disagrees_with_format_is_refused() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); // The topology is frozen by the *production* initialization, so the // mismatch is between two consumer-visible opens of one path rather // than between a hand-seeded root and an open. drop(open(&serial, temporary.path(), 4)); match StoreEngine::open(options(&serial, temporary.path(), 2)) { Err(StoreError::FormatMismatch(reason)) => { assert!(reason.contains("4 shards"), "{reason}") } other => panic!("expected FormatMismatch, got {other:?}"), } } #[test] fn opening_without_a_signer_is_refused_rather_than_deferred_to_the_first_submit() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); drop(open(&serial, temporary.path(), 1)); let mut without = options(&serial, temporary.path(), 1); without.signer = None; assert!(matches!( StoreEngine::open(without), Err(StoreError::InvalidConfiguration(_)) )); } // ----------------------------------------------------------------------- // The writer, end to end // ----------------------------------------------------------------------- #[test] fn a_committed_transaction_survives_close_and_reopen_through_recovery() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x20; 32]); let blob = ObjectId([0x77; 32]); let branch = set_branch("main", None, blob); let (create_receipt, push_receipt) = { let engine = open(&serial, temporary.path(), 1); let create = block_on(engine.submit(create_transaction(namespace, 1))) .expect("the create commits"); let push = block_on(engine.submit(push_transaction(namespace, 2, 0x77, Some(branch.clone())))) .expect("the push commits"); assert_eq!(create.repo_sequence, 0); assert_eq!(push.repo_sequence, 1); assert_eq!(push.objects_new, 1); assert_eq!( push.refs, vec![AppliedRefV1 { target: RefTarget::Branch("main".into()), old: None, new: Some(blob), force: false, }] ); assert_eq!( engine .transaction_status(namespace, OperationId([2; 16])) .expect("status"), TransactionStatus::Committed(push.clone()) ); (create, push) }; // Reopen through production recovery. Everything asserted below came // back off the device: the receipts, the applied refs, the repository // state, and the index entry with its object-type code. let engine = StoreEngine::open(options(&serial, temporary.path(), 1)).expect("reopen"); assert_eq!( engine .transaction_status(namespace, OperationId([1; 16])) .expect("status"), TransactionStatus::Committed(create_receipt) ); assert_eq!( engine .transaction_status(namespace, OperationId([2; 16])) .expect("status"), TransactionStatus::Committed(push_receipt) ); let root = engine.committed_root(); let repository = root.repo(&namespace).expect("the repository is bound"); assert_eq!(repository.repo_sequence, 1); assert_eq!(repository.genesis_authority, genesis_object().id); assert_eq!(repository.current_authority, genesis_object().id); assert_eq!( repository.refs.get(&RefTarget::Branch("main".into())), Some(&blob) ); // The object-type code this writer put in the index and the one // recovery reconstructs are the same table stated twice (see the // interface request on `object_type_code`). This is the assertion that // fails if they ever diverge. let located = root .index() .get(&IndexKey::new(namespace, blob)) .expect("the recovered index locates the pushed blob"); assert_eq!(located.object_type, object_type_code(ObjectType::Blob)); assert!( root.index() .get(&IndexKey::new(NamespaceId([0x21; 32]), blob)) .is_none(), "namespace isolation is a property of the key, on the recovered path too" ); } /// The frame the writer appends carries a signature over /// `SignedCommittedTransactionV1::signing_digest`, which is what the frozen /// `verify` checks — not over the bare event digest. /// /// A frame signed over anything else would be durable, correctly fenced, /// and verifiable against nothing, which no later recovery could detect. #[test] fn the_appended_frame_is_signed_over_the_digest_the_frozen_verifier_checks() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x30; 32]); { let engine = open(&serial, temporary.path(), 1); block_on(engine.submit(create_transaction(namespace, 1))).expect("commit"); } // Reopening runs production recovery, which seals the validated // prefix. Reading the frame back out of that segment means this test // inspects the bytes recovery accepted, not the bytes the writer // believes it wrote. drop(StoreEngine::open(options(&serial, temporary.path(), 1)).expect("reopen")); let signed = only_frame_payload(temporary.path()).committed; let digest = SignedCommittedTransactionV1::signing_digest( &signed.transaction, signed.source_key_epoch, &signed.durability_result, ) .expect("signing digest"); assert_eq!( signed.source_signature, TestSigner::expected_signature(&ObjectId(digest)) ); assert_eq!(signed.source_key_epoch, 7); assert_ne!( digest, signed.transaction.event_digest().expect("event digest").0, "the two digests must differ, or this test proves nothing" ); } /// Read the single frame a one-transaction store wrote, through the same /// sealed segment the reopen produced. fn only_frame_payload(root: &Path) -> TransactionFramePayloadV1 { let layout = RootLayout::new(root); let paths = layout.shard(0); // The root identity is read out of `FORMAT` rather than restated: the // root is now built by `StoreEngine::open`, which mints a fresh // `root_uuid` per root, and a hard-coded one here would be a second // definition of the identity every file in the root is bound to. let root_uuid = segment::read_format(&layout).expect("FORMAT").root_uuid; // Reopening sealed the validated prefix, so the frame is in a segment. let (manifest, _) = segment::load_manifest_with_fallback(&paths, &root_uuid) .expect("manifest") .expect("a manifest exists after the reopen sealed the prefix"); let range = manifest .retained_tail_ranges .last() .expect("one retained range"); let reader = segment::SegmentReader::open(&paths.segments().join(&range.filename), &root_uuid) .expect("segment"); let frame = reader .read_frame(range.first_shard_sequence) .expect("frame"); TransactionFramePayloadV1::decode_canonical(&frame.payload).expect("canonical payload") } /// Every frame the store durably wrote, in `shard_sequence` order, read /// back out of the sealed segments a reopen produced. fn all_frame_payloads(root: &Path) -> Vec { let layout = RootLayout::new(root); let paths = layout.shard(0); let root_uuid = segment::read_format(&layout).expect("FORMAT").root_uuid; let (manifest, _) = segment::load_manifest_with_fallback(&paths, &root_uuid) .expect("manifest") .expect("a manifest exists after the reopen sealed the prefix"); let mut payloads = Vec::new(); for range in &manifest.retained_tail_ranges { let reader = segment::SegmentReader::open(&paths.segments().join(&range.filename), &root_uuid) .expect("segment"); for sequence in range.first_shard_sequence..=range.last_shard_sequence { let frame = reader.read_frame(sequence).expect("frame"); payloads.push( TransactionFramePayloadV1::decode_canonical(&frame.payload) .expect("canonical payload"), ); } } payloads } /// Every durable frame carries a signature over the digest the frozen /// verifier recomputes, and its `previous_event_digest` is the event digest /// of the frame before it in its repository. /// /// This is the assertion a partial suffix repair fails. Dropping a member /// from a forming group re-chains every later member, and a repair that /// re-chained without re-signing leaves a durable, correctly fenced frame /// whose signature covers a chain that no longer exists — which nothing /// downstream of the append can detect. fn assert_every_frame_is_chained_and_signed(root: &Path) -> Vec { let payloads = all_frame_payloads(root); let mut previous: BTreeMap = BTreeMap::new(); for payload in &payloads { let signed = &payload.committed; let digest = SignedCommittedTransactionV1::signing_digest( &signed.transaction, signed.source_key_epoch, &signed.durability_result, ) .expect("signing digest"); assert_eq!( signed.source_signature, TestSigner::expected_signature(&ObjectId(digest)), "repo_sequence {} carries a signature over a message the signer never signed", signed.transaction.repo_sequence ); let repo = signed.transaction.repo_id; let expected = previous.get(&repo).copied().unwrap_or(ObjectId([0u8; 32])); assert_eq!( signed.transaction.previous_event_digest, expected, "repo_sequence {} is chained to an event that is not its predecessor", signed.transaction.repo_sequence ); previous.insert( repo, signed.transaction.event_digest().expect("event digest"), ); } payloads } /// P1-1. The committed event's `actor` is the evidence's principal, never /// the destination instance's signing key. /// /// The two are deliberately different values in this file. Under the defect /// this test exists for, `actor` was `TestSigner::public_key()`, the frame /// signed and fenced and published, and the first thing to notice was a /// mirror on another instance running `validate_mirror_application` against /// a transaction that could no longer be withdrawn. #[test] fn the_committed_event_takes_its_actor_from_the_evidence_not_from_the_signer() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x31; 32]); { let engine = open(&serial, temporary.path(), 1); block_on(engine.submit(create_transaction(namespace, 1))).expect("commit"); } drop(StoreEngine::open(options(&serial, temporary.path(), 1)).expect("reopen")); let signed = only_frame_payload(temporary.path()).committed; assert_eq!( signed.transaction.actor, evidence().actor(), "the committed event restates the evidence's actor" ); assert_eq!(signed.transaction.actor, EVIDENCE_ACTOR); assert_ne!( signed.transaction.actor, TestSigner::default().public_key(), "the destination signer's key is not the actor, and this store's two values \ differ so that substituting one for the other is visible" ); } /// Configured per-transaction ceilings, asserted through `submit` — the /// entry point a consumer calls — rather than through the codec maxima that /// happen to sit further down. #[test] fn configured_per_transaction_ceilings_are_enforced_at_submit() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x32; 32]); let mut configured = options(&serial, temporary.path(), 1); configured.max_objects_per_transaction = 1; configured.max_refs_per_transaction = 1; let engine = StoreEngine::open(configured).expect("open"); block_on(engine.submit(create_transaction(namespace, 1))).expect("create"); let authority = genesis_object().id; let two_objects = ValidatedTransaction::builder(PrivilegedConstruction::internal()) .namespace(namespace) .operation(OperationId([2; 16]), ObjectId([2; 32]), deadline()) .objects(vec![ StagedObject { id: ObjectId([0xb1; 32]), object_type: ObjectType::Blob, raw: vec![1; 8], }, StagedObject { id: ObjectId([0xb2; 32]), object_type: ObjectType::Blob, raw: vec![2; 8], }, ]) .refs(Vec::new()) .authority(Some(authority), Some(authority)) .evidence(evidence()) .build() .expect("a well formed but oversized transaction"); match block_on(engine.submit(two_objects)) { Err(StoreError::LimitExceeded { limit, observed, allowed, }) => { assert_eq!(limit, "max_objects_per_transaction"); assert_eq!((observed, allowed), (2, 1)); } other => panic!("the configured object ceiling must refuse, got {other:?}"), } let two_refs = ValidatedTransaction::builder(PrivilegedConstruction::internal()) .namespace(namespace) .operation(OperationId([3; 16]), ObjectId([3; 32]), deadline()) .objects(vec![StagedObject { id: ObjectId([0xb3; 32]), object_type: ObjectType::Blob, raw: vec![3; 8], }]) .refs(vec![ set_branch("main", None, ObjectId([0xb3; 32])), set_branch("other", None, ObjectId([0xb3; 32])), ]) .authority(Some(authority), Some(authority)) .evidence(evidence()) .build() .expect("a well formed but over-referenced transaction"); match block_on(engine.submit(two_refs)) { Err(StoreError::LimitExceeded { limit, observed, allowed, }) => { assert_eq!(limit, "max_refs_per_transaction"); assert_eq!((observed, allowed), (2, 1)); } other => panic!("the configured ref ceiling must refuse, got {other:?}"), } // Neither refusal reserved anything, and the shard still commits. for operation in [2u8, 3] { assert_eq!( engine .transaction_status(namespace, OperationId([operation; 16])) .expect("status"), TransactionStatus::Unknown ); } block_on(engine.submit(push_transaction(namespace, 4, 0xb4, None))).expect("push"); } /// Status occupancy is the size of the root that was just published, not of /// the one it replaced. /// /// Recording the previous root makes occupancy lag every insertion and /// every removal by one publication — wrong in exactly the direction that /// hides an approach to `max_status_entries`, since the reading is always /// one behind on the way up and never returns to zero on the way down. #[test] fn status_occupancy_reports_the_root_that_was_published() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x33; 32]); let engine = open(&serial, temporary.path(), 1); assert_eq!(engine.operation_status_metrics().occupancy, 0); block_on(engine.submit(create_transaction(namespace, 1))).expect("create"); assert_eq!( engine.operation_status_metrics().occupancy, 0, "a published group removes its status entries, and the metric must say so" ); assert_eq!(engine.operation_status_metrics().rejections, 0); block_on(engine.submit(push_transaction(namespace, 2, 0xc1, None))).expect("push"); assert_eq!(engine.operation_status_metrics().occupancy, 0); } /// Deliverable 6's concurrent-publication case, and the first test in this /// package to submit to more than one shard. /// /// Two shards publish into one `ArcSwap`, so the losing CAS /// must reload and merge against the newer root. Charter item 7: the retry /// is counted rather than asserted to have happened, and the no-lost-update /// property is checked against every receipt from both shards. #[test] fn concurrent_publication_from_two_shards_re_merges_and_never_loses_an_update() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let mut configured = options(&serial, temporary.path(), 2); configured.max_group_transactions = 2; configured.max_group_idle = Duration::from_millis(20); let engine = StoreEngine::open(configured).expect("open"); // Two namespaces that route to different shards, found rather than // assumed: the routing function is the authority on the mapping. let mut namespaces: Vec = Vec::new(); for byte in 0u8..=255 { let candidate = NamespaceId([byte; 32]); let shard = StoreOptions::shard_of(&candidate, 2); if namespaces.len() == usize::from(shard) { namespaces.push(candidate); } if namespaces.len() == 2 { break; } } assert_eq!(namespaces.len(), 2, "both shards must be reachable"); const ROUNDS: u8 = 24; std::thread::scope(|scope| { let handles: Vec<_> = namespaces .iter() .enumerate() .map(|(lane, namespace)| { let namespace = *namespace; let engine = &engine; let base = 1 + (lane as u8) * (ROUNDS + 1); scope.spawn(move || { block_on(engine.submit(create_transaction(namespace, base))) .expect("create"); for index in 0..ROUNDS { block_on(engine.submit(push_transaction( namespace, base + 1 + index, index, None, ))) .expect("push"); } }) }) .collect(); for handle in handles { handle.join().expect("no writer panics"); } }); // No lost update: every operation from both lanes is in the one root, // and each repository's sequence counted every one of its own. let root = engine.committed_root(); for (lane, namespace) in namespaces.iter().enumerate() { let base = 1 + (lane as u8) * (ROUNDS + 1); for index in 0..=ROUNDS { let key = OperationKey::new(*namespace, OperationId([base + index; 16])); assert!( root.terminal_status(&key).is_some(), "lane {lane} operation {index} is missing from the merged root" ); } assert_eq!( root.repo(namespace).expect("bound").repo_sequence, u64::from(ROUNDS) ); } assert_eq!(root.retained_generations().len(), 2); } /// Deliverable 6's re-merge, made deterministic. /// /// A competing root is published between the load and the compare-and-swap /// of the real `publish_subtree` loop, so the CAS provably fails exactly /// once. What must survive is both sides: the receipt this writer is /// publishing *and* the entry the competing publisher added. A lost update /// here is a re-merge against the stale root, which the previous test — /// two shards racing — can only ever catch by luck. /// /// The sibling test above submits to two shards concurrently and asserts no /// lost update; it cannot assert that any CAS contended, because whether it /// does depends on scheduling. This one owns that claim. #[test] fn a_contended_root_cas_re_merges_against_the_newer_root() { let probe_serial = probe::serial(); let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x71; 32]); // The competing subtree names the shard this namespace does *not* // route to, so it never disturbs the sequence bookkeeping of the shard // under test. Derived from the routing function rather than assumed. let publishing_shard = StoreOptions::shard_of(&namespace, 2); let other_shard = 1 - publishing_shard; // A marker only the competing publisher writes. If the writer re-merges // against the root it loaded rather than the newer one, this is gone. let marker = OperationKey::new(NamespaceId([0x72; 32]), OperationId([0x72; 16])); let competing = Arc::new(ShardSubtree::new( other_shard, 0, Arc::new(IndexDelta::from_options(&options( &serial, temporary.path(), 2, ))), None, Vector::new(), RepoMap::new(), { let mut statuses = TerminalStatusMap::new(); statuses.insert( marker, TerminalStatusEntry::Committed(RetainedReceipt { operation_digest: ObjectId([0x72; 32]), receipt: CommitReceipt { operation_id: marker.operation_id, repo_sequence: 0, current_authority: ObjectId([0; 32]), refs: Vec::new(), objects_new: 0, }, shard_sequence: 0, retry_until_micros: deadline(), first_visible_at_micros: now_micros(), receipt_visible_until_micros: deadline(), }), ); statuses }, GenerationMap::new(), )); let fired = Arc::new(std::sync::atomic::AtomicBool::new(false)); let armed = Arc::clone(&fired); // Filtered by root, because the hook is process-global and every other // test in this binary publishes through the same `publish_subtree`. // Without this, the one-shot perturbation is consumed by whichever // engine happens to publish first and this test observes no contention // at all — the Wave A lesson about one-shot global registries, one // layer up. let this_root = temporary.path().to_path_buf(); probe::install_cas_contention(&probe_serial, move |shared| { if shared.options.root != this_root { return; } // Once: the second pass through the loop must find an uncontended // root, or the writer would spin forever. if armed.swap(true, AtomicOrdering::SeqCst) { return; } let current = shared.committed.load(); shared.committed.store(Arc::new(current.merge(&competing))); }); let engine = StoreEngine::open(options(&serial, temporary.path(), 2)).expect("open"); let receipt = block_on(engine.submit(create_transaction(namespace, 1))) .expect("the contended publication still commits"); assert_eq!( engine.shared.root_cas_retries.load(AtomicOrdering::Relaxed), 1, "the committed-root CAS must have contended exactly once" ); let root = engine.committed_root(); assert!( root.terminal_status(&marker).is_some(), "the competing publisher's entry was lost: the re-merge used the stale root" ); assert_eq!( root.terminal_status(&OperationKey::new(namespace, OperationId([1; 16]))) .map(|entry| entry.transaction_status()), Some(TransactionStatus::Committed(receipt)), "this writer's own receipt was lost by the re-merge" ); } // ----------------------------------------------------------------------- // Scope 6.3 step 3: the pre-mark deadline recheck and its suffix repair // ----------------------------------------------------------------------- /// Submit `members` pushes into one forming group, the one at `expiring` /// carrying a deadline that passes while the group waits for its idle /// bound, and report each submission's outcome in submission order. /// /// The group is closed by the idle delay and never by the transaction /// ceiling, which is what puts real time between sequencing and the mark: /// scope 6.3 step 3 exists precisely because signing latency and the idle /// wait sit between the two. fn group_with_one_expiring_member( engine: &StoreEngine, namespace: NamespaceId, first_operation: u8, members: u8, expiring: u8, ) -> Vec> { let outcomes: Arc)>>> = Arc::new(Mutex::new(Vec::new())); std::thread::scope(|scope| { for index in 0..members { let outcomes = Arc::clone(&outcomes); scope.spawn(move || { // Staggered so the group forms in a known order. std::thread::sleep(Duration::from_millis(60 * u64::from(index))); let operation = first_operation + index; let retry_until = if index == expiring { now_micros() + 250_000 } else { deadline() }; let transaction = ValidatedTransaction::builder(PrivilegedConstruction::internal()) .namespace(namespace) .operation( OperationId([operation; 16]), ObjectId([operation; 32]), retry_until, ) .objects(vec![StagedObject { id: ObjectId([operation; 32]), object_type: ObjectType::Blob, raw: vec![operation; 64], }]) .refs(Vec::new()) .authority(Some(genesis_object().id), Some(genesis_object().id)) .evidence(evidence()) .build() .expect("a complete push transaction"); let outcome = block_on(engine.submit(transaction)); outcomes.lock().expect("outcomes").push((index, outcome)); }); } }); let mut collected = Arc::try_unwrap(outcomes) .map_err(|_| "outcomes outlived the scope") .expect("sole owner") .into_inner() .expect("outcomes"); collected.sort_by_key(|(index, _)| *index); collected.into_iter().map(|(_, outcome)| outcome).collect() } /// Scope 6.3 step 3 at the first, middle, and last position of a forming /// group, each with the later members already sequenced and signed. /// /// The expired member appends nothing, and every retained member is /// re-sequenced, re-chained, re-signed, and left with no sequence gap. /// A repair that re-chained without re-signing, or that shifted sequences /// without re-chaining, produces a durable and correctly fenced frame /// carrying a signature that verifies against nothing — which is why the /// assertion is made against the bytes recovery read back, not against the /// receipts. #[test] fn a_member_that_crosses_its_deadline_before_the_mark_appends_nothing() { for expiring in 0u8..3 { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x60 + expiring; 32]); let mut configured = options(&serial, temporary.path(), 1); // Never the transaction ceiling: the idle bound is what closes this // group, and it is long enough that the short deadline passes while // the group is still open. configured.max_group_transactions = 16; configured.max_group_idle = Duration::from_millis(900); let engine = StoreEngine::open(configured).expect("open"); block_on(engine.submit(create_transaction(namespace, 1))).expect("create"); let outcomes = group_with_one_expiring_member(&engine, namespace, 2, 3, expiring); let mut expected_repo_sequence = 1u64; for (index, outcome) in outcomes.iter().enumerate() { if index as u8 == expiring { match outcome { Err(StoreError::Conflict(reason)) => assert!( reason.contains("passed its signed deadline"), "position {expiring}: {reason}" ), other => panic!( "position {expiring}: an expired member must be refused, \ got {other:?}" ), } // Definitively absent: the reservation is released, so the // operation reads back Unknown rather than Pending forever. assert_eq!( engine .transaction_status(namespace, OperationId([2 + expiring; 16])) .expect("status"), TransactionStatus::Unknown, "position {expiring}: an expired member holds no reservation" ); } else { let receipt = outcome .as_ref() .unwrap_or_else(|e| panic!("position {expiring}: retained member {e:?}")); assert_eq!( receipt.repo_sequence, expected_repo_sequence, "position {expiring}: the retained suffix left a repo_sequence gap" ); expected_repo_sequence += 1; } } // The shard is not poisoned: nothing was appended for the dropped // member, so no sequence was consumed and no hole exists. block_on(engine.submit(push_transaction(namespace, 9, 0x9f, None))) .expect("the shard still admits mutation"); drop(engine); let engine = StoreEngine::open(options(&serial, temporary.path(), 1)).expect("reopen"); let payloads = assert_every_frame_is_chained_and_signed(temporary.path()); assert_eq!( payloads.len(), 4, "position {expiring}: one create, two retained members, one later push" ); for (index, payload) in payloads.iter().enumerate() { assert_eq!( payload.committed.transaction.repo_sequence, index as u64, "position {expiring}: durable repo_sequence is not gapless" ); } assert_eq!( engine .transaction_status(namespace, OperationId([2 + expiring; 16])) .expect("status"), TransactionStatus::Unknown, "position {expiring}: an expired member is absent after recovery too" ); } } #[test] fn a_repository_create_binds_its_genesis_permanently() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x40; 32]); let engine = open(&serial, temporary.path(), 1); block_on(engine.submit(create_transaction(namespace, 1))).expect("the first create"); match block_on(engine.submit(create_transaction(namespace, 2))) { Err(StoreError::Conflict(reason)) => { assert!(reason.contains("already bound to genesis"), "{reason}") } other => panic!("a second create must be refused, got {other:?}"), } } /// Phase 1 exit criterion "exact same-ref winner", asserted through /// `submit` rather than through the ref map it updates. #[test] fn exactly_one_writer_wins_a_contested_typed_ref() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x50; 32]); let engine = open(&serial, temporary.path(), 1); block_on(engine.submit(create_transaction(namespace, 1))).expect("create"); let first = block_on(engine.submit(push_transaction( namespace, 2, 0x60, Some(set_branch("main", None, ObjectId([0x60; 32]))), ))) .expect("the first writer wins"); assert_eq!(first.refs.len(), 1); // Same expected value, different new value: the loser must be a typed // conflict and must leave no partial state behind. match block_on(engine.submit(push_transaction( namespace, 3, 0x61, Some(set_branch("main", None, ObjectId([0x61; 32]))), ))) { Err(StoreError::Conflict(reason)) => { assert!(reason.contains("typed ref CAS"), "{reason}") } other => panic!("the loser must be a typed conflict, got {other:?}"), } let root = engine.committed_root(); assert_eq!( root.repo(&namespace) .expect("bound") .refs .get(&RefTarget::Branch("main".into())), Some(&ObjectId([0x60; 32])) ); assert!( root.index() .get(&IndexKey::new(namespace, ObjectId([0x61; 32]))) .is_none(), "a refused transaction publishes none of its objects" ); assert_eq!( engine .transaction_status(namespace, OperationId([3; 16])) .expect("status"), TransactionStatus::Unknown, "a definitively refused operation leaves no status entry behind" ); } // ----------------------------------------------------------------------- // Counters, not claims (charter item 7) // ----------------------------------------------------------------------- /// Group formation is bounded by transactions, bytes, and idle delay — /// asserted against the sequencer that forms the groups, and counted in /// `DurabilityCounters.fdatasync`, which is the only witness that says how /// many fences actually reached the device. /// /// This is the Wave A `GroupBuilder` carry-forward: the bounds are now /// observed through the production caller rather than through unit tests /// over a type nothing calls. #[test] fn group_formation_bounds_are_observed_as_fences_on_the_device() { // (a) the transaction bound: eight concurrent submissions with a // ceiling of four close exactly two groups. let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x70; 32]); let mut configured = options(&serial, temporary.path(), 1); configured.max_group_transactions = 4; // Long enough that the idle bound cannot be what closes these groups. configured.max_group_idle = Duration::from_secs(5); let engine = StoreEngine::open(configured).expect("open"); // The create is alone in its group, so it is the idle bound that // closes it — case (c), and the reason this engine cannot use a // multi-second idle delay for the whole test. let before_create = engine.durability_counters(0).expect("counters").fdatasync; assert_eq!(before_create, 0, "opening a recovered shard fences nothing"); let fences_after = commit_concurrently(&engine, namespace, 1, 8); assert_eq!( fences_after, 3, "one fence for the create's group and one for each of two full groups" ); assert!( fences_after < 9, "nine transactions must not cost nine fences" ); } #[test] fn a_group_is_also_closed_by_its_byte_ceiling() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x80; 32]); let mut configured = options(&serial, temporary.path(), 1); configured.max_group_transactions = 64; // One push frame is comfortably over 512 bytes once its evidence and // signed event are encoded, so the byte ceiling closes the group after // one transaction while the transaction ceiling is nowhere near. configured.max_group_bytes = 512; configured.max_group_idle = Duration::from_secs(5); let engine = StoreEngine::open(configured).expect("open"); let fences = commit_concurrently(&engine, namespace, 1, 4); assert_eq!( fences, 5, "the byte ceiling admits one transaction per group, so five \ transactions cost five fences" ); } /// Submit a create for `namespace` and then `pushes` concurrent pushes, /// and report the shard's fence count once every one of them has resolved. fn commit_concurrently( engine: &StoreEngine, namespace: NamespaceId, first_operation: u8, pushes: u8, ) -> u64 { block_on(engine.submit(create_transaction(namespace, first_operation))).expect("create"); std::thread::scope(|scope| { let handles: Vec<_> = (0..pushes) .map(|index| { let operation = first_operation + 1 + index; scope.spawn(move || { block_on(engine.submit(push_transaction( namespace, operation, 0x90 + index, None, ))) }) }) .collect(); for handle in handles { handle.join().expect("no writer panics").expect("commit"); } }); engine.durability_counters(0).expect("counters").fdatasync } // ----------------------------------------------------------------------- // Publication ordering (scope 6.3 steps 7 to 9) // ----------------------------------------------------------------------- /// Step 8 is strictly after step 7. /// /// The probe fires between them on the real writer. At that instant the /// receipt must already be in the committed root and the status entry must /// still be present. Removing the entry before the CAS fails the second /// assertion; publishing after the removal fails the first. Nothing else /// in this file can distinguish the two orders deterministically, because /// from outside they differ only by a window. #[test] fn the_committed_root_is_published_before_the_status_entry_is_removed() { let probe_serial = probe::serial(); let observations = Arc::new(Mutex::new(Vec::new())); let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0xa0; 32]); let recorded = Arc::clone(&observations); // Filtered by namespace: the hook is process-global and other tests // publish through it in parallel. Recording their groups would make // this test's count depend on the rest of the binary. probe::install(&probe_serial, move |shared, keys| { for key in keys.iter().filter(|key| key.namespace == namespace) { let committed = shared.committed.load().terminal_status(key).is_some(); let still_resolving = matches!( shared.status.load().get(key).map(|entry| entry.phase), Some(crate::roots::StatusPhase::Resolving) ); recorded .lock() .expect("probe mutex") .push((*key, committed, still_resolving)); } }); let engine = open(&serial, temporary.path(), 1); block_on(engine.submit(create_transaction(namespace, 1))).expect("create"); block_on(engine.submit(push_transaction(namespace, 2, 0xb0, None))).expect("push"); let observations = observations.lock().expect("probe mutex").clone(); assert_eq!(observations.len(), 2, "one observation per published group"); for (key, committed, still_resolving) in observations { assert!( committed, "operation {} must be in the committed root before its status entry is removed", key.operation_id.to_hex() ); assert!( still_resolving, "operation {} must still be Resolving at the instant its receipt becomes visible", key.operation_id.to_hex() ); } } /// The same property from the outside, through the API a consumer calls. /// /// A reader spinning on `transaction_status` may legitimately see /// `Unknown` before the operation is reserved. What it may never see is /// `Unknown` *after* it has already seen the operation, because that is /// the manufactured-`Unknown` race step 8 exists to close. #[test] fn a_status_reader_never_regresses_to_unknown_once_an_operation_is_visible() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0xc0; 32]); let engine = open(&serial, temporary.path(), 1); block_on(engine.submit(create_transaction(namespace, 1))).expect("create"); let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); let reader_stop = Arc::clone(&stop); std::thread::scope(|scope| { let observer = scope.spawn(|| { // Per operation, not per sweep: `Unknown` before an operation // is reserved is correct, and only a regression *after* it has // been seen is the manufactured `Unknown` step 8 closes. let mut seen = [false; 16]; let mut regressions = 0u64; while !reader_stop.load(AtomicOrdering::Relaxed) { for operation in 2u8..=17 { let index = usize::from(operation) - 2; let status = engine .transaction_status(namespace, OperationId([operation; 16])) .expect("status"); match status { TransactionStatus::Unknown => { if seen[index] { regressions += 1; } } TransactionStatus::Committed(_) | TransactionStatus::Pending { .. } | TransactionStatus::Resolving { .. } | TransactionStatus::Expired { .. } => seen[index] = true, } } } regressions }); for operation in 2u8..=17 { block_on(engine.submit(push_transaction(namespace, operation, operation, None))) .expect("push"); } stop.store(true, AtomicOrdering::Relaxed); assert_eq!( observer.join().expect("observer"), 0, "a committed operation may never read back as Unknown" ); }); } // ----------------------------------------------------------------------- // Failpoints: the publication half of the matrix (scope 6.3, oracle rows) // ----------------------------------------------------------------------- #[cfg(feature = "failpoints")] mod publication_failures { use levcs_protocol::oracle::{append_publication_expectation, ImmediateStatus}; use super::*; use crate::failpoints::{arm, disarm}; /// The frozen oracle's classification for a row, so each test below /// states which contract it is pinning rather than restating it. fn expectation(point: Failpoint) -> levcs_protocol::oracle::FailpointExpectation { append_publication_expectation(point.into()) } /// Scope 6.3 step 9. The fence succeeded and the root published, so /// the transaction is committed even though no waiter will ever be /// woken: a hung request, never an absent transaction. #[test] fn a_lost_wake_leaves_a_committed_and_queryable_receipt() { let expectation = expectation(Failpoint::AfterRootCasBeforeWaiterWake); assert!(!expectation.shard_poisoned); assert!(expectation.acknowledgment_allowed); assert_eq!(expectation.immediate_status, ImmediateStatus::Committed); let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0xd0; 32]); let engine = open(&serial, temporary.path(), 1); block_on(engine.submit(create_transaction(namespace, 1))).expect("create"); arm( &serial, Failpoint::AfterRootCasBeforeWaiterWake, FailpointAction::Fail, ); let hung = block_on_until( engine.submit(push_transaction(namespace, 2, 0xd1, None)), Duration::from_millis(750), ); disarm(&serial); assert!(hung.is_none(), "the waiter is never woken"); match engine .transaction_status(namespace, OperationId([2; 16])) .expect("status") { TransactionStatus::Committed(receipt) => { assert_eq!(receipt.repo_sequence, 1); assert_eq!(receipt.objects_new, 1); } other => panic!("a published transaction must stay queryable, got {other:?}"), } // The shard is not poisoned, so the next transaction commits. block_on(engine.submit(push_transaction(namespace, 3, 0xd2, None))) .expect("a later append is allowed"); } /// Scope 6.3 step 10, same rule one step later. #[test] fn a_lost_response_leaves_a_committed_and_queryable_receipt() { let expectation = expectation(Failpoint::BeforeResponse); assert!(!expectation.shard_poisoned); assert!(expectation.acknowledgment_allowed); assert_eq!(expectation.immediate_status, ImmediateStatus::Committed); let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0xe0; 32]); let engine = open(&serial, temporary.path(), 1); block_on(engine.submit(create_transaction(namespace, 1))).expect("create"); arm(&serial, Failpoint::BeforeResponse, FailpointAction::Fail); let lost = block_on(engine.submit(push_transaction(namespace, 2, 0xe1, None))); disarm(&serial); assert!(matches!(lost, Err(StoreError::Io(_)))); assert!(matches!( engine .transaction_status(namespace, OperationId([2; 16])) .expect("status"), TransactionStatus::Committed(_) )); block_on(engine.submit(push_transaction(namespace, 3, 0xe2, None))) .expect("a later append is allowed"); } /// Scope 3.7: steps 4 to 8 are the poison window. A failure between /// the `Resolving` mark and the committed-root publication leaves the /// operation `Resolving` — never `Unknown`, and never a receipt — and /// the shard admits no further mutation. #[test] fn a_failure_inside_the_publication_window_poisons_and_stays_resolving() { let expectation = expectation(Failpoint::BeforeRootCas); assert!(expectation.shard_poisoned); assert!(!expectation.acknowledgment_allowed); assert!(!expectation.later_append_allowed_before_recovery); assert_eq!(expectation.immediate_status, ImmediateStatus::Resolving); let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0xf0; 32]); let engine = open(&serial, temporary.path(), 1); block_on(engine.submit(create_transaction(namespace, 1))).expect("create"); arm(&serial, Failpoint::BeforeRootCas, FailpointAction::Fail); let poisoned = block_on(engine.submit(push_transaction(namespace, 2, 0xf1, None))); disarm(&serial); assert!(matches!(poisoned, Err(StoreError::ShardPoisoned { .. }))); match engine .transaction_status(namespace, OperationId([2; 16])) .expect("status") { TransactionStatus::Resolving { shard_sequence, operation_digest, .. } => { assert_eq!(shard_sequence, Some(1)); assert_eq!(operation_digest, ObjectId([2; 32])); } other => panic!("the group must remain Resolving, got {other:?}"), } assert!(matches!( block_on(engine.submit(push_transaction(namespace, 3, 0xf2, None))), Err(StoreError::ShardPoisoned { .. }) )); } /// The `Panic` action on a publication-side row (scope 4-A3 condition /// (b)). The failpoint names a *location*; the driver chooses the /// *action*, and the two axes are independent. /// /// The property under test is not the panic — it is that a panicking /// writer thread still resolves every caller it owes an outcome to. A /// completion that unwound with the frame it belonged to would leave /// `submit` waiting forever, which from outside is indistinguishable /// from a store that hung. #[test] fn a_writer_panic_inside_the_window_resolves_its_callers_rather_than_hanging() { let serial = writer_serial(); let expectation = expectation(Failpoint::DuringCommittedRootBuild); assert!(expectation.shard_poisoned); assert_eq!(expectation.immediate_status, ImmediateStatus::Resolving); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x28; 32]); let engine = open(&serial, temporary.path(), 1); block_on(engine.submit(create_transaction(namespace, 1))).expect("create"); arm( &serial, Failpoint::DuringCommittedRootBuild, FailpointAction::Panic, ); let outcome = block_on_until( engine.submit(push_transaction(namespace, 2, 0x29, None)), Duration::from_secs(5), ); disarm(&serial); match outcome { Some(Err(StoreError::ShardPoisoned { shard, .. })) => assert_eq!(shard, 0), other => panic!("a panicking writer must resolve its callers, got {other:?}"), } // The operation stays `Resolving`: step 8 never ran, and only // recovery may resolve it. assert!(matches!( engine .transaction_status(namespace, OperationId([2; 16])) .expect("status"), TransactionStatus::Resolving { .. } )); // The writer thread is gone, so the shard admits no further // mutation before a close and reopen through recovery. assert!(matches!( block_on(engine.submit(push_transaction(namespace, 3, 0x2a, None))), Err(StoreError::NotReady) )); } /// P1-5 case 1. A **panic** in scope 6.3 steps 1-3 is a pre-append /// failure, not a poisoning one. /// /// The single boundary catch treated every panic as a publication /// panic: the caller was told `ShardPoisoned` — which asserts the /// transaction may already be durable — and the reservation taken /// before sequencing was left behind, so the operation read back /// `Pending` forever for something that never reached the device. /// Nothing appended, so nothing can: the fate must match the phase. #[test] fn a_panic_before_the_mark_is_definitively_absent_and_does_not_poison() { let expectation = expectation(Failpoint::EvidenceHandoffFailure); assert!(!expectation.shard_poisoned); assert!(expectation.later_append_allowed_before_recovery); assert_eq!( expectation.immediate_status, ImmediateStatus::DefinitiveAbsent ); let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x1b; 32]); let engine = open(&serial, temporary.path(), 1); block_on(engine.submit(create_transaction(namespace, 1))).expect("create"); arm( &serial, Failpoint::EvidenceHandoffFailure, FailpointAction::Panic, ); let refused = block_on_until( engine.submit(push_transaction(namespace, 2, 0x1c, None)), Duration::from_secs(5), ); disarm(&serial); match refused { Some(Err(StoreError::Conflict(reason))) => assert!( reason.contains("pre-append"), "the refusal must say the transaction is absent, got {reason}" ), other => panic!( "a pre-append panic must be a definitive refusal, never a poisoned \ shard or a hang, got {other:?}" ), } assert_eq!( engine .transaction_status(namespace, OperationId([2; 16])) .expect("status"), TransactionStatus::Unknown, "a pre-append panic releases the reservation it took" ); assert_eq!( engine.operation_status_metrics().occupancy, 0, "no status entry survives a pre-append panic" ); // Steps 1-3 cannot poison, and a dead writer thread would make // `later_append_allowed_before_recovery` false whatever the error // said. The next transaction commits, contiguously. let later = block_on(engine.submit(push_transaction(namespace, 3, 0x1d, None))) .expect("a later append is allowed"); assert_eq!(later.repo_sequence, 1); } /// P1-5 case 2. A request that overflows the open byte-bounded group is /// not a member of the group that publication is about to write. /// /// The old order added the new request to the group's bookkeeping and /// then published the *old* group, so a publication panic reported the /// trailing request as poisoned though it never appended, and left its /// reservation `Pending` with nothing to resolve it. It also made the /// trailing frame's `shard_sequence` a prediction about how many /// members the closing group would keep — which scope 6.3 step 3 can /// change. #[test] fn a_request_that_overflows_a_publishing_group_is_not_reported_as_its_member() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x1e; 32]); let mut configured = options(&serial, temporary.path(), 1); configured.max_group_transactions = 64; // Wide enough that one small frame leaves the group open, and // narrow enough that a small frame plus a large one overflows it. // Stated as two sizes rather than one measured constant so the // overflow does not depend on the exact framing overhead. configured.max_group_bytes = 8 * 1024; configured.max_group_idle = Duration::from_secs(5); let engine = StoreEngine::open(configured).expect("open"); block_on(engine.submit(create_transaction(namespace, 1))).expect("create"); let large = || { ValidatedTransaction::builder(PrivilegedConstruction::internal()) .namespace(namespace) .operation(OperationId([3; 16]), ObjectId([3; 32]), deadline()) .objects(vec![StagedObject { id: ObjectId([0x3f; 32]), object_type: ObjectType::Blob, raw: vec![0x3f; 32 * 1024], }]) .refs(Vec::new()) .authority(Some(genesis_object().id), Some(genesis_object().id)) .evidence(evidence()) .build() .expect("a complete push transaction") }; // A panic and not a `Fail`: the old code released the trailing // request's reservation explicitly on the `Fail` path, so the leak // only appears when the publication unwinds and the single boundary // catch is the only thing left to resolve anyone. arm( &serial, Failpoint::AfterMarkedResolving, FailpointAction::Panic, ); let (small, big) = std::thread::scope(|scope| { let small = scope.spawn(|| { block_on_until( engine.submit(push_transaction(namespace, 2, 0x2f, None)), Duration::from_secs(10), ) }); let big = scope.spawn(|| { // Long enough that the small frame is already in the open // group, short enough that the idle bound has not closed it. std::thread::sleep(Duration::from_millis(300)); block_on_until(engine.submit(large()), Duration::from_secs(10)) }); (small.join().expect("small"), big.join().expect("big")) }); disarm(&serial); assert!( matches!(small, Some(Err(StoreError::ShardPoisoned { .. }))), "the member that was marked Resolving is inside the poison window, got {small:?}" ); match big { Some(Err(StoreError::Conflict(ref reason))) => assert!( reason.contains("pre-append"), "the trailing request never appended, so its refusal must say so: {reason}" ), ref other => panic!( "a request that never joined the published group is pre-append, never \ poisoned, got {other:?}" ), } // The point of the test: the trailing request appended nothing and // holds nothing. Under the old order it was left `Pending` with no // owner, which no recovery resolves because recovery only ever sees // what reached the device. assert_eq!( engine .transaction_status(namespace, OperationId([3; 16])) .expect("status"), TransactionStatus::Unknown, "a request that never joined the published group leaves no reservation" ); assert!( matches!( engine .transaction_status(namespace, OperationId([2; 16])) .expect("status"), TransactionStatus::Resolving { .. } ), "the group that was marked Resolving stays Resolving" ); assert_eq!( engine.operation_status_metrics().occupancy, 1, "exactly one operation — the one inside the poison window — is retained" ); } /// P1-5 case 3. Scope 6.3 steps 9 and 10 cannot poison, so a panic /// there resolves every caller with the receipt it is owed and leaves /// the writer running. /// /// The fence succeeded and the root published: the transaction is /// committed and unwithdrawable. The old boundary catch unwound after /// the waiters had been drained out of writer-owned storage, so it /// could not reach them at all, and it killed the writer — which makes /// `later_append_allowed_before_recovery` false for a row the frozen /// oracle marks `shard_poisoned: false`. #[test] fn a_panic_after_publication_still_hands_back_the_receipt_and_keeps_the_writer() { let expectation = expectation(Failpoint::AfterRootCasBeforeWaiterWake); assert!(!expectation.shard_poisoned); assert!(expectation.acknowledgment_allowed); assert!(expectation.later_append_allowed_before_recovery); assert_eq!(expectation.immediate_status, ImmediateStatus::Committed); let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x1f; 32]); let engine = open(&serial, temporary.path(), 1); block_on(engine.submit(create_transaction(namespace, 1))).expect("create"); arm( &serial, Failpoint::AfterRootCasBeforeWaiterWake, FailpointAction::Panic, ); let outcome = block_on_until( engine.submit(push_transaction(namespace, 2, 0x2b, None)), Duration::from_secs(5), ); disarm(&serial); match outcome { Some(Ok(receipt)) => assert_eq!(receipt.repo_sequence, 1), other => panic!( "a published transaction is committed; steps 9-10 may not turn it into \ anything else, got {other:?}" ), } assert!(matches!( engine .transaction_status(namespace, OperationId([2; 16])) .expect("status"), TransactionStatus::Committed(_) )); let later = block_on(engine.submit(push_transaction(namespace, 3, 0x2c, None))) .expect("the writer survives a non-poisoning panic"); assert_eq!(later.repo_sequence, 2); } /// Scope 6.3 step 2, as reclassified by contract review 2026-07-26-A: /// definitively absent, shard not poisoned, later append allowed. A /// routine `SignerError::Unavailable` must not take the shard down. #[test] fn an_evidence_handoff_failure_is_definitively_absent() { let expectation = expectation(Failpoint::EvidenceHandoffFailure); assert!(!expectation.shard_poisoned); assert!(!expectation.acknowledgment_allowed); assert!(expectation.later_append_allowed_before_recovery); assert_eq!( expectation.immediate_status, ImmediateStatus::DefinitiveAbsent ); let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x18; 32]); let engine = open(&serial, temporary.path(), 1); block_on(engine.submit(create_transaction(namespace, 1))).expect("create"); arm( &serial, Failpoint::EvidenceHandoffFailure, FailpointAction::Fail, ); let refused = block_on(engine.submit(push_transaction(namespace, 2, 0x19, None))); disarm(&serial); assert!(matches!( refused, Err(StoreError::Signer(SignerError::Unavailable)) )); assert_eq!( engine .transaction_status(namespace, OperationId([2; 16])) .expect("status"), TransactionStatus::Unknown, "a definitively absent operation releases its reservation" ); // Not poisoned, and no sequence was consumed: the next append is // contiguous with the create rather than leaving a hole. let later = block_on(engine.submit(push_transaction(namespace, 3, 0x1a, None))) .expect("a later append is allowed"); assert_eq!(later.repo_sequence, 1); } } } // --------------------------------------------------------------------------- // Index maintenance (scope 3.6, deliverable 1) // --------------------------------------------------------------------------- #[cfg(test)] mod index_maintenance_tests { use std::path::Path; use super::tests::*; use super::*; use crate::index::IndexKey; use crate::segment::{index_run_filename, read_current, read_manifest, RootLayout}; /// A root whose index ceilings make sealing reachable in a few groups. /// /// `max_active_index_entries` is the lever the acceptance point names: /// crossing it is `DeltaPressure::SealRequired`. One object per transaction /// means one entry per group, so the ceiling is also the layer count that /// crosses it. fn sealing_options(serial: &WriterSerial, root: &Path, entries: u64) -> StoreOptions { let mut options = options(serial, root, 1); options.max_active_index_entries = entries; // Sequential submits must each close their own group, or "three groups" // is one group of three and no layer accumulates. options.max_group_transactions = 1; options } fn shard_paths(root: &Path, shard: u16) -> crate::segment::ShardPaths { RootLayout::new(root).shard(shard) } /// Every index run the manifest `CURRENT` names, which is the only list /// that makes a run authoritative. fn manifest_runs(root: &Path, shard: u16, root_uuid: [u8; 16]) -> Vec { let paths = shard_paths(root, shard); let pointer = read_current(&paths, &root_uuid) .expect("read CURRENT") .expect("a root that has been opened has a CURRENT"); read_manifest(&paths, pointer.generation, &root_uuid) .expect("read the current manifest") .index_runs .into_iter() .map(|(_, name)| name) .collect() } fn root_uuid_of(root: &Path) -> [u8; 16] { crate::segment::read_format(&RootLayout::new(root)) .expect("FORMAT") .root_uuid } /// Commit `count` single-object pushes, each in its own group. fn push_groups( engine: &StoreEngine, namespace: NamespaceId, first: u8, count: u8, ) -> Vec { let mut objects = Vec::new(); for step in 0..count { let blob = first + step; block_on(engine.submit(push_transaction(namespace, blob, blob, None))) .unwrap_or_else(|error| panic!("push {blob} commits: {error:?}")); objects.push(ObjectId([blob; 32])); } objects } /// The acceptance point, and the discard it pays for. /// /// Three groups leave three layers and no run. The fourth submission is /// admitted only after the seal, so the numbers this reads are exactly: /// one run, and a backlog holding only the group that ran after it. #[test] fn crossing_seal_required_seals_before_admitting_more_work() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let engine = StoreEngine::open(sealing_options(&serial, temporary.path(), 3)) .expect("open a fresh root"); let namespace = NamespaceId([0x41; 32]); block_on(engine.submit(create_transaction(namespace, 1))).expect("create"); // The create is itself a group carrying one object, so two pushes bring // the accumulation to the ceiling. push_groups(&engine, namespace, 0x60, 2); let before = engine.index_maintenance(); assert_eq!( (before.sealed_runs, before.unsealed_delta_layers), (0, 3), "three groups must leave three unsealed layers and no run" ); // The fourth submission is refused — but only *after* the seal it // triggered, and for the replay ceiling rather than for the pressure // that caused the seal. An entry-pressure seal lands the shard exactly // on what a reopen could rebuild, because the frames it indexed are all // still in `active/`; see `seal_index_if_required`. let refused = block_on(engine.submit(push_transaction(namespace, 0x70, 0x70, None))) .expect_err("the shard is at the ceiling a reopen would have to rebuild"); assert!( matches!( refused, StoreError::LimitExceeded { limit: "max_active_index_entries", .. } ), "expected the replay ceiling, got {refused:?}" ); let after = engine.index_maintenance(); assert_eq!( (after.sealed_runs, after.unsealed_delta_layers), (1, 0), "the seal must publish one run and discard exactly the three layers it covered" ); // File presence is not publication; the manifest is. let uuid = root_uuid_of(temporary.path()); assert_eq!( manifest_runs(temporary.path(), 0, uuid), vec![index_run_filename(1)], "the run must be named by the current manifest" ); assert!(shard_paths(temporary.path(), 0) .indexes() .join(index_run_filename(1)) .is_file()); } /// Sealing is only correct if the run answers what the layers answered. /// /// Checked twice: against the live root once the layers are gone, and /// against a root rebuilt by production recovery after the engine is /// dropped. The second half is also the "crash after publication" case — /// nothing shuts the store down cleanly here. #[test] fn a_sealed_run_answers_every_covered_object_before_and_after_reopen() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x42; 32]); let mut covered = Vec::new(); let mut reopen_options = || { // Sealed by the fan-out ceiling, not by entry pressure. An // entry-pressure seal leaves the shard *at* the replay ceiling by // construction — see `replayable_index_entries` — so a reopen test // built on it would be asserting against a store that is one // transaction from refusing. let mut options = sealing_options(&serial, temporary.path(), 4_000_000); options.max_index_runs = 3; options.max_open_index_runs = 3; options }; { let engine = StoreEngine::open(reopen_options()).expect("open a fresh root"); block_on(engine.submit(create_transaction(namespace, 2))).expect("create"); covered.push(genesis_object().id); covered.extend(push_groups(&engine, namespace, 0x80, 2)); push_groups(&engine, namespace, 0x90, 1); assert_eq!(engine.index_maintenance().sealed_runs, 1, "the seal ran"); let root = engine.committed_root(); for object in &covered { let key = IndexKey::new(namespace, *object); // Without this the lookup below would pass on a store that // sealed nothing: a delta layer still holding the key answers // it, and the run is never consulted. assert!( root.index() .delta_layers() .iter() .all(|layer| layer.delta.get(&key).is_none()), "{} is still answered by a delta layer, so this asserts nothing about the run", object.to_hex() ); let location = root .index() .get(&key) .unwrap_or_else(|| panic!("{} is not in the sealed run", object.to_hex())); assert!( root.object_source(0, location.segment_generation) .expect("resolve") .is_some(), "the run's location must resolve to a file the root still pins" ); } } let engine = StoreEngine::open(reopen_options()).expect("reopen through production recovery"); assert_eq!( engine.index_maintenance().sealed_runs, 1, "a manifest-published run must be recovered" ); let root = engine.committed_root(); for object in &covered { assert!( root.index() .get(&IndexKey::new(namespace, *object)) .is_some(), "{} did not survive the reopen", object.to_hex() ); } } /// A run that reached the device but never a manifest is not a run. #[test] fn a_run_no_manifest_names_is_ignored_on_reopen() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x43; 32]); { let engine = StoreEngine::open(sealing_options(&serial, temporary.path(), 3)) .expect("open a fresh root"); block_on(engine.submit(create_transaction(namespace, 3))).expect("create"); } // Exactly what an interrupted seal leaves: a complete, valid run at the // name the next generation would have used, with no manifest naming it. let uuid = root_uuid_of(temporary.path()); let mut delta = IndexDelta::new(16, 1 << 20); delta .insert( IndexKey::new(namespace, ObjectId([0xEE; 32])), IndexLocation { segment_generation: 1, frame_offset: 0, frame_len: 1, object_type: ObjectType::Blob as u8, shard_sequence: 1, }, ) .expect("one entry"); let orphan = shard_paths(temporary.path(), 0) .indexes() .join(index_run_filename(9)); std::fs::write( &orphan, IndexRunBuilder::new(uuid, 9, 9) .build(&delta) .expect("encode"), ) .expect("write the orphan"); let engine = StoreEngine::open(sealing_options(&serial, temporary.path(), 3)) .expect("reopen with an orphan present"); assert_eq!( engine.index_maintenance().sealed_runs, 0, "an index run no manifest references must not become authoritative by existing" ); assert!( engine .committed_root() .index() .get(&IndexKey::new(namespace, ObjectId([0xEE; 32]))) .is_none(), "the orphan's entries must be invisible" ); assert!(orphan.is_file(), "and it is left alone, not reclaimed here"); } /// The discard is scoped to the shard that sealed. #[test] fn only_the_sealing_shards_layers_are_discarded() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let mut options = sealing_options(&serial, temporary.path(), 4_000_000); options.shard_count = 4; // Sealed by fan-out so the shard can keep running afterwards. options.max_index_runs = 4; options.max_open_index_runs = 4; let engine = StoreEngine::open(options).expect("open a fresh root"); // Two namespaces that route to different shards. let mut namespaces = Vec::new(); for byte in 0u8..64 { let candidate = NamespaceId([byte; 32]); let shard = StoreOptions::shard_of(&candidate, 4); if !namespaces.iter().any(|(s, _)| *s == shard) { namespaces.push((shard, candidate)); } if namespaces.len() == 2 { break; } } let (sealing_shard, sealing_namespace) = namespaces[0]; let (other_shard, other_namespace) = namespaces[1]; block_on(engine.submit(create_transaction(other_namespace, 0x11))).expect("create"); push_groups(&engine, other_namespace, 0xA0, 1); let other_layers_before = engine .committed_root() .index() .delta_layers() .iter() .filter(|layer| layer.shard_index == other_shard) .count(); assert!( other_layers_before > 0, "the other shard must have a backlog" ); block_on(engine.submit(create_transaction(sealing_namespace, 0x12))).expect("create"); // Four layers, then a fifth submission whose admission crosses the // fan-out ceiling and seals. push_groups(&engine, sealing_namespace, 0xB0, 4); let root = engine.committed_root(); assert_eq!( root.index() .delta_layers() .iter() .filter(|layer| layer.shard_index == other_shard) .count(), other_layers_before, "shard {other_shard} lost layers to a seal on shard {sealing_shard}" ); assert_eq!( root.index().sealed_run_count(), 1, "exactly the sealing shard published a run" ); } /// The ceilings recovery enforces are the ceilings the writer enforces. /// /// With room for one run, the second seal is refused — and refused as /// `LimitExceeded` naming the ceiling, not raised, not bypassed, and not the /// `NotImplemented` the unimplemented path used to answer. The shard stays /// usable, because nothing durable was attempted. #[test] fn the_run_ceilings_are_enforced_rather_than_raised() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let mut options = sealing_options(&serial, temporary.path(), 4_000_000); options.max_index_runs = 1; options.max_open_index_runs = 1; let engine = StoreEngine::open(options).expect("open a fresh root"); let namespace = NamespaceId([0x44; 32]); block_on(engine.submit(create_transaction(namespace, 4))).expect("create"); // One layer already crosses the fan-out ceiling, so this seals. push_groups(&engine, namespace, 0xC0, 1); assert_eq!(engine.index_maintenance().sealed_runs, 1); let refused = block_on(engine.submit(push_transaction(namespace, 0xC5, 0xC5, None))) .expect_err("a second run does not fit under max_index_runs = 1"); match refused { StoreError::LimitExceeded { limit, allowed, .. } => { assert_eq!(limit, "max_index_runs"); assert_eq!(allowed, 1); } other => panic!("expected LimitExceeded, got {other:?}"), } assert_eq!( engine.index_maintenance().sealed_runs, 1, "the refusal must not have installed a run" ); assert_eq!( manifest_runs(temporary.path(), 0, root_uuid_of(temporary.path())).len(), 1 ); } /// A seal that fails after the device has moved leaves no usable writer. /// /// The fence inside the run's own write is made to fail. That is the first /// durable step of the seal and it happens at admission, before this /// transaction has appended anything — so the fault cannot be consumed by a /// group append and lands where the test means it to. #[cfg(feature = "failpoints")] #[test] fn a_failed_seal_never_lets_the_writer_continue() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let mut options = sealing_options(&serial, temporary.path(), 4_000_000); options.max_index_runs = 1; options.max_open_index_runs = 1; let engine = StoreEngine::open(options).expect("open a fresh root"); let namespace = NamespaceId([0x45; 32]); block_on(engine.submit(create_transaction(namespace, 5))).expect("create"); crate::sys::arm(&serial, crate::sys::Fault::FenceEio); let failed = block_on(engine.submit(push_transaction(namespace, 0xD0, 0xD0, None))) .expect_err("a seal whose fence fails must not report success"); crate::sys::disarm(&serial); assert!( matches!(failed, StoreError::ShardPoisoned { .. }), "expected the shard to poison, got {failed:?}" ); let next = block_on(engine.submit(push_transaction(namespace, 0xD1, 0xD1, None))) .expect_err("a poisoned shard admits no further work"); assert!( matches!(next, StoreError::ShardPoisoned { .. }), "expected ShardPoisoned, got {next:?}" ); assert_eq!( engine.index_maintenance().sealed_runs, 0, "a failed seal must publish nothing" ); } }