From b3415da9f33016ded0771a248362748cef9ec279 Mon Sep 17 00:00:00 2001 From: Levi Neuwirth Date: Fri, 31 Jul 2026 15:05:51 -0400 Subject: [PATCH] Reclaim displaced single-generation index runs --- crates/levcs-store/src/engine.rs | 226 ++++++++++++++++++++---- crates/levcs-store/src/index.rs | 35 ++++ crates/levcs-store/src/recovery.rs | 136 +++++++++++--- doc/instance-throughput-rewrite-plan.md | 54 ++++++ doc/phase1-storage-spine-scope.md | 27 ++- 5 files changed, 409 insertions(+), 69 deletions(-) diff --git a/crates/levcs-store/src/engine.rs b/crates/levcs-store/src/engine.rs index b6d9ab5..2c65a5f 100644 --- a/crates/levcs-store/src/engine.rs +++ b/crates/levcs-store/src/engine.rs @@ -7285,36 +7285,101 @@ mod index_maintenance_tests { } /// Every path that names the frames something other than the identity they - /// carry owes the same refusal, so the tests assert it through one function - /// rather than through copies that can drift apart the way the code did. - fn assert_the_open_refuses_naming_the_run( + /// carry owes the same reclamation, so the tests assert it through one + /// function rather than through copies that can drift apart the way the code + /// did. + /// + /// Contract review 2026-07-30-B refused these roots; 2026-07-30-G reclaims + /// them. "Reclaimed" is four claims and the test is worth nothing without + /// all four: the open succeeds, the manifest stops naming the run, every + /// object the run covered still resolves, and the `.idx` is still on the + /// device. A store that merely opened would be indistinguishable from one + /// that dropped the objects on the floor. + fn assert_the_open_reclaims_the_run( options: StoreOptions, root: &Path, covered: IndexKey, + resolvable: &[ObjectId], ) { - let (run, named) = the_published_runs_generation(root, covered); - let runs = [run]; + let (run, displaced) = the_published_runs_generation(root, covered); + let root_uuid = segment::read_format(&RootLayout::new(root)) + .expect("FORMAT") + .root_uuid; - match StoreEngine::open(options) { - Err(StoreError::Corruption(message)) => assert!( - message.contains(&runs[0]), - "the refusal must name the run that cannot be resolved, not just \ - report a generation: {message}" - ), - Err(other) => panic!("expected Corruption naming the run, got {other:?}"), - // Not prose: the state this refuses is verified here, so removing - // the refusal reports what it costs rather than a bare expectation. - Ok(opened) => { - let committed = opened.committed_root(); - let pinned = committed.object_source(0, named).expect("resolve"); + let opened = StoreEngine::open(options).unwrap_or_else(|error| { + panic!( + "the displaced run {run} covers only the identity being displaced, so replay \ + rebuilds every entry it held; refusing is an outage on a root that has lost \ + nothing: {error:?}" + ) + }); + + assert!( + !manifest_runs(root, 0, root_uuid).contains(&run), + "the manifest still names {run}; a row is what makes a run authoritative, so \ + leaving it hands the unresolvable run to the next open" + ); + assert_eq!( + opened.shared.recovery_reports[0] + .reclaimed_index_runs + .iter() + .filter_map(|path| path.file_name()) + .filter_map(|name| name.to_str()) + .collect::>(), + vec![run.as_str()], + "a reclamation is invisible in the opened store, so it has to be reported" + ); + + let committed = opened.committed_root(); + for object in resolvable { + let key = IndexKey::new(covered.namespace, *object); + let location = committed.index().get(&key).unwrap_or_else(|| { panic!( - "the open succeeded; published run {} names generation {named} and the \ - reopened root pins {pinned:?} there — every lookup reaching the run \ - rather than the replay delta above it reads nothing", - runs[0] - ); - } + "{} was covered by the reclaimed run and is now in no index at all", + object.to_hex() + ) + }); + assert!( + committed + .object_source(0, location.segment_generation) + .expect("resolve the source of a reclaimed run's entry") + .is_some(), + "{} resolves to generation {} and nothing pins it", + object.to_hex(), + location.segment_generation + ); } + + // The manifest governs the *next* open. This session must stop consulting + // and stop pinning the run in the same breath, or a direct run consumer + // — `StoreEngine::checkpoint` is one — reads locations that resolve to + // nothing for as long as the process lives. A lookup would not show it: + // the replay delta sits above the runs and answers first. + assert!( + committed + .index() + .sealed_runs() + .iter() + .all(|sealed| !sealed.references_segment_generation(displaced)), + "a run this session still consults holds locations against the displaced \ + generation {displaced}, which now resolves to nothing" + ); + assert!( + committed + .retained_generations() + .values() + .filter(|generation| generation.id.shard_index == 0) + .flat_map(|generation| generation.index_runs.iter()) + .all(|retained| retained.path().file_name() != Some(std::ffi::OsStr::new(&run))), + "{run} is still pinned by the retained generation, so it is still part of \ + what this generation claims to hold" + ); + + assert!( + root.join("shards/00/indexes").join(&run).exists(), + "the reclaimed {run} must be left on the device; recovery unlinking it would \ + destroy the evidence for a state this store has only just learned to handle" + ); } /// The generation the published run names for `covered` — read out of the @@ -7347,18 +7412,23 @@ mod index_maintenance_tests { (runs[0].clone(), location.segment_generation) } - /// Contract review 2026-07-30-B: the two states an occupied identity leaves. + /// Contract reviews 2026-07-30-B and 2026-07-30-G: what an occupied identity + /// costs. /// /// The tail's identity is generation 1 and a published run holds locations /// against it. An orphan `.seg` occupies that name, so recovery must seal - /// the frames under a different one — and the run then stays authoritative - /// through the manifest while resolving to nothing. Recovery refuses. + /// the frames under a different one — and the run would then stay + /// authoritative through the manifest while resolving to nothing. B refused + /// the open. G reclaims the run instead, which is sound here and only here: + /// every location it holds names the tail, and the tail's frames are above + /// any checkpoint horizon, so replay rebuilds all of them. /// - /// Replay masks this while a lookup goes through the delta above the run, so - /// the assertion is the open itself. `StoreEngine::checkpoint` is a direct + /// Replay masks the dangling state while a lookup goes through the delta + /// above the run, which is why the assertion is what the *manifest* names + /// rather than what a lookup answers. `StoreEngine::checkpoint` is a direct /// run consumer and would not be masked. #[test] - fn an_orphan_holding_a_published_runs_identity_refuses_the_open() { + fn an_orphan_holding_a_published_runs_identity_reclaims_it() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x4A; 32]); @@ -7378,10 +7448,11 @@ mod index_maintenance_tests { orphan_segment_at(temporary.path(), 0, 1); } - assert_the_open_refuses_naming_the_run( + assert_the_open_reclaims_the_run( configure(), temporary.path(), IndexKey::new(namespace, ObjectId([0x60; 32])), + &[ObjectId([0x60; 32]), ObjectId([0x61; 32])], ); } @@ -7394,9 +7465,10 @@ mod index_maintenance_tests { /// the identity it reuses is still not the one the frames carry, so the /// published run naming generation 1 is stranded exactly as it would be by a /// fresh fallback. Resumption is a reason to keep a choice, not a reason to - /// skip the check on it. + /// skip the check on it — and under 2026-07-30-G the check reclaims rather + /// than refuses, on the resumed path exactly as on the fresh one. #[test] - fn a_resumed_fallback_refuses_on_the_identity_it_resumes() { + fn a_resumed_fallback_reclaims_on_the_identity_it_resumes() { let serial = writer_serial(); let temporary = tempfile::tempdir().expect("tempdir"); let namespace = NamespaceId([0x4C; 32]); @@ -7417,10 +7489,11 @@ mod index_maintenance_tests { resumable_prefix_at(temporary.path(), 0, 2); } - assert_the_open_refuses_naming_the_run( + assert_the_open_reclaims_the_run( configure(), temporary.path(), IndexKey::new(namespace, ObjectId([0x60; 32])), + &[ObjectId([0x60; 32]), ObjectId([0x61; 32])], ); } @@ -7480,15 +7553,98 @@ mod index_maintenance_tests { ); // Now occupy the identity the frames carry, so the next open must - // displace them — and owes the refusal for the run that names them. + // displace them — and owes the reclamation for the run that names them. orphan_segment_at(temporary.path(), 0, 2); - assert_the_open_refuses_naming_the_run( + assert_the_open_reclaims_the_run( configure(), temporary.path(), IndexKey::new(namespace, ObjectId([0x60; 32])), + &[ObjectId([0x60; 32]), ObjectId([0x61; 32])], ); } + /// The case reclamation must *not* reach, which is the whole reason it is + /// conditional. + /// + /// A run may hold locations against more than one identity: a reopen replays + /// what the previous session sealed into a segment and what its journal + /// still holds, and a seal over that backlog covers both. Displacing the + /// tail's identity strands only the entries naming the tail — the ones + /// naming the segment are fine, and they may sit below a checkpoint horizon, + /// which replay does not touch. Discarding a run like that would delete the + /// only index those objects have. + /// + /// So recovery still refuses here, and the refusal is the good outcome: an + /// operator gets a diagnosable outage on a root that has lost nothing, + /// rather than a store that opens and has silently dropped an index. The + /// precondition — that the run really does name two generations — is + /// asserted, or this test would pass while proving only that something + /// refused. + #[test] + fn a_run_naming_more_than_the_displaced_identity_is_refused_not_discarded() { + let serial = writer_serial(); + let temporary = tempfile::tempdir().expect("tempdir"); + let namespace = NamespaceId([0x4F; 32]); + let configure = || { + let mut options = sealing_options(&serial, temporary.path(), 4_000_000); + // Fan-out, not entry pressure: the seal has to land once the replay + // delta and a fresh layer are both present, so the run it writes + // spans the segment recovery sealed and the journal above it. + options.max_index_runs = 2; + options.max_open_index_runs = 2; + options + }; + { + let engine = StoreEngine::open(configure()).expect("open a fresh root"); + block_on(engine.submit(create_transaction(namespace, 2))).expect("create"); + block_on(engine.submit(push_transaction(namespace, 0x60, 0x60, None))) + .expect("a first object, which the next open seals into a segment"); + } + { + let engine = StoreEngine::open(configure()).expect("reopen over the sealed prefix"); + block_on(engine.submit(push_transaction(namespace, 0x61, 0x61, None))) + .expect("a second layer above the replayed one"); + block_on(engine.submit(push_transaction(namespace, 0x62, 0x62, None))) + .expect("crossing the fan-out ceiling, which seals across both"); + assert_eq!(engine.index_maintenance().sealed_runs, 1, "the seal ran"); + } + + let (run, older) = the_published_runs_generation( + temporary.path(), + IndexKey::new(namespace, ObjectId([0x60; 32])), + ); + let (_, displaced) = the_published_runs_generation( + temporary.path(), + IndexKey::new(namespace, ObjectId([0x61; 32])), + ); + assert_ne!( + older, displaced, + "{run} names one identity, so it is the reclaimable case and this test is not \ + exercising the refusal it claims to" + ); + + // Occupy the identity the journal's frames carry, so the next open must + // displace it — and finds a run it cannot discard. + orphan_segment_at(temporary.path(), 0, displaced); + match StoreEngine::open(configure()) { + Err(StoreError::Corruption(message)) => { + assert!( + message.contains(&run) && message.contains("other generations"), + "the refusal must name the run and why it cannot be discarded: {message}" + ); + } + Err(other) => panic!("expected Corruption naming the run, got {other:?}"), + Ok(opened) => { + let committed = opened.committed_root(); + panic!( + "the open succeeded; {run} also names generation {older}, whose frames \ + replay need not rebuild, and generation {older} now pins {:?}", + committed.object_source(0, older) + ); + } + } + } + /// Resumption itself is not the hazard, and a guard that treated it as one /// would refuse every interrupted recovery on a root that has ever sealed. /// diff --git a/crates/levcs-store/src/index.rs b/crates/levcs-store/src/index.rs index 2f12996..669bce8 100644 --- a/crates/levcs-store/src/index.rs +++ b/crates/levcs-store/src/index.rs @@ -1148,6 +1148,41 @@ impl IndexRun { false } + /// Does every entry in this run name `generation`, and is there at least + /// one? + /// + /// Granted to recovery by contract review 2026-07-30-G, the sibling of + /// [`IndexRun::references_segment_generation`] and its one caller's second + /// question. Knowing a run holds locations against the identity recovery is + /// about to displace says the run is *affected*; knowing it holds nothing + /// else says the run is **reclaimable**, because replay rebuilds every + /// location it held and the run is redundant rather than lost. + /// + /// The distinction is the whole safety argument for discarding one. A run + /// mixing the displaced identity with an older one covers frames replay may + /// not rebuild — anything below the checkpoint horizon is not replayed at + /// all — and discarding that run would delete the only index those objects + /// have. So this is deliberately not `!references_some_other_generation` + /// with an empty run counting as reclaimable: emptiness is answered here + /// too, and an empty run is not something to reclaim on this evidence. + pub fn references_only_segment_generation(&self, generation: u64) -> bool { + let mut seen = false; + for i in 0..self.section_count { + let section = self.section(i); + for j in 0..section.entry_count { + if self + .entry_location(section.first_entry + j, §ion) + .segment_generation + != generation + { + return false; + } + seen = true; + } + } + seen + } + fn find_section(&self, namespace: &NamespaceId) -> Option { let mut lo = 0u64; let mut hi = self.section_count; diff --git a/crates/levcs-store/src/recovery.rs b/crates/levcs-store/src/recovery.rs index 3034d2e..7939830 100644 --- a/crates/levcs-store/src/recovery.rs +++ b/crates/levcs-store/src/recovery.rs @@ -1431,6 +1431,15 @@ pub struct ShardRecoveryReport { /// copied or sealed any prefix. The inode is byte-for-byte crash evidence. pub preserved_journal: Option, pub promotions: Vec, + /// Published index runs this recovery dropped from the manifest because the + /// seal displaced the only identity they held (contract review + /// 2026-07-30-G). + /// + /// Reported rather than inferred. A reclamation is invisible in the opened + /// store — the objects are all still there, rebuilt by replay — so without + /// this an operator cannot tell a root that reclaimed a run from one that + /// never had it, and the `.idx` left on the device has no other explanation. + pub reclaimed_index_runs: Vec, /// Scope 3.8 step 12: readiness is true only after replay and /// catalog/genesis validation complete. A computed field, never a default. pub ready: bool, @@ -1445,6 +1454,7 @@ impl ShardRecoveryReport { active_journal: None, checkpoint_sequence: None, offline_rebuild_required: false, + reclaimed_index_runs: Vec::new(), adopted_shard_sequences: Vec::new(), tail_stop: None, quarantined: None, @@ -1948,6 +1958,7 @@ fn recover_shard_under_lock( logical: logical_generation, tail: tail_generation, manifest: manifest_generation, + discarded: reclaimed_runs, } = recovery_generations_for_journal( &paths, &root_uuid, @@ -2058,7 +2069,42 @@ fn recover_shard_under_lock( let first = full_scan.frames.first().expect("non-empty").shard_sequence; let last = full_scan.frames.last().expect("non-empty").shard_sequence; - let (mut retained_tail_ranges, index_runs, checkpoints) = selection + // Inside the seal, because the seal is what displaces + // the identity. `logical` is chosen before the journal + // is scanned, so it can name a fallback generation for a + // journal that turns out to hold no frame — and then + // nothing is renamed, the surviving tail keeps + // `preferred`, and a run holding locations against + // `preferred` goes on resolving. Reclaiming it out there + // would discard a run that was never stranded. + // + // A reclaimed run must stop being authoritative in the + // same instant it stops resolving: dropped from what this + // generation retains, from what the layered index + // consults, and from the manifest published below. + // Leaving it in any one of the three is the dangling-run + // state the reclamation exists to end. + if !reclaimed_runs.is_empty() { + let reclaimed: std::collections::BTreeSet<&Path> = + reclaimed_runs.iter().map(PathBuf::as_path).collect(); + // Matched by pointer, not by name. Both lists were + // built from the same manifest rows in the same loop, + // so the `Arc` identifies the run exactly; re-deriving + // a name here would be a second way of saying which + // run this is, and the two could disagree. + let dropped: Vec> = retained_index_runs + .iter() + .filter(|retained| reclaimed.contains(retained.path())) + .map(|retained| Arc::clone(retained.run())) + .collect(); + retained_index_runs + .retain(|retained| !reclaimed.contains(retained.path())); + sealed_runs_newest_first + .retain(|run| !dropped.iter().any(|gone| Arc::ptr_eq(gone, run))); + report.reclaimed_index_runs = reclaimed_runs.clone(); + } + + let (mut retained_tail_ranges, mut index_runs, checkpoints) = selection .as_ref() .map(|selected| { ( @@ -2068,6 +2114,20 @@ fn recover_shard_under_lock( ) }) .unwrap_or((Vec::new(), Vec::new(), Vec::new())); + // The row is what makes a run authoritative, so this is + // where a reclamation actually takes effect: every + // earlier step only stopped *this* session consulting + // the run, and a row left here would hand it back to the + // next open. + if !reclaimed_runs.is_empty() { + let gone: std::collections::BTreeSet<&std::ffi::OsStr> = reclaimed_runs + .iter() + .filter_map(|path| path.file_name()) + .collect(); + index_runs.retain(|(_, filename)| { + !gone.contains(std::ffi::OsStr::new(filename)) + }); + } retained_tail_ranges.push(TailRange { generation: logical_generation, first_shard_sequence: first, @@ -2633,6 +2693,17 @@ pub(crate) struct RecoveryGenerations { /// The generation of the manifest this recovery installs. A counter over /// manifests, which an index run's manifest also advances. manifest: u64, + /// Published index runs this recovery **reclaims**: paths whose every + /// location named the identity the seal below is displacing. + /// + /// Empty unless the frames are being renamed, which is the only thing that + /// can strand a run. A path here is dropped from the manifest this recovery + /// installs, from the retained generation, and from the layered index — all + /// three, or a run that is no longer authoritative goes on answering + /// lookups. The file itself is left on the device: nothing but a manifest + /// makes a run authoritative, and unlinking it during recovery would destroy + /// the evidence for a state this store has only just learned to handle. + discarded: Vec, } /// Choose all three. @@ -2828,40 +2899,52 @@ fn recovery_generations_for_journal( // choose against it. It is what every decision below is measured from. let preferred = active_tail_logical_generation(selection); - // Contract review 2026-07-30-B, and the reason it is a closure rather than - // two copies: displacement can be decided here or in a session that crashed, - // and the two paths must not drift. + // Contract reviews 2026-07-30-B and 2026-07-30-G, and the reason it is a + // closure rather than two copies: displacement can be decided here or in a + // session that crashed, and the two paths must not drift. // // Naming the frames anything but `preferred` is what breaks a published run // holding locations against `preferred` — the manifest goes on naming the // run, `object_source` answers `None` for every location in it, and a reader // that reaches the run rather than the replay delta above it reads nothing - // and reports nothing. Replay masks that for exactly as long as nothing - // consumes runs directly, which is not a property to build a checkpointer - // on. + // and reports nothing. // - // So recovery refuses. Not because refusing is good — it is an outage on a - // root whose data is all present — but because the alternative is a store - // that opens and lies. The closure is for recovery to discard a run whose - // covered identity was not preserved, at which point this becomes successful - // reclamation; it is a change to what recovery *reclaims* and belongs with - // the checkpointing work that will exercise it (scope §6.5). - let refuse_if_displacement_strands_a_run = |chosen: u64| -> Result<(), StoreError> { + // B refused every such root. G reclaims the ones it can prove, and the proof + // is what `preferred` *is*: one past the last retained tail range, which is + // the identity of the **active journal** and of nothing else. Frames at that + // identity are above the manifest's committed prefix, therefore above any + // checkpoint horizon, therefore replayed in full — so a run holding only + // locations at `preferred` is rebuilt entry for entry by the delta this + // recovery is about to construct, at the generation the frames actually + // receive. Discarding it loses nothing; keeping it publishes locations that + // resolve to nothing. + // + // A run that mixes `preferred` with any other identity is still refused, and + // that is not conservatism. Its other entries may name a segment below the + // checkpoint horizon, which replay does not touch, so discarding the run + // would delete the only index those objects have — trading a diagnosable + // outage for silent loss, which is the trade B was written to prevent. + let mut discarded: Vec = Vec::new(); + let mut reclaim_or_refuse_displaced_runs = |chosen: u64| -> Result<(), StoreError> { if chosen == preferred { return Ok(()); } - match published_runs + for retained in published_runs .iter() - .find(|retained| retained.run().references_segment_generation(preferred)) + .filter(|retained| retained.run().references_segment_generation(preferred)) { - Some(retained) => Err(StoreError::Corruption(format!( - "this recovery must name its frames generation {chosen} rather than {preferred}, \ - and published index run {} holds locations against {preferred}; recovery cannot \ - yet discard a run whose covered identity was not preserved", - retained.path().display() - ))), - None => Ok(()), + if !retained.run().references_only_segment_generation(preferred) { + return Err(StoreError::Corruption(format!( + "this recovery must name its frames generation {chosen} rather than \ + {preferred}, and published index run {} holds locations against {preferred} \ + alongside locations against other generations; discarding it would drop \ + entries replay does not rebuild", + retained.path().display() + ))); + } + discarded.push(retained.path().to_path_buf()); } + Ok(()) }; // Resuming: the identity was already chosen and the artifact carries it. @@ -2872,11 +2955,12 @@ fn recovery_generations_for_journal( // resumable artifact *at* `preferred` displaces nothing and the guard is a // no-op on it. if let Some(logical) = resumable.into_iter().next() { - refuse_if_displacement_strands_a_run(logical)?; + reclaim_or_refuse_displaced_runs(logical)?; return Ok(RecoveryGenerations { logical, tail: preferred, manifest, + discarded, }); } @@ -2886,6 +2970,7 @@ fn recovery_generations_for_journal( logical: preferred, tail: preferred, manifest, + discarded, }); } @@ -2905,11 +2990,12 @@ fn recovery_generations_for_journal( .ok_or_else(|| { StoreError::Corruption("no logical generation remains for a recovery segment".into()) })?; - refuse_if_displacement_strands_a_run(logical)?; + reclaim_or_refuse_displaced_runs(logical)?; Ok(RecoveryGenerations { logical, tail: preferred, manifest, + discarded, }) } diff --git a/doc/instance-throughput-rewrite-plan.md b/doc/instance-throughput-rewrite-plan.md index bcf42c5..79f3764 100644 --- a/doc/instance-throughput-rewrite-plan.md +++ b/doc/instance-throughput-rewrite-plan.md @@ -1676,6 +1676,60 @@ B4 surface this commit touches — the emitter's remaining claims are unchanged. **Next, and in this dispatch:** recovery discarding an index run whose covered identity was not preserved, which turns 2026-07-30-B's refusal into reclamation. +##### Contract review 2026-07-30-G + +Recovery run-discard, which 2026-07-30-B deferred and named as the thing that would turn its refusal +into reclamation. It is a change to what recovery *reclaims*, not an addition to it, which is why it +is its own commit with its own mutation evidence. + +**The state.** A `.seg` occupies the identity the active tail's frames already carry, so recovery +must seal them under a different generation. A published index run holding locations against the +displaced identity then stays authoritative through the manifest while resolving to nothing. B +refused every root in that state. Refusing is an outage on a root that has lost nothing, and B said +so at the time. + +**What makes reclamation sound, and it is `preferred` itself.** `active_tail_logical_generation` is +one past the last retained tail range, so it is the identity of the **active journal** and of nothing +else. Frames at that identity sit above the manifest's committed prefix, therefore above any +checkpoint horizon, therefore replayed in full. A run holding *only* locations at `preferred` is +rebuilt entry for entry by the delta recovery is about to construct, at the generation the frames +actually receive. Discarding it loses nothing; keeping it publishes locations that resolve to +nothing. + +**What is still refused, and why that is the good outcome.** A run may name more than one identity: a +reopen replays what the previous session sealed into a segment *and* what its journal still holds, +and a seal over that backlog covers both. Displacing the tail strands only the tail's entries — the +segment's may sit below a checkpoint horizon, which replay does not touch. Discarding that run would +delete the only index those objects have, trading a diagnosable outage for silent loss, which is the +trade B existed to prevent. So the refusal survives for mixed runs and now says why. + +**Frozen-seam amendment to `index.rs` (A2).** `references_only_segment_generation`, the sibling of +B's `references_segment_generation` and the second question its one caller has to ask: the first says +a run is *affected*, this one says it is *reclaimable*. Read-only, exact rather than a range test, +and deliberately false for an empty run — emptiness is not something to reclaim on this evidence. + +**A reclamation is three removals, not one.** The manifest row is what makes a run authoritative, so +dropping it governs the next open; but this session must also stop consulting the run and stop +pinning it, or a direct run consumer — `StoreEngine::checkpoint` is one — reads dangling locations +for as long as the process lives. A lookup will not reveal it, because the replay delta sits above +the runs and answers first. The `.idx` itself is **left on the device**: nothing but a manifest makes +a run authoritative, and unlinking during recovery would destroy the evidence for a state this store +has only just learned to handle. `ShardRecoveryReport::reclaimed_index_runs` reports it, because a +reclamation is otherwise invisible in the opened store. + +**One placement that matters.** The reclamation is applied inside the seal, not where the generation +is chosen. `logical` is decided before the journal is scanned, so it can name a fallback for a +journal that turns out to hold no frame — and then nothing is renamed, the surviving tail keeps +`preferred` (2026-07-30-B amendment 2), and a run at `preferred` goes on resolving. Reclaiming there +would discard a run that was never stranded. + +**Mutation evidence.** Four mutations, each caught: treating every run as reclaimable fails the mixed +-run refusal; treating none as reclaimable fails all three reclamation regressions; leaving the +manifest row fails them on the authority assertion; and leaving the run in this session's index and +pins fails them on the direct-consumer assertion. That last mutation **survived the first version of +these tests**, because the replay delta shadows the run for ordinary lookups — the assertion that +catches it was added after the mutation showed the gap rather than before. + ##### Contract review 2026-07-30-F Two P1s and a P2 against the checkpoint-equivalence half of 2026-07-30-E. The namespace and diff --git a/doc/phase1-storage-spine-scope.md b/doc/phase1-storage-spine-scope.md index c502558..c5dda44 100644 --- a/doc/phase1-storage-spine-scope.md +++ b/doc/phase1-storage-spine-scope.md @@ -1710,13 +1710,21 @@ costs: - **Nothing published names the displaced identity.** Recovery falls back to a free generation and succeeds. An orphan segment is a documented state in which the active journal stays the authority, and refusing here would turn a recoverable root into an outage. -- **An index run the manifest names holds locations against it.** Recovery refuses the open with - `Corruption`, naming the run. Proceeding would publish a manifest whose run is authoritative and - resolves to nothing; the replay delta above it hides that from every lookup until the first - consumer that reads runs directly, which is the checkpointer. +- **An index run the manifest names holds locations against it, and against nothing else.** Recovery + **reclaims** it: the row leaves the manifest it installs, the run leaves the retained generation + and the layered index, and the open succeeds. Sound because the displaced identity is the active + journal's — one past the last retained tail range, and nothing else can carry it — so its frames + are above every checkpoint horizon and replay rebuilds each entry the run held. The `.idx` stays + on the device and `ShardRecoveryReport::reclaimed_index_runs` records what happened. +- **The run also holds locations against some other identity.** Recovery still refuses the open with + `Corruption`, naming the run. Its other entries may name a segment below the checkpoint horizon, + which replay does not touch, so discarding the run would delete the only index those objects have. + A diagnosable outage on a root that has lost nothing is the right outcome; publishing a manifest + whose run is authoritative and resolves to nothing is not — the replay delta above it hides that + from every lookup until the first consumer that reads runs directly, which is the checkpointer. The test is what the recovery *names its frames*, not how it got there: an interrupted recovery -resuming a fallback an earlier session chose strands the same run, and takes the same refusal. A +resuming a fallback an earlier session chose strands the same run, and takes the same treatment. A resumed seal at the identity the frames already carry displaces nothing and opens normally. The fallback applies only to frames being **sealed**. A journal that survives recovery keeps the @@ -1726,10 +1734,11 @@ from the manifest across every open — the property the split rests on. Giving fallback identity moved a number nothing recorded, and the next open, deriving the old one, searched published runs for a generation the frames no longer carried. -The closure is for recovery to **discard an index run whose covered identity was not preserved**, -which turns the refusal into successful reclamation. It is a change to what recovery reclaims and -belongs with the checkpointing work that will exercise it. Until then a root carrying both an orphan -segment and a run against the identity it holds does not open. +**Closed by contract review 2026-07-30-G.** Recovery discards an index run whose covered identity was +not preserved, which turned the refusal into successful reclamation for every run it can prove +redundant. The refusal survives only where the proof does not hold — a run naming more than the +displaced identity — and it is no longer a placeholder for missing capability but the correct answer +for a run whose other entries replay would not rebuild. ### 6.5 B3 — StagingSessions