From 4f799e32a4b8057ca0b88fd89df1145322707f46 Mon Sep 17 00:00:00 2001 From: Levi Neuwirth Date: Sun, 9 Aug 2026 20:15:11 +0200 Subject: [PATCH] Let a sealed index run be read back entry by entry Nothing in the store has ever removed an index run. `seal_index` pushes one per seal, adoption pushes the runs staging materialized, and each successor generation carries the predecessor's forward. That was invisible while nothing checkpointed on a cadence; wiring the P2 harness to checkpoint inside its measured window made it immediate, and a 125-second run died with `max_open_index_runs (observed: 33, allowed: 32)`. The frozen workload is arithmetically unreachable without compaction: 300s warmup plus 900s measured against 32 open runs allows at most one checkpoint per ~37 seconds and ends at the ceiling with no headroom for a fan-out-triggered seal. Raising the ceiling is the disclosed-weakening pattern retired earlier today, and would make `runs_sealed` describe a run whose fan-out grows unbounded; shortening the measured window is a Phase 0 contract. Merging runs is the only option that does not trade what the number means for the ability to produce one. A merge cannot be written against an API that answers point lookups only, so `IndexRun::entries` is granted by contract review 2026-08-09-C -- requested rather than emitted, because `index.rs` is a frozen Wave A interface. It is additive and read-only; no byte of the format moves. The order is the one already on the device, so a caller that re-encodes what it reads produces the layout it consumed. It reports no per-entry error: every value is decoded from bytes the run validated at `open`, and a run that could not be trusted entry by entry should not have opened. The test asserts the walk against the run's own `get` rather than against the delta it was built from. The delta is what the encoder was given; the question is whether the decoder reads back what was written, and a merge built on an iterator that disagreed with `get` would relocate entries silently. The compaction that consumes this is not here. Its shape is recorded in the review: partition by generation domain, because a run section is per-namespace with a 16-bit generation span and adopted-projection generations sit at `1 << 63`, so journal-backed and staging-owned runs can never merge into one. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JGdH5V43XWnj1PdHqiPktQ --- crates/levcs-store/src/index.rs | 84 +++++++++++++++++++++++++ doc/instance-throughput-rewrite-plan.md | 40 ++++++++++++ 2 files changed, 124 insertions(+) diff --git a/crates/levcs-store/src/index.rs b/crates/levcs-store/src/index.rs index 669bce8..57164a8 100644 --- a/crates/levcs-store/src/index.rs +++ b/crates/levcs-store/src/index.rs @@ -1126,6 +1126,36 @@ impl IndexRun { /// at all — but a section covering the range does not mean an entry in it /// does, and answering `true` on the range alone would turn recoverable /// roots into outages for a generation no entry mentions. + /// Every entry this run holds, in stored order. + /// + /// Granted to B1 by contract review 2026-08-09-C, for index-run compaction. + /// Merging sealed runs is the only way the open-run count stays under + /// `max_open_index_runs` across a measured campaign -- nothing else in the + /// store ever removes a run -- and a merge cannot be written against an API + /// that answers only point lookups. + /// + /// Read-only and additive: no byte of the format moves, and the order is + /// the one already on the device (sections ascending by namespace, entries + /// ascending within a section), so a caller that re-encodes what it reads + /// produces the same layout it consumed. + /// + /// Deliberately not `Iterator`-returning-`Result`: every value here is + /// decoded from bytes this run already validated at `open`, so there is no + /// per-entry failure to report. A run that could not be trusted entry by + /// entry is one that should not have opened. + pub fn entries(&self) -> impl Iterator + '_ { + (0..self.section_count).flat_map(move |i| { + let section = self.section(i); + (0..section.entry_count).map(move |j| { + let at = section.first_entry + j; + ( + IndexKey::new(section.namespace, self.entry_object(at)), + self.entry_location(at, §ion), + ) + }) + }) + } + pub fn references_segment_generation(&self, generation: u64) -> bool { for i in 0..self.section_count { let section = self.section(i); @@ -1322,6 +1352,60 @@ impl ObjectIndex { #[cfg(test)] mod tests { + /// Everything `entries` yields is what `get` answers for the same key, and + /// it yields every entry the run counts. + /// + /// Asserted against the run's own point lookup rather than against the + /// delta it was built from: the delta is what the encoder was given, and + /// the question is whether the decoder reads back what was written. A merge + /// built on an iterator that disagreed with `get` would produce a run whose + /// entries moved. + #[test] + fn entries_yields_every_entry_and_agrees_with_get() { + let mut delta = IndexDelta::new(1_000, 1 << 20); + let mut expected = Vec::new(); + // Several namespaces, so the walk crosses section boundaries, and + // several generations within each, so the packed deltas vary. + for ns in 0..4u8 { + for object in 0..7u8 { + let key = IndexKey::new(NamespaceId([ns; 32]), ObjectId([ns * 16 + object; 32])); + let location = IndexLocation { + segment_generation: 40 + u64::from(object), + frame_offset: 512 * u64::from(object), + frame_len: 128 + u32::from(object), + object_type: 1, + shard_sequence: 900 + u64::from(object), + }; + delta.insert(key, location).expect("insert"); + expected.push((key, location)); + } + } + + let bytes = IndexRunBuilder::new(ROOT, 9, 9) + .build(&delta) + .expect("build"); + let run = IndexRun::from_bytes(RunBytes::Owned(bytes), &ROOT).expect("open"); + + let seen: Vec<_> = run.entries().collect(); + assert_eq!( + seen.len() as u64, + run.entry_count(), + "the iterator must yield exactly the entries the run counts" + ); + for (key, location) in &seen { + assert_eq!( + run.get(key), + Some(*location), + "entries and get disagree for {key:?}" + ); + } + + expected.sort_by_key(|(key, _)| *key); + let mut sorted = seen.clone(); + sorted.sort_by_key(|(key, _)| *key); + assert_eq!(sorted, expected, "every inserted entry must be read back"); + } + use super::*; const ROOT: [u8; 16] = [7u8; 16]; diff --git a/doc/instance-throughput-rewrite-plan.md b/doc/instance-throughput-rewrite-plan.md index 743194b..0027ba9 100644 --- a/doc/instance-throughput-rewrite-plan.md +++ b/doc/instance-throughput-rewrite-plan.md @@ -1839,6 +1839,46 @@ The integration suite's `cleanup_is_deferred_and_names_its_deliverable` is repla deleted — the distinction it stood for, that "nothing to do" and "cannot answer yet" are different results, is now asserted the other way round at the public surface. +##### Contract review 2026-08-09-C + +`crates/levcs-store/src/index.rs` — `IndexRun::entries`. A frozen Wave A interface, so +requested rather than emitted. Additive and read-only; no byte of the run format moves. + +**Nothing in the store has ever removed an index run.** `seal_index` pushes one per seal and +the adoption path pushes the runs staging materialized; the successor generation carries the +predecessor's forward. That was invisible while nothing checkpointed on a cadence. Wiring the +P2 harness to take checkpoints inside its measured window made it immediate: a 125-second run +died with `max_open_index_runs (observed: 33, allowed: 32)`. + +**The frozen workload is arithmetically unreachable without compaction.** 300 s warmup plus +900 s measured against a hard ceiling of 32 open runs allows at most one checkpoint per ~37 s, +and ends at the ceiling with no headroom for a fan-out-triggered seal. The alternatives were +raising the ceiling — the disclosed-weakening pattern retired the same day by 2026-08-09-B, and +it would make `index_maintenance = runs_sealed` describe a run whose fan-out grows unbounded — +or shortening the measured window, which is a Phase 0 contract. Merging runs is the only option +that does not trade what the number means for the ability to produce one. + +**A merge cannot be written against the existing API**, which answers point lookups only: +`get`, `may_contain`, `entry_count`, `references_segment_generation`, +`references_only_segment_generation`. `entries` yields `(IndexKey, IndexLocation)` in the order +already on the device — sections ascending by namespace, entries ascending within a section — so +a caller that re-encodes what it reads produces the layout it consumed. It returns no `Result` +per entry: every value is decoded from bytes the run validated at `open`, and a run that could +not be trusted entry by entry should not have opened. +`entries_yields_every_entry_and_agrees_with_get` asserts the walk against the run's own `get` +rather than against the delta it was built from, because the question is whether the decoder +reads back what the encoder wrote. + +**Compaction partitions by generation domain, and that is a rule rather than a preference.** A +run section is per-namespace with a 16-bit generation span, and adopted-projection generations +sit in the band at `1 << 63`. Journal-backed runs may merge with each other and staging-owned +artifact runs with each other; a merge across the two is unencodable. Today's `seal_index` is +safe only because adopted entries never enter a shard delta, so compaction is the first +operation that could mix them. + +Scope is deliberately narrower than Phase 4's `CompactionBackup`: sealed index runs only, no +segment compaction, no event-floor retention, no backup interaction. + ##### Contract review 2026-08-09-B `bench/result-schema.json` — `run_conditions.index_run_ceiling`. Requested after index-delta