Close the Wave A matrix carry-forward (B4 deliverable 1)
The Wave A review recorded that the crash matrix structurally could not
express its own first blocker: no failpoint corrupts a frame inside a
sealed segment, and none moves a journal between shards of one root. A
green matrix on a defect it cannot represent is the same trap as a helper
nothing calls, so the finding was carried forward rather than closed.
Add the two physical crash-image generators as a `damage` subcommand on
the crash driver, and two tests that drive them through the same
production `reconcile` path the matrix and the recovery script use.
- sealed-frame-corruption flips one payload byte in the first frame of
a segment the drive actually sealed and installed. The footer stays
structurally valid, so only frame verification can reject it.
- cross-shard-journal-movement relocates shard 1's active journal under
shard 0 of the same root. Root UUID validation cannot see this; the
journal header's shard index must be bound to the directory being
opened.
Both generators mutate a production-written image and fence the mutation;
neither synthesizes a footer, frame, manifest, or checksum, so a passing
test cannot be an artifact of the harness agreeing with itself. Each
source image is required to be unambiguous — exactly one segment or
journal — so the result does not depend on directory iteration order.
The assertions pin the refusal to its own cause rather than to any
non-zero exit: the frame case must fail on the digest recomputation and
the movement case on the shard binding, and neither may publish a partial
adoption result.
crash_matrix: 26 passed. scripts/check-phase1.sh green.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
f4c787418e
commit
cd37f8bc04
|
|
@ -37,6 +37,8 @@
|
|||
//! | 70 | harness-internal error |
|
||||
//! | 101 | Rust panic (the `Panic` action, unwinding with destructors) |
|
||||
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::ExitCode;
|
||||
|
||||
|
|
@ -45,7 +47,7 @@ use levcs_protocol::oracle::{AckRecord, ExternalAckJournal};
|
|||
use levcs_store::drive::faults::Fault;
|
||||
use levcs_store::drive::points::{Failpoint, FailpointAction};
|
||||
use levcs_store::drive::{DriveRecovery, ShardDrive, DRIVE_PREALLOCATE_BYTES};
|
||||
use levcs_store::format::{frame_total_len, JOURNAL_HEADER_LEN};
|
||||
use levcs_store::format::{frame_total_len, FRAME_HEADER_LEN, JOURNAL_HEADER_LEN};
|
||||
use levcs_store::types::NamespaceId;
|
||||
|
||||
/// The adopted-sequence property, shared with `store-bench` and with
|
||||
|
|
@ -429,6 +431,193 @@ fn journal_acks(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Subcommand: damage — deterministic physical crash-image generators
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
enum DamageKind {
|
||||
SealedFrameCorruption,
|
||||
CrossShardJournalMovement,
|
||||
}
|
||||
|
||||
impl DamageKind {
|
||||
fn parse(value: &str) -> Option<Self> {
|
||||
if value == "sealed-frame-corruption" {
|
||||
Some(Self::SealedFrameCorruption)
|
||||
} else if value == "cross-shard-journal-movement" {
|
||||
Some(Self::CrossShardJournalMovement)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn name(self) -> &'static str {
|
||||
match self {
|
||||
Self::SealedFrameCorruption => "sealed-frame-corruption",
|
||||
Self::CrossShardJournalMovement => "cross-shard-journal-movement",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct DamageArgs {
|
||||
root: PathBuf,
|
||||
kind: DamageKind,
|
||||
shard: u16,
|
||||
source_shard: u16,
|
||||
destination_shard: u16,
|
||||
}
|
||||
|
||||
/// Return the only file with `extension` under `directory`.
|
||||
///
|
||||
/// These generators deliberately require an unambiguous source image. Picking
|
||||
/// an arbitrary segment or journal from a larger store would make a passing
|
||||
/// campaign depend on directory iteration order and could mutate an
|
||||
/// unreferenced artifact instead of the production authority.
|
||||
fn only_file_with_extension(directory: &Path, extension: &str) -> Result<PathBuf, String> {
|
||||
let mut matches = Vec::new();
|
||||
let entries =
|
||||
fs::read_dir(directory).map_err(|e| format!("reading {}: {e}", directory.display()))?;
|
||||
for entry in entries {
|
||||
let path = entry
|
||||
.map_err(|e| format!("reading an entry under {}: {e}", directory.display()))?
|
||||
.path();
|
||||
if path
|
||||
.extension()
|
||||
.is_some_and(|candidate| candidate == extension)
|
||||
{
|
||||
matches.push(path);
|
||||
}
|
||||
}
|
||||
matches.sort();
|
||||
if matches.len() != 1 {
|
||||
return Err(format!(
|
||||
"{} must contain exactly one .{extension} file, found {}",
|
||||
directory.display(),
|
||||
matches.len()
|
||||
));
|
||||
}
|
||||
Ok(matches.pop().expect("length checked"))
|
||||
}
|
||||
|
||||
fn sync_directory(path: &Path) -> Result<(), String> {
|
||||
File::open(path)
|
||||
.and_then(|directory| directory.sync_all())
|
||||
.map_err(|e| format!("syncing directory {}: {e}", path.display()))
|
||||
}
|
||||
|
||||
fn run_damage(args: DamageArgs) -> ExitCode {
|
||||
println!("damage_schema=1");
|
||||
println!("damage_kind={}", args.kind.name());
|
||||
|
||||
let result = match args.kind {
|
||||
DamageKind::SealedFrameCorruption => damage_sealed_frame(&args.root, args.shard),
|
||||
DamageKind::CrossShardJournalMovement => {
|
||||
damage_cross_shard_journal(&args.root, args.source_shard, args.destination_shard)
|
||||
}
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(path) => {
|
||||
println!("damage_phase=complete");
|
||||
println!("damaged_path={}", path.display());
|
||||
ExitCode::SUCCESS
|
||||
}
|
||||
Err(error) => {
|
||||
println!("damage_phase=failed");
|
||||
eprintln!("store-crash-driver: damage: {error}");
|
||||
ExitCode::from(EX_DATAERR)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Flip one byte in the first sealed frame's payload and fence the mutation.
|
||||
///
|
||||
/// The source must be a real segment created by `ShardDrive::seal_and_install`;
|
||||
/// the generator does not synthesize a footer, frame, manifest, or checksum.
|
||||
/// The changed byte lies after both the journal and frame headers, so the
|
||||
/// segment footer remains structurally valid while production frame
|
||||
/// verification must reject the referenced authority.
|
||||
fn damage_sealed_frame(root: &Path, shard: u16) -> Result<PathBuf, String> {
|
||||
let segments = root
|
||||
.join("shards")
|
||||
.join(format!("{shard:02}"))
|
||||
.join("segments");
|
||||
let segment = only_file_with_extension(&segments, "seg")?;
|
||||
let offset = (JOURNAL_HEADER_LEN + FRAME_HEADER_LEN) as u64;
|
||||
let mut file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(&segment)
|
||||
.map_err(|e| format!("opening {}: {e}", segment.display()))?;
|
||||
if file
|
||||
.metadata()
|
||||
.map_err(|e| format!("stat {}: {e}", segment.display()))?
|
||||
.len()
|
||||
<= offset
|
||||
{
|
||||
return Err(format!(
|
||||
"{} has no first-frame payload byte at offset {offset}",
|
||||
segment.display()
|
||||
));
|
||||
}
|
||||
file.seek(SeekFrom::Start(offset))
|
||||
.map_err(|e| format!("seeking {}: {e}", segment.display()))?;
|
||||
let mut byte = [0u8; 1];
|
||||
file.read_exact(&mut byte)
|
||||
.map_err(|e| format!("reading {}: {e}", segment.display()))?;
|
||||
byte[0] ^= 0x80;
|
||||
file.seek(SeekFrom::Start(offset))
|
||||
.map_err(|e| format!("seeking {}: {e}", segment.display()))?;
|
||||
file.write_all(&byte)
|
||||
.map_err(|e| format!("writing {}: {e}", segment.display()))?;
|
||||
file.sync_data()
|
||||
.map_err(|e| format!("fencing {}: {e}", segment.display()))?;
|
||||
sync_directory(&segments)?;
|
||||
Ok(segment)
|
||||
}
|
||||
|
||||
/// Move the only active journal from one shard directory into another.
|
||||
///
|
||||
/// This reproduces the Wave A review's same-root cross-shard image. A footer
|
||||
/// cannot detect it: the journal header's `shard_index` is the authority, and
|
||||
/// production recovery must bind that value to the directory it is opening.
|
||||
fn damage_cross_shard_journal(
|
||||
root: &Path,
|
||||
source_shard: u16,
|
||||
destination_shard: u16,
|
||||
) -> Result<PathBuf, String> {
|
||||
if source_shard == destination_shard {
|
||||
return Err("source and destination shards must differ".to_string());
|
||||
}
|
||||
let shards = root.join("shards");
|
||||
let source_dir = shards.join(format!("{source_shard:02}")).join("active");
|
||||
let destination_dir = shards
|
||||
.join(format!("{destination_shard:02}"))
|
||||
.join("active");
|
||||
let source = only_file_with_extension(&source_dir, "journal")?;
|
||||
let name = source
|
||||
.file_name()
|
||||
.ok_or_else(|| format!("{} has no file name", source.display()))?;
|
||||
let destination = destination_dir.join(name);
|
||||
if destination.exists() {
|
||||
return Err(format!(
|
||||
"destination {} is already occupied",
|
||||
destination.display()
|
||||
));
|
||||
}
|
||||
fs::rename(&source, &destination).map_err(|e| {
|
||||
format!(
|
||||
"moving {} to {}: {e}",
|
||||
source.display(),
|
||||
destination.display()
|
||||
)
|
||||
})?;
|
||||
sync_directory(&source_dir)?;
|
||||
sync_directory(&destination_dir)?;
|
||||
Ok(destination)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Subcommand: soak
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -719,12 +908,15 @@ impl Args {
|
|||
}
|
||||
|
||||
const USAGE: &str = "\
|
||||
store-crash-driver <append|soak|reconcile> [flags]
|
||||
store-crash-driver <append|damage|soak|reconcile> [flags]
|
||||
|
||||
append --root P --shard N [--shard-count N] [--create] --seed U64
|
||||
--group-len N [--victim N] [--point NAME] [--action ACTION]
|
||||
[--fault FAULT] [--ack-journal P] [--path drive|submit]
|
||||
[--seal] [--seal-fault FAULT]
|
||||
damage --root P --kind sealed-frame-corruption [--shard N]
|
||||
damage --root P --kind cross-shard-journal-movement
|
||||
[--source-shard N] [--destination-shard N]
|
||||
soak --root P --shard N [--shard-count N] --seed U64 --group-len N
|
||||
--ack-journal P [--max-groups N] [--path drive|submit]
|
||||
reconcile --root P --shard N [--ack-journal P] [--fault FAULT]
|
||||
|
|
@ -803,6 +995,19 @@ fn dispatch(args: &Args) -> Result<ExitCode, String> {
|
|||
}));
|
||||
}
|
||||
|
||||
if args.subcommand == "damage" {
|
||||
let kind_name = args.required("kind")?;
|
||||
let kind = DamageKind::parse(kind_name)
|
||||
.ok_or_else(|| format!("unknown damage kind {kind_name:?}"))?;
|
||||
return Ok(run_damage(DamageArgs {
|
||||
root: PathBuf::from(args.required("root")?),
|
||||
kind,
|
||||
shard: args.parsed::<u16>("shard", 0)?,
|
||||
source_shard: args.parsed::<u16>("source-shard", 1)?,
|
||||
destination_shard: args.parsed::<u16>("destination-shard", 0)?,
|
||||
}));
|
||||
}
|
||||
|
||||
if args.subcommand == "reconcile" {
|
||||
return Ok(run_reconcile(ReconcileArgs {
|
||||
root: PathBuf::from(args.required("root")?),
|
||||
|
|
|
|||
|
|
@ -715,6 +715,186 @@ fn seed_committed_group(root: &std::path::Path, ack: &std::path::Path, seed: u64
|
|||
run.sequences("appended_sequences")
|
||||
}
|
||||
|
||||
fn generate_damage(root: &std::path::Path, arguments: &[&str]) -> std::process::Output {
|
||||
std::process::Command::new(harness::DRIVER)
|
||||
.arg("damage")
|
||||
.arg("--root")
|
||||
.arg(root)
|
||||
.args(arguments)
|
||||
.output()
|
||||
.expect("spawning physical crash-image generator")
|
||||
}
|
||||
|
||||
/// Wave A review finding 1, physical image 1: a manifest-referenced segment
|
||||
/// whose footer remains valid while one of its frames no longer does.
|
||||
///
|
||||
/// This is generated from a segment the drive actually sealed and installed,
|
||||
/// and the assertion is against the same `reconcile` command the matrix and
|
||||
/// recovery script use. There is no harness-only validator in between. The
|
||||
/// pre-fix Wave A reopen accepted this exact class by trusting only footer
|
||||
/// offsets, so reverting that validation makes this test fail.
|
||||
#[test]
|
||||
fn production_recovery_rejects_a_corrupt_frame_in_a_sealed_segment() {
|
||||
let directory = tempfile::tempdir().expect("tempdir");
|
||||
let root = directory.path().join("root");
|
||||
let ack = directory.path().join("ack.journal");
|
||||
|
||||
let seed = std::process::Command::new(harness::DRIVER)
|
||||
.args(["append", "--root"])
|
||||
.arg(&root)
|
||||
.args([
|
||||
"--shard",
|
||||
"0",
|
||||
"--shard-count",
|
||||
"1",
|
||||
"--create",
|
||||
"--seed",
|
||||
"49374",
|
||||
"--group-len",
|
||||
"2",
|
||||
"--point",
|
||||
"BeforeAppend",
|
||||
"--action",
|
||||
"continue",
|
||||
"--fault",
|
||||
"none",
|
||||
])
|
||||
.arg("--ack-journal")
|
||||
.arg(&ack)
|
||||
.arg("--seal")
|
||||
.output()
|
||||
.expect("seeding sealed segment");
|
||||
assert!(
|
||||
seed.status.success(),
|
||||
"the source must be a production-sealed segment.\nstdout:\n{}\nstderr:\n{}",
|
||||
String::from_utf8_lossy(&seed.stdout),
|
||||
String::from_utf8_lossy(&seed.stderr)
|
||||
);
|
||||
|
||||
let damage = generate_damage(
|
||||
&root,
|
||||
&["--kind", "sealed-frame-corruption", "--shard", "0"],
|
||||
);
|
||||
assert!(
|
||||
damage.status.success(),
|
||||
"the sealed-frame generator must produce its image.\nstdout:\n{}\nstderr:\n{}",
|
||||
String::from_utf8_lossy(&damage.stdout),
|
||||
String::from_utf8_lossy(&damage.stderr)
|
||||
);
|
||||
assert!(
|
||||
String::from_utf8_lossy(&damage.stdout).contains("damage_phase=complete"),
|
||||
"the generator must affirm that it fenced the mutation"
|
||||
);
|
||||
|
||||
let reconcile = harness::run_reconcile(&root, 0, &ack);
|
||||
assert_eq!(
|
||||
reconcile.exit_code,
|
||||
Some(65),
|
||||
"production recovery must refuse the corrupt authority, not panic or \
|
||||
publish it. stderr:\n{}",
|
||||
reconcile.stderr
|
||||
);
|
||||
assert_eq!(reconcile.get("recovery_ok"), Some("false"));
|
||||
assert!(
|
||||
reconcile
|
||||
.get("recovery_error")
|
||||
.is_some_and(|error| error.contains("frame digest does not recompute")),
|
||||
"the refusal must come from frame validation, not an unrelated setup \
|
||||
failure: {:?}",
|
||||
reconcile.get("recovery_error")
|
||||
);
|
||||
assert_eq!(
|
||||
reconcile.get("adopted_sequences"),
|
||||
None,
|
||||
"a rejected recovery must publish no partial adoption result"
|
||||
);
|
||||
}
|
||||
|
||||
/// Wave A review finding 1, physical image 2: a valid journal from shard 1
|
||||
/// moved beneath shard 0 of the same root.
|
||||
///
|
||||
/// Root UUID validation alone cannot catch this. Production recovery must bind
|
||||
/// the journal header's shard index to the directory being recovered. The
|
||||
/// pre-fix Wave A reopen adopted the moved journal whole, so this is the second
|
||||
/// generator needed for the matrix to catch that original blocker.
|
||||
#[test]
|
||||
fn production_recovery_rejects_a_journal_moved_between_shards() {
|
||||
let directory = tempfile::tempdir().expect("tempdir");
|
||||
let root = directory.path().join("root");
|
||||
let ack = directory.path().join("ack.journal");
|
||||
|
||||
let seed = std::process::Command::new(harness::DRIVER)
|
||||
.args(["append", "--root"])
|
||||
.arg(&root)
|
||||
.args([
|
||||
"--shard",
|
||||
"1",
|
||||
"--shard-count",
|
||||
"2",
|
||||
"--create",
|
||||
"--seed",
|
||||
"49375",
|
||||
"--group-len",
|
||||
"2",
|
||||
"--point",
|
||||
"BeforeAppend",
|
||||
"--action",
|
||||
"continue",
|
||||
"--fault",
|
||||
"none",
|
||||
])
|
||||
.arg("--ack-journal")
|
||||
.arg(&ack)
|
||||
.output()
|
||||
.expect("seeding shard-1 journal");
|
||||
assert!(
|
||||
seed.status.success(),
|
||||
"the source must be a production-written shard-1 journal.\nstdout:\n{}\nstderr:\n{}",
|
||||
String::from_utf8_lossy(&seed.stdout),
|
||||
String::from_utf8_lossy(&seed.stderr)
|
||||
);
|
||||
|
||||
let damage = generate_damage(
|
||||
&root,
|
||||
&[
|
||||
"--kind",
|
||||
"cross-shard-journal-movement",
|
||||
"--source-shard",
|
||||
"1",
|
||||
"--destination-shard",
|
||||
"0",
|
||||
],
|
||||
);
|
||||
assert!(
|
||||
damage.status.success(),
|
||||
"the cross-shard generator must produce its image.\nstdout:\n{}\nstderr:\n{}",
|
||||
String::from_utf8_lossy(&damage.stdout),
|
||||
String::from_utf8_lossy(&damage.stderr)
|
||||
);
|
||||
|
||||
let reconcile = harness::run_reconcile(&root, 0, &ack);
|
||||
assert_eq!(
|
||||
reconcile.exit_code,
|
||||
Some(65),
|
||||
"production recovery must refuse the moved journal, not panic or adopt \
|
||||
it. stderr:\n{}",
|
||||
reconcile.stderr
|
||||
);
|
||||
assert_eq!(reconcile.get("recovery_ok"), Some("false"));
|
||||
assert!(
|
||||
reconcile
|
||||
.get("recovery_error")
|
||||
.is_some_and(|error| error.contains("journal belongs to shard 1, not shard 0")),
|
||||
"the refusal must be the cross-shard binding check: {:?}",
|
||||
reconcile.get("recovery_error")
|
||||
);
|
||||
assert_eq!(
|
||||
reconcile.get("adopted_sequences"),
|
||||
None,
|
||||
"a rejected recovery must publish no partial adoption result"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enospc_during_append_is_refused_and_leaves_an_openable_store() {
|
||||
let directory = tempfile::tempdir().expect("tempdir");
|
||||
|
|
|
|||
Loading…
Reference in New Issue