diff --git a/bench/result-schema.json b/bench/result-schema.json index b427747..57ac257 100644 --- a/bench/result-schema.json +++ b/bench/result-schema.json @@ -14,6 +14,7 @@ "workload", "hardware", "deployment", + "run_conditions", "measurement", "counts", "bytes", @@ -65,6 +66,9 @@ "deployment": { "$ref": "#/$defs/deployment" }, + "run_conditions": { + "$ref": "#/$defs/run_conditions" + }, "measurement": { "$ref": "#/$defs/measurement" }, @@ -465,10 +469,12 @@ ], "properties": { "persistent_data_mount": { - "const": true + "type": "boolean", + "description": "Re-pinned to true by the \"reference_profile\" rule in allOf, and a passing run must declare that fidelity. Relaxed from an unconditional const so a diagnostic run on a non-persistent mount is representable-and-disqualified rather than unrepresentable (contract review 2026-07-28-C)." }, "tmpfs": { - "const": false + "type": "boolean", + "description": "Re-pinned to false by the \"reference_profile\" rule in allOf. A tmpfs run can never be a pass, because outcome \"pass\" requires reference_profile fidelity, which requires tmpfs false." }, "overlay": { "const": false @@ -502,6 +508,103 @@ } } }, + "run_conditions": { + "type": "object", + "additionalProperties": false, + "description": "The conditions the run was obtained under, as values a consumer can check. Contract review 2026-07-28-C: a bundle whose caveats live only in a human report reads as unconditional to everyone who receives it, and a free-text caveat field is not a condition anything can check. Every field is a closed enumeration or a boolean; there is no prose member and no catch-all value. The verification claims in $defs.verification are conditioned on these declarations, so a run that did not perform a check has no way to assert the claim that names it.", + "required": [ + "initialization_path", + "mutation_path", + "checkpointing", + "index_maintenance", + "index_run_ceiling", + "receipt_reconciliation", + "objects_new_source", + "commit_id_uniqueness", + "build_profile", + "environment_fidelity" + ], + "properties": { + "initialization_path": { + "enum": [ + "store_engine_open", + "segment_initialize_root", + "shard_drive_create" + ], + "description": "How the store root the run measured was created. Only \"store_engine_open\" is the production entry point, and a passing storage_primitive run must declare it. B1 landed startup state 1, so the submit path now builds its root that way and the earlier segment::initialize_root seeding is retired; \"segment_initialize_root\" is retained because a bundle emitted before that change must still be readable and must still be readable as what it was. The journal seam declares \"shard_drive_create\", which is its own creation path and not a weakening." + }, + "mutation_path": { + "enum": [ + "store_engine_submit", + "journal_drive" + ], + "description": "The entry point the measured transactions actually went through. This is the branch discriminator for the storage_primitive verification rules in allOf: \"journal_drive\" is the Wave A journal seam below engine.rs, which has no sequencer, no receipts, and no index, and may not assert what it cannot observe." + }, + "checkpointing": { + "enum": [ + "exercised", + "enabled_not_reached", + "unimplemented", + "disabled_by_configuration" + ], + "description": "Scope §7 requires that the P2 runs not have been achieved with checkpointing disabled. That clause lived only in prose until this review; it is now mechanical, because a passing storage_primitive run must declare \"exercised\". \"unimplemented\" is today's honest value: StoreEngine::checkpoint returns NotImplemented, so no checkpoint is taken." + }, + "index_maintenance": { + "enum": [ + "runs_sealed", + "deltas_retained_in_memory", + "no_index_in_path" + ], + "description": "Whether the index reached a steady state. \"deltas_retained_in_memory\" is a run that holds every delta layer it published and whose lookup fan-out grows for its whole duration, which is not the steady state a P2 measurement is of; a passing storage_primitive run must declare \"runs_sealed\". \"no_index_in_path\" is the journal seam, where nothing below engine.rs touches an index at all." + }, + "index_run_ceiling": { + "enum": [ + "store_default", + "raised_because_index_sealing_unimplemented" + ], + "description": "Whether StoreOptions::max_index_runs was left at the store default (64) or raised so the run could reach its measured seconds despite unimplemented index-delta sealing. Cross-checked against resources.configured_ceilings.max_index_runs by the two \"index run ceiling\" rules in allOf: \"store_default\" bounds the recorded value at 64 and the raised value floors it at 65, so a bundle that declares one and records the other is invalid. The recorded value must be the value the run configured, never a constant restated here." + }, + "receipt_reconciliation": { + "enum": [ + "exact_receipts_reconciled", + "canonical_receipt_digest_reconciled", + "acceptance_of_any_committed_status", + "no_receipts_in_path" + ], + "description": "What the run reconciled each acknowledged operation against. verification.operation_receipts_reconciled is required exactly when this is one of the first two values and forbidden under the last two. Today's harness reads back TransactionStatus and accepts any Committed(_) payload, and the receipt_digest it journals is a digest of the operation id rather than of the receipt, so it must declare \"acceptance_of_any_committed_status\" and consequently may not assert the claim. \"no_receipts_in_path\" is the journal seam, which produces no receipts to reconcile." + }, + "objects_new_source": { + "enum": [ + "summed_from_receipts", + "derived_from_transaction_count" + ], + "description": "Where counts.objects_new came from. verification.objects_new_equals_three_per_commit is forbidden under \"derived_from_transaction_count\": a harness that computes objects_new as transactions * 3 and then asserts the flag has written a tautology, not a check, and that is what the claim was originally excluded for. \"summed_from_receipts\" means each committed receipt's own objects_new was summed independently and compared against a separately counted 3 * counts.counted_commits." + }, + "commit_id_uniqueness": { + "enum": [ + "checked_globally_across_ack_records", + "inferred_from_seed_domains", + "not_checked" + ], + "description": "How blob, tree, and commit identifier uniqueness was established. verification.unique_blob_tree_commit_ids is forbidden under the last two values. Per-record uniqueness is not uniqueness: two acknowledgment records may each be internally distinct and still share a commit id. Distinct generator seed domains make a collision unlikely rather than absent, which is an argument and not a check, so \"inferred_from_seed_domains\" is named here rather than folded into the passing value." + }, + "build_profile": { + "enum": [ + "debug", + "release", + "release_with_debug_assertions" + ], + "description": "The cargo profile the measured binary was built with. \"reference_profile\" environment fidelity requires \"release\"; a debug run is recordable, and is thereby disqualified from a pass rather than silently comparable to a release number." + }, + "environment_fidelity": { + "enum": [ + "reference_profile", + "diagnostic" + ], + "description": "Whether the run met the frozen environment. \"reference_profile\" re-pins deployment.persistent_data_mount true, deployment.tmpfs false, build_profile \"release\", and a named hardware profile. \"diagnostic\" admits a non-persistent mount or a debug build and is mechanically disqualified: outcome may only be \"fail\" or \"preliminary\" and no verdict may be \"pass\". outcome \"pass\" requires \"reference_profile\" at every gate, so nothing about this relaxes what a claim costs." + } + } + }, "measurement": { "type": "object", "additionalProperties": false, @@ -672,6 +775,16 @@ "configured_ceilings": { "type": "object", "minProperties": 1, + "required": [ + "max_index_runs" + ], + "properties": { + "max_index_runs": { + "type": "integer", + "minimum": 1, + "description": "The StoreOptions::max_index_runs the run actually configured, read back from the options the store opened with — never the store default restated here, and never omitted. Bounded against run_conditions.index_run_ceiling by the two \"index run ceiling\" rules in allOf, so a defaulted or contradicted value is invalid rather than merely unverified. It is a ceiling this workload genuinely reaches: index-delta sealing being unimplemented makes it the limit submit refuses at." + } + }, "additionalProperties": { "type": "number", "minimum": 0 @@ -752,11 +865,11 @@ }, "unique_blob_tree_commit_ids": { "const": true, - "description": "Not applicable at storage_primitive, where no object graph exists; omitted there rather than falsified, since false would claim the check was applicable and failed." + "description": "Every blob, tree, and commit identifier the run acknowledged is distinct, checked globally across every recovered acknowledgment record rather than within each record and never inferred from distinct generator seed domains. Earnable at storage_primitive only through StoreEngine::submit, where the transactions carry real objects; forbidden on the journal seam, which has none. Gated on run_conditions.commit_id_uniqueness (contract review 2026-07-28-C)." }, "objects_new_equals_three_per_commit": { "const": true, - "description": "Counted independently of the transaction total, not derived from it. A harness that computes objects_new as transactions * 3 and then asserts this flag has written a tautology, not a check. Absent at storage_primitive, which creates no objects." + "description": "Receipt-reported objects_new, summed independently across every committed receipt, equals a separately counted 3 * counts.counted_commits. Both sides must be counted; deriving either from the transaction total makes the assertion unfailable, which is what this claim was originally excluded for. Earnable at storage_primitive only through StoreEngine::submit, where the store stages the canonical three objects per commit and reports them on the receipt. Gated on run_conditions.objects_new_source (contract review 2026-07-28-C)." }, "blobs_recomputed": { "const": true, @@ -768,7 +881,7 @@ }, "operation_receipts_reconciled": { "const": true, - "description": "Not applicable at storage_primitive, where no object graph exists; omitted there rather than falsified, since false would claim the check was applicable and failed." + "description": "Every acknowledged operation was read back and its receipt reconciled against the receipt the run recorded when it acknowledged, either exactly or through a frozen canonical receipt digest. Approved in principle for the StoreEngine::submit path and NOT EARNED by the emitter as it stands: store-bench accepts any TransactionStatus::Committed(_) payload without comparing it, and the receipt_digest it journals is a digest of the operation id rather than of the receipt, so it must declare run_conditions.receipt_reconciliation = \"acceptance_of_any_committed_status\" and is thereby forbidden from asserting this. Expressible now so that landing the reconciliation is an emitter change and not a second schema amendment (contract review 2026-07-28-C)." }, "metadata_complete": { "const": true, @@ -1018,26 +1131,11 @@ "verification": { "not": { "anyOf": [ - { - "required": [ - "unique_blob_tree_commit_ids" - ] - }, - { - "required": [ - "objects_new_equals_three_per_commit" - ] - }, { "required": [ "blobs_recomputed" ] }, - { - "required": [ - "operation_receipts_reconciled" - ] - }, { "required": [ "metadata_complete" @@ -1054,13 +1152,33 @@ "commits_in_recovered_closure" ] } - } + }, + "description": "The claims forbidden at this gate on every path, whichever entry point was measured. blobs_recomputed and metadata_complete require an object graph the store is forbidden to traverse (plan §5.1), and commits_in_recovered_closure requires the ref closure that traversal would produce, so neither path can earn them and this is not branch-conditional. The three claims that became earnable through StoreEngine::submit are ruled on by the two mutation-path rules below." }, "else": { "properties": { "promotable": { "const": true }, + "run_conditions": { + "properties": { + "initialization_path": { + "const": "store_engine_open" + }, + "mutation_path": { + "const": "store_engine_submit" + }, + "checkpointing": { + "enum": [ + "exercised", + "enabled_not_reached" + ] + }, + "index_maintenance": { + "const": "runs_sealed" + } + } + }, "workload": { "properties": { "validation_flags": { @@ -1142,6 +1260,501 @@ } } }, + { + "title": "the journal-drive seam may not assert what it cannot observe", + "description": "Contract review 2026-07-28-C. The three claims that became earnable through StoreEngine::submit stay forbidden here, and the provenance declarations are pinned to the only values the seam can truthfully make. Forbidding the claims alone would not be enough: a drive-path bundle could otherwise declare exact receipt reconciliation or a global uniqueness check it has no receipts and no objects to perform, and the pins are what make that combination invalid rather than merely unverified.", + "if": { + "properties": { + "gate": { + "const": "storage_primitive" + }, + "run_conditions": { + "properties": { + "mutation_path": { + "const": "journal_drive" + } + }, + "required": [ + "mutation_path" + ] + } + }, + "required": [ + "gate", + "run_conditions" + ] + }, + "then": { + "properties": { + "verification": { + "not": { + "anyOf": [ + { + "required": [ + "unique_blob_tree_commit_ids" + ] + }, + { + "required": [ + "objects_new_equals_three_per_commit" + ] + }, + { + "required": [ + "operation_receipts_reconciled" + ] + } + ] + } + }, + "run_conditions": { + "properties": { + "index_maintenance": { + "const": "no_index_in_path" + }, + "receipt_reconciliation": { + "const": "no_receipts_in_path" + }, + "objects_new_source": { + "const": "derived_from_transaction_count" + }, + "commit_id_uniqueness": { + "const": "not_checked" + } + } + } + } + } + }, + { + "title": "the production submit path at storage_primitive earns two claims and must state them", + "description": "Contract review 2026-07-28-C. StoreEngine::submit stages the canonical three objects per commit and reports them on its own receipt, and the transactions carry real blob, tree, and commit identifiers, so these two claims are no longer inapplicable — they are required, with the provenance that makes each a check rather than a restatement. operation_receipts_reconciled is deliberately absent from this list: it is approved in principle and not yet earned, and the receipt-reconciliation rule below is what decides it.", + "if": { + "properties": { + "gate": { + "const": "storage_primitive" + }, + "run_conditions": { + "properties": { + "mutation_path": { + "const": "store_engine_submit" + } + }, + "required": [ + "mutation_path" + ] + } + }, + "required": [ + "gate", + "run_conditions" + ] + }, + "then": { + "properties": { + "verification": { + "properties": { + "unique_blob_tree_commit_ids": { + "const": true + }, + "objects_new_equals_three_per_commit": { + "const": true + } + }, + "required": [ + "unique_blob_tree_commit_ids", + "objects_new_equals_three_per_commit" + ] + }, + "run_conditions": { + "properties": { + "initialization_path": { + "enum": [ + "store_engine_open", + "segment_initialize_root" + ] + }, + "index_maintenance": { + "enum": [ + "runs_sealed", + "deltas_retained_in_memory" + ] + }, + "receipt_reconciliation": { + "enum": [ + "exact_receipts_reconciled", + "canonical_receipt_digest_reconciled", + "acceptance_of_any_committed_status" + ] + } + } + } + } + } + }, + { + "title": "objects_new_equals_three_per_commit requires an independently summed objects_new", + "description": "Applies at every gate, not only storage_primitive. A count derived from the transaction total makes the claim unfailable wherever it is asserted.", + "if": { + "properties": { + "run_conditions": { + "properties": { + "objects_new_source": { + "const": "derived_from_transaction_count" + } + }, + "required": [ + "objects_new_source" + ] + } + }, + "required": [ + "run_conditions" + ] + }, + "then": { + "properties": { + "verification": { + "not": { + "required": [ + "objects_new_equals_three_per_commit" + ] + } + } + } + } + }, + { + "title": "unique_blob_tree_commit_ids requires a global uniqueness check", + "description": "Applies at every gate. Per-record uniqueness and distinct seed domains are both weaker than the claim: the first cannot see a collision between two records, and the second is an argument about likelihood rather than an observation.", + "if": { + "properties": { + "run_conditions": { + "properties": { + "commit_id_uniqueness": { + "enum": [ + "inferred_from_seed_domains", + "not_checked" + ] + } + }, + "required": [ + "commit_id_uniqueness" + ] + } + }, + "required": [ + "run_conditions" + ] + }, + "then": { + "properties": { + "verification": { + "not": { + "required": [ + "unique_blob_tree_commit_ids" + ] + } + } + } + } + }, + { + "title": "operation_receipts_reconciled is forbidden where no receipt was reconciled", + "description": "Contract review 2026-07-28-C, and the reason the claim can be landed before it is earned. Accepting any Committed status is not reconciling a receipt; it reports that something committed, which acknowledged_sequences_reconciled already says. The journal seam declares no_receipts_in_path and is covered by the same clause. Both values are named here and both earning values are named in the companion rule below, so neither rule has an else branch that would fire on a bundle with no declaration at all.", + "if": { + "properties": { + "run_conditions": { + "properties": { + "receipt_reconciliation": { + "enum": [ + "acceptance_of_any_committed_status", + "no_receipts_in_path" + ] + } + }, + "required": [ + "receipt_reconciliation" + ] + } + }, + "required": [ + "run_conditions" + ] + }, + "then": { + "properties": { + "verification": { + "not": { + "required": [ + "operation_receipts_reconciled" + ] + } + } + } + } + }, + { + "title": "operation_receipts_reconciled is required where a receipt was reconciled", + "description": "A run that reconciled exact receipts, or a frozen canonical receipt digest, must assert the claim rather than leave it optional: the check was performed and its result is a required part of the record. On the journal-drive path this rule and the mutation-path rule combine to make an exact-receipt declaration unsatisfiable, which is the intended reading — a seam with no receipts cannot have reconciled any. Stated as its own rule rather than as the else of the rule above so that neither fires on a bundle carrying no declaration at all: absence of run_conditions is refused by the top-level required list, and one refusal reported once is worth more than the same defect reported under a claim the run never mentioned.", + "if": { + "properties": { + "run_conditions": { + "properties": { + "receipt_reconciliation": { + "enum": [ + "exact_receipts_reconciled", + "canonical_receipt_digest_reconciled" + ] + } + }, + "required": [ + "receipt_reconciliation" + ] + } + }, + "required": [ + "run_conditions" + ] + }, + "then": { + "properties": { + "verification": { + "properties": { + "operation_receipts_reconciled": { + "const": true + } + }, + "required": [ + "operation_receipts_reconciled" + ] + } + } + } + }, + { + "title": "index run ceiling: a declared store default bounds the recorded value", + "if": { + "properties": { + "run_conditions": { + "properties": { + "index_run_ceiling": { + "const": "store_default" + } + }, + "required": [ + "index_run_ceiling" + ] + } + }, + "required": [ + "run_conditions" + ] + }, + "then": { + "properties": { + "resources": { + "properties": { + "configured_ceilings": { + "properties": { + "max_index_runs": { + "maximum": 64 + } + } + } + } + } + } + } + }, + { + "title": "index run ceiling: a declared raise must record a raised value", + "if": { + "properties": { + "run_conditions": { + "properties": { + "index_run_ceiling": { + "const": "raised_because_index_sealing_unimplemented" + } + }, + "required": [ + "index_run_ceiling" + ] + } + }, + "required": [ + "run_conditions" + ] + }, + "then": { + "properties": { + "resources": { + "properties": { + "configured_ceilings": { + "properties": { + "max_index_runs": { + "minimum": 65 + } + } + } + } + } + } + } + }, + { + "title": "reference-profile fidelity re-pins the environment the deployment schema used to pin outright", + "description": "Contract review 2026-07-28-C. deployment.persistent_data_mount and deployment.tmpfs were unconditional consts, which made a diagnostic tmpfs run unrepresentable rather than disqualified. They are re-pinned here, and outcome \"pass\" requires this fidelity at every gate, so nothing a claim used to cost has changed.", + "if": { + "properties": { + "run_conditions": { + "properties": { + "environment_fidelity": { + "const": "reference_profile" + } + }, + "required": [ + "environment_fidelity" + ] + } + }, + "required": [ + "run_conditions" + ] + }, + "then": { + "properties": { + "deployment": { + "properties": { + "persistent_data_mount": { + "const": true + }, + "tmpfs": { + "const": false + } + } + }, + "run_conditions": { + "properties": { + "build_profile": { + "const": "release" + } + } + }, + "hardware": { + "properties": { + "profile": { + "enum": [ + "minimum-30k", + "release-60k" + ] + } + } + } + } + } + }, + { + "title": "a diagnostic run is representable and mechanically disqualified", + "description": "Contract review 2026-07-28-C. The alternative was what stood before it: a debug, tmpfs diagnostic run could not be encoded at all, so its number lived on a console and in prose. Recording it is worth nothing unless the record also refuses to let it be read as a result, so the outcome is bounded and no verdict may be pass.", + "if": { + "properties": { + "run_conditions": { + "properties": { + "environment_fidelity": { + "const": "diagnostic" + } + }, + "required": [ + "environment_fidelity" + ] + } + }, + "required": [ + "run_conditions" + ] + }, + "then": { + "properties": { + "outcome": { + "enum": [ + "fail", + "preliminary" + ] + }, + "verdicts": { + "additionalProperties": { + "enum": [ + "fail", + "not-applicable" + ] + } + } + } + } + }, + { + "title": "a passing run at any gate must have met the reference environment", + "if": { + "properties": { + "outcome": { + "const": "pass" + } + }, + "required": [ + "outcome" + ] + }, + "then": { + "properties": { + "run_conditions": { + "properties": { + "environment_fidelity": { + "const": "reference_profile" + } + } + } + } + } + }, + { + "title": "a passing storage_primitive run must have measured the production path in a steady state", + "description": "Scope §7 requires the P2 runs not to have been achieved with checkpointing disabled, and a run holding every index delta in memory for its whole duration is not the steady state a P2 number describes. Both were prose until contract review 2026-07-28-C; here they are conditions a consumer checks. A root seeded outside StoreEngine::open is disclosed rather than forbidden — but it cannot ride into a pass.", + "if": { + "properties": { + "gate": { + "const": "storage_primitive" + }, + "outcome": { + "const": "pass" + } + }, + "required": [ + "gate", + "outcome" + ] + }, + "then": { + "properties": { + "run_conditions": { + "properties": { + "initialization_path": { + "const": "store_engine_open" + }, + "mutation_path": { + "const": "store_engine_submit" + }, + "checkpointing": { + "const": "exercised" + }, + "index_maintenance": { + "const": "runs_sealed" + } + } + } + } + } + }, { "title": "a passing run must meet the section 3 one-minute-window rule", "if": { diff --git a/crates/levcs-protocol/tests/phase0_benchmark_contracts.rs b/crates/levcs-protocol/tests/phase0_benchmark_contracts.rs index 90a196d..c32f9dc 100644 --- a/crates/levcs-protocol/tests/phase0_benchmark_contracts.rs +++ b/crates/levcs-protocol/tests/phase0_benchmark_contracts.rs @@ -157,9 +157,14 @@ fn result_schema_requires_integrity_durability_and_all_independent_verdicts() { schema["$defs"]["workload"]["properties"]["writer_group_limit"]["maximum"].as_u64(), Some(512) ); + // `tmpfs` and `persistent_data_mount` stopped being unconditional consts in + // contract review 2026-07-28-C so a diagnostic run is representable. They + // are re-pinned under `reference_profile` fidelity, which a passing run + // must declare — asserted in full by + // `a_diagnostic_environment_is_representable_but_can_never_pass`. assert_eq!( - schema["$defs"]["deployment"]["properties"]["tmpfs"]["const"].as_bool(), - Some(false) + schema["$defs"]["deployment"]["properties"]["tmpfs"]["type"].as_str(), + Some("boolean") ); assert_eq!( schema["$defs"]["deployment"]["properties"]["remote_storage"]["const"].as_bool(), @@ -225,11 +230,13 @@ fn result_schema_requires_integrity_durability_and_all_independent_verdicts() { .iter() .find(|rule| { // The per-gate latency ceilings became `outcome`-conditional too, - // so `outcome == "pass"` alone now matches four other rules. The - // window rule is the one that applies to every gate: it keys on - // `outcome` and nothing else. + // so `outcome == "pass"` alone now matches four other rules, and + // since contract review 2026-07-28-C the environment-fidelity rule + // keys on `outcome` and nothing else as well. The window rule is the + // gate-independent one that constrains `measurement`. rule["if"]["properties"]["outcome"]["const"] == "pass" && rule["if"]["properties"]["gate"].is_null() + && !rule["then"]["properties"]["measurement"].is_null() }) .expect("missing the outcome-conditional window rule"); assert_eq!( @@ -455,10 +462,17 @@ fn verification_claims_are_pinned_per_gate_like_validation_flags() { "every instance gate must still re-pin closure to true" ); - // The five object-graph claims are *forbidden* at storage_primitive, not - // merely optional. There is no object graph at this layer, so `false` - // would be its own untrue statement: it claims the check applied and did - // not pass. Absence is the only honest encoding. + // Two of the five object-graph claims are forbidden at storage_primitive on + // *every* path, not merely optional: no path below the instance can + // recompute a blob or complete object metadata, so `false` would be its own + // untrue statement — it claims the check applied and did not pass. Absence + // is the only honest encoding. + // + // The other three moved to the mutation-path rules in contract review + // 2026-07-28-C, because `StoreEngine::submit` genuinely earns two of them + // and can express the third. They are still forbidden on the journal-drive + // seam, which is pinned by + // `the_drive_path_may_not_assert_the_claims_the_submit_path_earns`. const GRAPH_CLAIMS: &[&str] = &[ "unique_blob_tree_commit_ids", "objects_new_equals_three_per_commit", @@ -466,15 +480,22 @@ fn verification_claims_are_pinned_per_gate_like_validation_flags() { "operation_receipts_reconciled", "metadata_complete", ]; + const FORBIDDEN_ON_EVERY_PATH: &[&str] = &["blobs_recomputed", "metadata_complete"]; let forbidden = rule["then"]["properties"]["verification"]["not"]["anyOf"] .as_array() - .expect("storage_primitive must forbid the object-graph claims outright"); - for claim in GRAPH_CLAIMS { + .expect("storage_primitive must forbid the unearnable claims outright"); + assert_eq!( + forbidden.len(), + FORBIDDEN_ON_EVERY_PATH.len(), + "the gate-wide forbidden set must be exactly the claims no path can earn; \ + anything else belongs in a mutation-path rule where the path is named" + ); + for claim in FORBIDDEN_ON_EVERY_PATH { assert!( forbidden .iter() .any(|clause| clause["required"][0] == serde_json::json!(claim)), - "{claim} must be forbidden at storage_primitive, not left optional" + "{claim} must be forbidden at storage_primitive on every path" ); } @@ -738,3 +759,757 @@ fn federation_workload_freezes_projection_rtt_partition_and_digest_gates() { Some(true) ); } + +// --------------------------------------------------------------------------- +// Contract review 2026-07-28-C: the run-conditions block and the two +// mutation-path branches +// --------------------------------------------------------------------------- +// +// Scope §6.6 requires contract tests for both the submit and journal-drive +// branches. The tests above assert the *shape* of the schema — which rule +// exists, what it pins — which is necessary and is not sufficient: a +// conditional can be structurally present and still admit the document it was +// written to reject. Everything below validates whole bundles through the same +// validator `scripts/verify-store-recovery.sh` and `store-bench`'s own tests +// use, so the gate, the emitter, and this file cannot disagree about what valid +// means. +// +// There is no JSON Schema crate in the workspace and adding one is a +// `Cargo.toml` change; validation therefore shells out to `python3` with +// `jsonschema`. A missing interpreter or module is a test *failure*, never a +// skip. A validation that silently does not run reads as a passing suite, which +// is exactly how a schema stops being a contract. + +/// Exit 0 clean, exit 1 with one error per line on stdout, exit 2 if the +/// validator itself is unavailable. +const VALIDATE_PY: &str = "\ +import json, sys +try: + import jsonschema +except ImportError: + sys.stderr.write('jsonschema is not installed\\n') + sys.exit(2) +schema = json.load(open(sys.argv[1])) +instance = json.load(open(sys.argv[2])) +validator = jsonschema.Draft202012Validator( + schema, format_checker=jsonschema.FormatChecker() +) +errors = sorted(validator.iter_errors(instance), key=lambda e: list(e.path)) +for error in errors: + sys.stdout.write(f'{list(error.path)}: {error.message}\\n') +sys.exit(1 if errors else 0) +"; + +/// Every validator error for `bundle`, or an empty vector. +fn schema_errors(bundle: &serde_json::Value) -> Vec { + let directory = tempfile::tempdir().expect("tempdir"); + let instance = directory.path().join("bundle.json"); + std::fs::write( + &instance, + serde_json::to_vec_pretty(bundle).expect("encode"), + ) + .expect("write"); + let schema = repository_root().join("bench/result-schema.json"); + + let output = std::process::Command::new("python3") + .arg("-c") + .arg(VALIDATE_PY) + .arg(&schema) + .arg(&instance) + .output() + .expect( + "python3 must be available: these tests validate bundles against \ + bench/result-schema.json, and a validation that cannot run is not a \ + passing test", + ); + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + match output.status.code() { + Some(0) => Vec::new(), + Some(1) => stdout.lines().map(str::to_string).collect(), + other => panic!( + "the schema validator could not run (exit {other:?}). jsonschema must be \ + installed; a skipped validation would let the schema and the emitter \ + drift with nothing to notice.\nstdout: {stdout}\nstderr: {stderr}" + ), + } +} + +fn assert_valid(bundle: &serde_json::Value, why: &str) { + let errors = schema_errors(bundle); + assert!(errors.is_empty(), "{why}\n{}", errors.join("\n")); +} + +fn assert_invalid(bundle: &serde_json::Value, why: &str) { + assert!(!schema_errors(bundle).is_empty(), "{why}"); +} + +const HEX64: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +/// A bundle that a `--path submit` run can honestly emit *today*, and the exact +/// shape §6.6's emitter follow-up must produce. +/// +/// Every declaration in `run_conditions` is the truthful value for the emitter +/// as it stands at `store-bench.rs`: the root is seeded by +/// `segment::initialize_root`, `StoreEngine::checkpoint` is unimplemented, the +/// index-run ceiling is raised to 1,000,000 because delta sealing is +/// unimplemented, and the reconciliation accepts any `Committed(_)` status. So +/// the fixture is not a hypothetical: if this validates and the emitter's output +/// does not, the difference is the emitter's to close. +fn submit_path_bundle() -> serde_json::Value { + serde_json::json!({ + "schema_version": 1, + "gate": "storage_primitive", + "run_id": "engine-wave-b-0123456789abcdef", + "attestation": { + "signer": format!("ed25519:{HEX64}"), + "key_epoch": 0, + "content_digest": HEX64, + "signature": format!("{HEX64}{HEX64}") + }, + "source": { + "revision": "e050b6de050b6de050b6de050b6de050b6de050b", + "dirty_tree_digest": HEX64, + "cargo_lock_digest": HEX64, + "rustc": "rustc 1.90.0", + "rustflags": "" + }, + "artifacts": { + "binary_digest": HEX64, + "config_digest": HEX64, + "workload_digest": HEX64, + "corpus_digest": HEX64, + "raw_metrics_digest": HEX64, + "telemetry_versions": { "store-bench": "0.1.0" } + }, + "workload": { + "name": "canonical-small-commit", + "seed": 126_394_451_485_337i64, + "topology": "many-repo", + "selection": "uniform", + "client_batch_commits": 1, + "writer_group_limit": 512, + "persistent_clients": 64, + "validation_flags": { + "request_signature": false, + "replay": false, + "pack_hash_and_framing": false, + "outer_embedded_type_match": false, + "complete_graph": false, + "authority_and_role": false, + "instance_policy": false, + "repository_policy": false, + "typed_ref_cas": true, + "fast_forward": false, + "durability_fence_before_response": true + }, + "generator": "blake3-xof(seed || repo_ordinal_le || ref_ordinal_le || commit_ordinal_le)" + }, + "hardware": { + "profile": "diagnostic", + "cpu": "AMD Ryzen 7 9800X3D", + "numa": "nodes=1", + "governor": "performance", + "microcode": "0x0b404023", + "ram_bytes": 68_719_476_736i64, + "swap_events": 0, + "filesystem": "btrfs", + "mount_options": ["nodatacow"], + "nvme": "Samsung SSD 990 PRO 1TB", + "firmware": "4B2QJXD7", + "write_cache": "enabled", + "barriers": "enabled", + "scheduler": "none", + "temperature_celsius": 41.0, + "nic": "none (in-process P2)", + "driver": "none (in-process P2)", + "link_mbps": 1, + "mtu": 1500, + "kernel": "Linux 6.18.30-p1-gentoo-dist" + }, + "deployment": { + "persistent_data_mount": true, + "tmpfs": false, + "overlay": false, + "remote_storage": false, + "durability_enabled": true, + "systemd": "none (in-process P2)", + "cgroup": "none (in-process P2)", + "proxy": "none (in-process P2)", + "tls": "none (in-process P2)", + "store_directory_attributes": "nodatacow" + }, + "run_conditions": { + "initialization_path": "segment_initialize_root", + "mutation_path": "store_engine_submit", + "checkpointing": "unimplemented", + "index_maintenance": "deltas_retained_in_memory", + "index_run_ceiling": "raised_because_index_sealing_unimplemented", + "receipt_reconciliation": "acceptance_of_any_committed_status", + "objects_new_source": "summed_from_receipts", + "commit_id_uniqueness": "checked_globally_across_ack_records", + "build_profile": "debug", + "environment_fidelity": "diagnostic" + }, + "measurement": { + "warmup_seconds": 0, + "measured_seconds": 2, + "repetition": 1, + "started_at": "2026-07-28T00:00:00Z", + "ended_at": "2026-07-28T00:00:02Z", + "one_minute_windows": [8052.0], + "histogram_format": "ascending-micros-csv/blake3", + "coordinated_omission_corrected": false, + "windows_meeting_target_percent": 100.0, + "windows_below_floor_count": 0 + }, + "counts": { + "offered_requests": 16104, + "accepted_requests": 16104, + "rejected_requests": 0, + "duplicate_requests": 0, + "acknowledged_requests": 16104, + "counted_commits": 16104, + "objects_new": 48312 + }, + "bytes": { "raw": 21_004_800, "pack_compressed": 0, "application": 21_004_800, "wire": 0 }, + "latency_micros": { + "p50": 900, "p95": 3000, "p99": 7000, "max": 40000, + "histogram_digest": HEX64 + }, + "resources": { + "configured_ceilings": { + "max_index_runs": 1_000_000, + "writer_group_transactions": 512.0, + "journal_preallocate_bytes": 67_108_864.0, + "free_space_required_bytes": 1_073_741_824.0, + "latency_p99_ceiling_micros": 50_000.0 + }, + "observed_peaks": { "free_space_available_bytes": 1_099_511_627_776.0 }, + "time_series_digest": HEX64, + "cpu_percent": 0.0, + "storage_utilization_percent": 0.0, + "memory_current_bytes": 0, + "open_fds": 0, + "compaction_debt_returned_low": true, + "no_growth_passed": true + }, + "durability": { + "external_ack_journal_digest": HEX64, + "ack_journal_fenced_before_count": true, + "recovery_reconciled": true, + "acknowledged_loss": 0, + "torn_transactions": 0 + }, + "verification": { + "setup_traffic_excluded": true, + "commits_in_recovered_closure": false, + "acknowledged_sequences_reconciled": true, + "unique_blob_tree_commit_ids": true, + "objects_new_equals_three_per_commit": true + }, + "verdicts": { + "storage_primitive": "not-applicable", + "in_process_protocol": "not-applicable", + "deployed_30k": "not-applicable", + "deployed_60k": "not-applicable", + "recovery": "not-applicable", + "overload": "not-applicable", + "compaction": "not-applicable", + "federation": "not-applicable", + "release": "not-applicable" + }, + "promotable": false, + "outcome": "preliminary", + "storage": { + "index_bytes_per_object": 47.0, + "checkpoint_lookup_fanout": 1.0, + "evidence_signing_micros_p50": 12.0, + "fences": 1007, + "transactions": 16104, + "trim_settle_seconds": 0.0, + "store_directory_attributes_verified": true + } + }) +} + +/// The Wave A journal seam. The same bundle with every declaration reduced to +/// what a path below `engine.rs` can observe, and the three claims gone. +fn drive_path_bundle() -> serde_json::Value { + let mut bundle = submit_path_bundle(); + bundle["run_id"] = serde_json::json!("skeleton-wave-a-0123456789abcdef"); + bundle["run_conditions"] = serde_json::json!({ + "initialization_path": "shard_drive_create", + "mutation_path": "journal_drive", + "checkpointing": "unimplemented", + "index_maintenance": "no_index_in_path", + "index_run_ceiling": "store_default", + "receipt_reconciliation": "no_receipts_in_path", + "objects_new_source": "derived_from_transaction_count", + "commit_id_uniqueness": "not_checked", + "build_profile": "debug", + "environment_fidelity": "diagnostic" + }); + bundle["resources"]["configured_ceilings"]["max_index_runs"] = serde_json::json!(64); + bundle["storage"]["evidence_signing_micros_p50"] = serde_json::json!(0.0); + bundle["verification"] = serde_json::json!({ + "setup_traffic_excluded": true, + "commits_in_recovered_closure": false, + "acknowledged_sequences_reconciled": true + }); + bundle +} + +/// The negative control for every test below it. +/// +/// Without it, a `schema_errors` that returns empty for all input — a validator +/// that never ran, a schema that failed to load — reads as a green suite. The +/// substring version of the emitter's own tests stayed green against a bundle +/// with four validator errors for exactly this reason. +#[test] +fn both_reference_bundles_validate_and_the_validator_can_still_fail() { + assert_valid( + &submit_path_bundle(), + "the submit-path reference bundle must validate; it is the shape the \ + emitter follow-up has to produce", + ); + assert_valid( + &drive_path_bundle(), + "the journal-seam reference bundle must validate; the drive path stays \ + emittable after the amendment", + ); + + let mut broken = submit_path_bundle(); + broken["storage"] = serde_json::Value::Null; + assert_invalid( + &broken, + "the validator must reject something, or every assertion below is vacuous", + ); +} + +/// Amendment 1. The run conditions are required and enumerated, never prose. +#[test] +fn a_bundle_without_machine_readable_run_conditions_is_not_a_bundle() { + let text = std::fs::read_to_string(repository_root().join("bench/result-schema.json")).unwrap(); + let schema: serde_json::Value = serde_json::from_str(&text).unwrap(); + + assert!( + schema["required"] + .as_array() + .unwrap() + .iter() + .any(|v| v == "run_conditions"), + "run_conditions must be required at every gate: a bundle whose caveats \ + live only in a human report reads as unconditional to everyone who \ + receives it" + ); + assert_required_names( + &schema, + &["$defs", "run_conditions"], + &[ + "initialization_path", + "mutation_path", + "checkpointing", + "index_maintenance", + "index_run_ceiling", + "receipt_reconciliation", + "objects_new_source", + "commit_id_uniqueness", + "build_profile", + "environment_fidelity", + ], + ); + + // Charter item 6: every value named, no catch-all, and no free-text member + // a consumer would have to read rather than check. + let properties = schema["$defs"]["run_conditions"]["properties"] + .as_object() + .expect("run_conditions must declare its properties"); + for (name, spec) in properties { + assert!( + spec.get("enum").is_some() || spec["type"] == "boolean", + "run_conditions.{name} must be a closed enumeration or a boolean; a \ + free-text caveat is not a condition anything can check" + ); + if let Some(values) = spec["enum"].as_array() { + for value in values { + let value = value.as_str().unwrap_or_default(); + assert!( + !value.is_empty() && value != "other" && value != "unknown", + "run_conditions.{name} must not carry a catch-all value" + ); + } + } + } + + // Every field is enforced, not merely recorded: a bundle missing any one of + // them is invalid. + for field in properties.keys() { + let mut bundle = submit_path_bundle(); + bundle["run_conditions"] + .as_object_mut() + .unwrap() + .remove(field.as_str()); + assert_invalid( + &bundle, + &format!("a bundle omitting run_conditions.{field} must be rejected"), + ); + } + + let mut absent = submit_path_bundle(); + absent.as_object_mut().unwrap().remove("run_conditions"); + assert_invalid( + &absent, + "a bundle with no run_conditions block must be rejected", + ); +} + +/// Amendments 2 and 4, and the half of amendment 3 that is landable. +/// +/// The submit path earns `unique_blob_tree_commit_ids` and +/// `objects_new_equals_three_per_commit` and must state them. It does not earn +/// `operation_receipts_reconciled` — and the schema says so mechanically rather +/// than in a comment, because the emitter can only declare the reconciliation +/// it performed and that declaration forbids the claim. +#[test] +fn the_submit_path_must_state_the_claims_it_now_earns() { + for claim in [ + "unique_blob_tree_commit_ids", + "objects_new_equals_three_per_commit", + ] { + let mut bundle = submit_path_bundle(); + bundle["verification"] + .as_object_mut() + .unwrap() + .remove(claim); + assert_invalid( + &bundle, + &format!( + "a submit-path bundle omitting {claim} must be rejected: the claim \ + is earnable through StoreEngine::submit, so silence about it is a \ + missing result and not an inapplicable one" + ), + ); + } + + // Amendment 2's condition. `objects_new` summed from the transaction count + // makes the claim unfailable, so the declaration that says so forbids it — + // at this gate and at every other. + let mut tautology = submit_path_bundle(); + tautology["run_conditions"]["objects_new_source"] = + serde_json::json!("derived_from_transaction_count"); + assert_invalid( + &tautology, + "objects_new_equals_three_per_commit may not be asserted over a count \ + derived from the transaction total: both sides would come from the same \ + number and the check could not fail", + ); + + // Amendment 4's condition. Per-record uniqueness and distinct seed domains + // are both named, and both refuse the claim. + for weaker in ["inferred_from_seed_domains", "not_checked"] { + let mut bundle = submit_path_bundle(); + bundle["run_conditions"]["commit_id_uniqueness"] = serde_json::json!(weaker); + assert_invalid( + &bundle, + &format!( + "unique_blob_tree_commit_ids may not be asserted with \ + commit_id_uniqueness={weaker}: uniqueness must be checked globally \ + across every recovered ACK record" + ), + ); + } + + // Amendment 3. Expressible, correctly constrained, and not earned today. + let mut unearned = submit_path_bundle(); + unearned["verification"]["operation_receipts_reconciled"] = serde_json::json!(true); + assert_invalid( + &unearned, + "operation_receipts_reconciled may not be asserted while the run declares \ + that it accepted any Committed status: store-bench.rs compares no receipt \ + and journals a digest of the operation id, not of the receipt", + ); + + let mut earned = submit_path_bundle(); + earned["run_conditions"]["receipt_reconciliation"] = + serde_json::json!("exact_receipts_reconciled"); + assert_invalid( + &earned, + "a run that declares exact receipt reconciliation must assert the claim \ + rather than leave it optional", + ); + earned["verification"]["operation_receipts_reconciled"] = serde_json::json!(true); + assert_valid( + &earned, + "the claim must be expressible once the reconciliation is real, so landing \ + it is an emitter change and not a second schema amendment", + ); + + // The two that stay unavailable to this workload at either path. + for claim in [ + "blobs_recomputed", + "metadata_complete", + "commits_in_recovered_closure", + ] { + let mut bundle = submit_path_bundle(); + bundle["verification"][claim] = serde_json::json!(true); + assert_invalid( + &bundle, + &format!("{claim} must stay unavailable at storage_primitive"), + ); + } +} + +/// The branch-conditional half. A schema that merely *permitted* the three +/// claims on both paths would hand the journal seam a way to assert what it +/// cannot observe, which is a worse defect than the one being fixed. +#[test] +fn the_drive_path_may_not_assert_the_claims_the_submit_path_earns() { + for claim in [ + "unique_blob_tree_commit_ids", + "objects_new_equals_three_per_commit", + "operation_receipts_reconciled", + ] { + let mut bundle = drive_path_bundle(); + bundle["verification"][claim] = serde_json::json!(true); + assert_invalid( + &bundle, + &format!( + "a journal-drive bundle asserting {claim} must be rejected: there \ + are no objects, no receipts, and no index below engine.rs" + ), + ); + } + + // And it may not reach the claims by lying about provenance either: the + // seam's declarations are pinned to the only values it can make. + for (field, value) in [ + ("objects_new_source", "summed_from_receipts"), + ( + "commit_id_uniqueness", + "checked_globally_across_ack_records", + ), + ("receipt_reconciliation", "exact_receipts_reconciled"), + ("index_maintenance", "runs_sealed"), + ] { + let mut bundle = drive_path_bundle(); + bundle["run_conditions"][field] = serde_json::json!(value); + assert_invalid( + &bundle, + &format!( + "a journal-drive bundle declaring {field}={value} must be rejected: \ + forbidding the claim while permitting the declaration that earns it \ + leaves the same hole one field over" + ), + ); + } +} + +/// `max_index_runs` must be the value the run configured. Absent is invalid, +/// and a declaration that contradicts the recorded value is invalid in both +/// directions — which is as close to "not a default" as a schema can get +/// without reading the process's options. +#[test] +fn the_configured_index_run_ceiling_must_be_recorded_and_consistent() { + let mut absent = submit_path_bundle(); + absent["resources"]["configured_ceilings"] + .as_object_mut() + .unwrap() + .remove("max_index_runs"); + assert_invalid( + &absent, + "a bundle that does not record max_index_runs must be rejected: the ceiling \ + is the limit this workload actually reaches", + ); + + // Declared raised, recorded as the default. + let mut defaulted = submit_path_bundle(); + defaulted["resources"]["configured_ceilings"]["max_index_runs"] = serde_json::json!(64); + assert_invalid( + &defaulted, + "a run that raised the ceiling may not record the store default", + ); + + // Declared default, recorded as the raise the emitter actually configures. + let mut mislabelled = drive_path_bundle(); + mislabelled["resources"]["configured_ceilings"]["max_index_runs"] = + serde_json::json!(1_000_000); + assert_invalid( + &mislabelled, + "a run that declares the store default may not record a raised ceiling", + ); + + // The store default the schema bounds `store_default` at must still be the + // library's default, or the two have drifted and the bound means nothing. + let options = + std::fs::read_to_string(repository_root().join("crates/levcs-store/src/options.rs")) + .unwrap(); + assert!( + options.contains("max_index_runs: 64,"), + "bench/result-schema.json bounds a declared store_default at 64; if \ + StoreOptions::default changes, that bound must change with it" + ); +} + +/// The truthful-environment ruling. B4's debug/tmpfs diagnostic run was +/// unrepresentable — `persistent_data_mount` and `tmpfs` were unconditional +/// consts — so its number lived on a console and in prose. It is representable +/// now, and mechanically disqualified: recording a diagnostic run is worth +/// nothing unless the record also refuses to let it be read as a result. +#[test] +fn a_diagnostic_environment_is_representable_but_can_never_pass() { + let mut diagnostic = submit_path_bundle(); + diagnostic["deployment"]["persistent_data_mount"] = serde_json::json!(false); + diagnostic["deployment"]["tmpfs"] = serde_json::json!(true); + assert_valid( + &diagnostic, + "a tmpfs debug diagnostic run must be recordable as itself rather than \ + being unrepresentable and therefore console-only", + ); + + // Disqualified, not merely labelled. + let mut passing = diagnostic.clone(); + passing["outcome"] = serde_json::json!("pass"); + assert_invalid(&passing, "a diagnostic run may never be a pass at any gate"); + + let mut verdict = diagnostic.clone(); + verdict["verdicts"]["storage_primitive"] = serde_json::json!("pass"); + assert_invalid( + &verdict, + "a diagnostic run may not pronounce a passing verdict on any gate", + ); + + // And the reference environment is unchanged for anything claiming it. + for (field, value) in [ + ("persistent_data_mount", serde_json::json!(false)), + ("tmpfs", serde_json::json!(true)), + ] { + let mut reference = submit_path_bundle(); + reference["run_conditions"]["environment_fidelity"] = + serde_json::json!("reference_profile"); + reference["run_conditions"]["build_profile"] = serde_json::json!("release"); + reference["hardware"]["profile"] = serde_json::json!("minimum-30k"); + reference["deployment"][field] = value.clone(); + assert_invalid( + &reference, + &format!( + "reference_profile fidelity must re-pin deployment.{field}: relaxing \ + the unconditional const without re-pinning it here would let a \ + promotable bundle be measured on tmpfs" + ), + ); + } + + let mut debug_reference = submit_path_bundle(); + debug_reference["run_conditions"]["environment_fidelity"] = + serde_json::json!("reference_profile"); + assert_invalid( + &debug_reference, + "reference_profile fidelity requires a release build and a named hardware \ + profile; a debug diagnostic-profile run may not claim it", + ); +} + +/// Scope §7's two prose clauses, made mechanical: the P2 runs must not have +/// been achieved with checkpointing disabled, and a run holding every index +/// delta in memory for its whole duration is not the steady state a P2 number +/// describes. Neither could be checked from a bundle before this review. +#[test] +fn a_passing_storage_primitive_run_must_declare_a_production_steady_state() { + // The honest values today make a pass impossible, which is the point. + let mut passing = submit_path_bundle(); + passing["outcome"] = serde_json::json!("pass"); + passing["verdicts"]["storage_primitive"] = serde_json::json!("pass"); + passing["run_conditions"]["environment_fidelity"] = serde_json::json!("reference_profile"); + passing["run_conditions"]["build_profile"] = serde_json::json!("release"); + passing["hardware"]["profile"] = serde_json::json!("minimum-30k"); + assert_invalid( + &passing, + "today's harness may not emit a passing P2 bundle: checkpointing is \ + unimplemented and the index holds every delta in memory", + ); + + passing["run_conditions"]["checkpointing"] = serde_json::json!("exercised"); + passing["run_conditions"]["index_maintenance"] = serde_json::json!("runs_sealed"); + passing["run_conditions"]["index_run_ceiling"] = serde_json::json!("store_default"); + passing["resources"]["configured_ceilings"]["max_index_runs"] = serde_json::json!(64); + assert_invalid( + &passing, + "a P2 pass must have been measured against a root the production entry \ + point created; ROOT_SEEDED_BY_NON_PRODUCTION_PATH is disclosable but not \ + promotable", + ); + + passing["run_conditions"]["initialization_path"] = serde_json::json!("store_engine_open"); + assert_valid( + &passing, + "a run that met every condition must still be able to pass, or the rule is \ + a prohibition rather than a gate", + ); + + // The journal seam can never reach a P2 pass, whatever it declares. + let mut seam = drive_path_bundle(); + seam["outcome"] = serde_json::json!("pass"); + seam["run_conditions"]["environment_fidelity"] = serde_json::json!("reference_profile"); + seam["run_conditions"]["build_profile"] = serde_json::json!("release"); + seam["hardware"]["profile"] = serde_json::json!("minimum-30k"); + seam["run_conditions"]["checkpointing"] = serde_json::json!("exercised"); + assert_invalid( + &seam, + "a journal-drive measurement may never be a passing storage_primitive run", + ); +} + +/// The half of every per-gate split that is easy to lose: relaxing something +/// for one branch must never un-pin it for the others. It has been lost once +/// already, by the edit that introduced the assertion saying so. +#[test] +fn the_instance_gates_are_not_loosened_by_the_storage_path_split() { + let text = std::fs::read_to_string(repository_root().join("bench/result-schema.json")).unwrap(); + let schema: serde_json::Value = serde_json::from_str(&text).unwrap(); + let rule = schema["allOf"] + .as_array() + .unwrap() + .iter() + .find(|r| { + r["if"]["properties"]["gate"]["const"] == "storage_primitive" && r.get("else").is_some() + }) + .expect("missing the per-gate rule"); + let conditions = &rule["else"]["properties"]["run_conditions"]["properties"]; + assert_eq!( + conditions["mutation_path"]["const"].as_str(), + Some("store_engine_submit"), + "no instance gate may be measured on the journal seam" + ); + assert_eq!( + conditions["initialization_path"]["const"].as_str(), + Some("store_engine_open"), + "no instance gate may be measured against a root seeded outside production" + ); + assert_eq!( + conditions["index_maintenance"]["const"].as_str(), + Some("runs_sealed") + ); + assert_eq!( + conditions["checkpointing"]["enum"], + serde_json::json!(["exercised", "enabled_not_reached"]), + "an instance gate may not declare checkpointing unimplemented or disabled" + ); + + // And the five graph claims are still required and true at every instance + // gate, which the storage-path split must not have touched. + for claim in [ + "unique_blob_tree_commit_ids", + "objects_new_equals_three_per_commit", + "blobs_recomputed", + "operation_receipts_reconciled", + "metadata_complete", + "commits_in_recovered_closure", + ] { + assert_eq!( + rule["else"]["properties"]["verification"]["properties"][claim]["const"].as_bool(), + Some(true), + "{claim} must stay pinned true at every non-storage gate" + ); + } +} diff --git a/crates/levcs-store/src/bin/store-bench.rs b/crates/levcs-store/src/bin/store-bench.rs index f602150..f3e6d8b 100644 --- a/crates/levcs-store/src/bin/store-bench.rs +++ b/crates/levcs-store/src/bin/store-bench.rs @@ -40,13 +40,10 @@ //! # What is blocked //! //! `emit-skeleton --path submit` now drives the production `StoreEngine::submit` -//! (scope 6.6 deliverable 3). It is still not a P2 run, and three of B1's -//! unimplemented deliverables are why — every one of them a bound on the -//! bundle, not merely on this file: +//! (scope 6.6 deliverable 3) over a root the production `StoreEngine::open` +//! built. It is still not a P2 run, and two of B1's unimplemented deliverables +//! are why — both of them a bound on the bundle, not merely on this file: //! -//! * `StoreEngine::open` refuses startup state 1, so the root is created by -//! `segment::initialize_root`. The measured path is production; the path -//! that built the store it measures is not. //! * `submit` refuses after `max_index_runs` group publications, because //! sealing the in-memory index delta into an `IndexRun` is unimplemented. //! The ceiling is raised for the run, which means every delta layer ever @@ -56,14 +53,60 @@ //! Section 7 requires that a P2 run not have been achieved with //! checkpointing disabled. This one was. //! -//! The bundle records none of those three, because `bench/result-schema.json` -//! is `additionalProperties: false` throughout and has no field for the -//! conditions a run was produced under. That is an amendment request to the -//! lead, not an edit: a bundle whose caveats live only in a report is a bundle -//! that reads as unconditional to everyone who receives it. +//! The third bound is gone rather than re-worded: `StoreEngine::open` refused +//! startup state 1, so the root was created by `segment::initialize_root` and +//! the run measured the production path over a store production had not built. +//! B1 landed state 1, `run_engine` builds through `open`, and the bundle now +//! declares `initialization_path: store_engine_open` — observed from the +//! `FORMAT` marker being absent before that call and present after, not +//! labelled. +//! +//! The bundle records what remains, as values a consumer checks rather than as +//! prose a consumer reads. Contract review 2026-07-28-C added the +//! `run_conditions` block for exactly this, and every member of it is derived +//! here from what the run configured or observed — +//! `initialization_path` from what the `FORMAT` marker did across the open, +//! `index_run_ceiling` by comparing the configured `max_index_runs` to the +//! store default, `index_maintenance` from the `IndexRun` files on the device +//! *read back and validated*, `checkpointing` from what +//! `StoreEngine::checkpoint` answered when this run called it, `build_profile` +//! from `cfg!`, and `environment_fidelity` from all four conditions the +//! reference profile re-pins. A hardcoded block would be the same caveat in a +//! different syntax. +//! +//! # What still has to land, recorded as a refusal rather than as a plan +//! +//! **Index steady state.** `index_maintenance: runs_sealed` says the index +//! reached a steady state. That is a statement about *bounded maintenance* — +//! deltas published and not yet sealed — and not about how many files are in a +//! directory, so the derivation requires validated index-run files **and** a +//! reading of the outstanding backlog. Nothing seals today and `StoreEngine` +//! exposes no such reading, so the honest value is unchanged +//! (`deltas_retained_in_memory`) and the declaration is unearnable rather than +//! inferable. **Interface request to B1:** a counter of sealed index runs and +//! outstanding index deltas, in the shape of `DurabilityCounters`. Until it +//! exists, a run that finds sealed runs on the device refuses by name. +//! +//! **Hardware profile.** `hardware.profile` is derived by comparing what this +//! process can read off the host against the complete `[[profile]]` tables of +//! `bench/reference-hardware.toml`. The privileged half — NVMe model and +//! firmware, the I/O scheduler of the device under the root, and the NIC, +//! driver, and link rate — belongs to the deployed-node harness, and it must +//! arrive as **facts to compare, never a profile name to accept**. The two +//! frozen profiles are identical outside their `[profile.network]` tables, so +//! without the link facts they cannot be told apart and the derivation refuses +//! rather than picking. +//! +//! Those declarations are what the claims are conditioned on: the submit path +//! asserts `unique_blob_tree_commit_ids` and +//! `objects_new_equals_three_per_commit` because it performed both checks, and +//! it may not assert `operation_receipts_reconciled` because it declares +//! `receipt_reconciliation: acceptance_of_any_committed_status` — which is the +//! truthful value while the reconciliation accepts any `Committed(_)` and the +//! journalled `receipt_digest` is `blake3(operation_id)`. //! //! The `run` subcommand — warmup, three repetitions, per-repetition fresh -//! roots, trim settle — stays blocked on the same three. +//! roots, trim settle — stays blocked on the same two. //! //! Real Ed25519 attestation signing is also blocked: `levcs-store` has no //! signing dependency and scope §1 forbids any `levcs_identity::` path in this @@ -113,10 +156,42 @@ pub struct FrozenWorkload { max_writer_group_transactions: u64, } -/// The frozen hardware profile's filesystem expectations. +/// One complete `[[profile]]` table of `bench/reference-hardware.toml`. +/// +/// The whole table, not the two filesystem fields the precheck needs. Naming +/// `hardware.profile` means deciding *which* frozen profile a host is, and that +/// is an exact comparison against everything the profile pins — a harness that +/// handed the emitter a profile *label* would be handing it an unverified claim +/// wearing a verified field's name. +pub struct ReferenceProfile { + pub name: String, + pub purpose: String, + /// `(table, key) -> value as written`, with string quotes stripped and + /// numbers, booleans, and arrays left in their frozen spelling so a + /// comparison is against the file rather than against a re-parse of it. + pub facts: Vec<(String, String, String)>, +} + +impl ReferenceProfile { + /// The value this profile pins for `[profile.] `, or `None` + /// where it pins nothing — an unpinned fact is not a constraint, and must + /// not be compared as though it were. + pub fn fact(&self, table: &str, key: &str) -> Option<&str> { + self.facts + .iter() + .find(|(t, k, _)| t == table && k == key) + .map(|(_, _, value)| value.as_str()) + } +} + +/// The frozen hardware profile: the filesystem expectations the two prechecks +/// enforce, plus every frozen profile in full. pub struct FrozenProfile { store_directory_attributes: String, store_directories: Vec, + /// Every `[[profile]]` table, in file order. Used to derive + /// `hardware.profile` by comparison rather than by assertion. + profiles: Vec, } /// Minimal TOML scanning. @@ -168,51 +243,93 @@ mod toml_scan { None } - pub fn string_array(text: &str, key: &str) -> Option> { - for line in text.lines() { - let line = line.trim(); - if line.starts_with('#') { - continue; - } - let (found, value) = match line.split_once('=') { - Some(parts) => parts, - None => continue, - }; - if found.trim() != key { - continue; - } - let value = value.trim(); - let inner = value.strip_prefix('[')?.strip_suffix(']')?; - return Some( - inner - .split(',') - .map(|entry| entry.trim().trim_matches('"').to_string()) - .filter(|entry| !entry.is_empty()) - .collect(), - ); - } - None - } + /// The `[[profile]]` array-of-tables of `bench/reference-hardware.toml`, + /// with every `[profile.
]` sub-table attached to the profile it + /// belongs to. + /// + /// A flat "find every occurrence of this key" scan cannot do this: it + /// cannot tell which profile a value belongs to, which is exactly what + /// naming a profile requires. Strict throughout — an array-of-tables header + /// this file does not model, or a key/value line outside any table, is an + /// error rather than a line quietly skipped, because a profile silently + /// missing a fact would be a profile the comparison could not fail against. + /// + /// Returns `(table, key, value)` triples per profile, where `table` is the + /// empty string for keys written directly under `[[profile]]`. + #[allow(clippy::type_complexity)] + pub fn profile_tables(text: &str) -> Result>, String> { + let mut profiles: Vec> = Vec::new(); + let mut table = String::new(); + // `None` until the first `[[profile]]`; the file's own preamble keys + // (schema_version, frozen_at, notes) belong to no profile. + let mut inside_profile = false; + let mut inside_preamble = true; - /// Every occurrence of a scalar key, so a value repeated across profiles - /// can be checked for agreement rather than read once and assumed. - pub fn all_scalars(text: &str, key: &str) -> Vec { - let mut out = Vec::new(); - for line in text.lines() { - let line = line.trim(); - if line.starts_with('#') { + for (number, raw) in text.lines().enumerate() { + let line = raw.trim(); + if line.is_empty() || line.starts_with('#') { continue; } - if let Some((found, value)) = line.split_once('=') { - if found.trim() == key { - let value = value.trim(); - if let Some(value) = value.strip_prefix('"').and_then(|v| v.strip_suffix('"')) { - out.push(value.to_string()); + if let Some(header) = line.strip_prefix("[[").and_then(|l| l.strip_suffix("]]")) { + if header.trim() != "profile" { + return Err(format!( + "line {}: unsupported array-of-tables [[{header}]]; this reader \ + models [[profile]] and nothing else, and skipping it would leave \ + a table the profile comparison never sees", + number + 1 + )); + } + profiles.push(Vec::new()); + table.clear(); + inside_profile = true; + inside_preamble = false; + continue; + } + if let Some(header) = line.strip_prefix('[').and_then(|l| l.strip_suffix(']')) { + let header = header.trim(); + match header.strip_prefix("profile.") { + Some(sub) if inside_profile => { + table = sub.to_string(); + } + Some(_) => { + return Err(format!( + "line {}: [{header}] appears before any [[profile]]", + number + 1 + )) + } + None => { + return Err(format!( + "line {}: unsupported table [{header}]; every table in this \ + file belongs to a profile", + number + 1 + )) } } + continue; } + let (key, value) = line.split_once('=').ok_or_else(|| { + format!( + "line {}: neither a table header nor key = value", + number + 1 + ) + })?; + if inside_preamble { + continue; + } + let key = key.trim().to_string(); + let value = value.trim(); + let value = value + .strip_prefix('"') + .and_then(|v| v.strip_suffix('"')) + .unwrap_or(value) + .to_string(); + profiles + .last_mut() + .ok_or_else(|| format!("line {}: a value outside every profile", number + 1))? + .push((table.clone(), key, value)); } - out + + Ok(profiles) } } @@ -233,30 +350,112 @@ fn load_frozen_workload(path: &Path) -> Result { fn load_frozen_profile(path: &Path) -> Result { let text = std::fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?; - let attributes = toml_scan::all_scalars(&text, "store_directory_attributes"); - if attributes.is_empty() { + let tables = + toml_scan::profile_tables(&text).map_err(|e| format!("{}: {e}", path.display()))?; + if tables.is_empty() { + return Err(format!("{} declares no [[profile]]", path.display())); + } + + let mut profiles = Vec::new(); + for facts in tables { + let named = |table: &str, key: &str| -> Option { + facts + .iter() + .find(|(t, k, _)| t == table && k == key) + .map(|(_, _, value)| value.clone()) + }; + let name = named("", "name").ok_or_else(|| { + format!( + "{} has a [[profile]] with no name; an unnamed profile cannot be \ + compared against and cannot be named in a bundle", + path.display() + ) + })?; + let purpose = named("", "purpose").unwrap_or_default(); + profiles.push(ReferenceProfile { + name, + purpose, + facts, + }); + } + + // Two profiles with one name would make `hardware.profile` ambiguous the + // moment a host matched either. + for (index, profile) in profiles.iter().enumerate() { + if profiles[..index] + .iter() + .any(|other| other.name == profile.name) + { + return Err(format!( + "{} declares two [[profile]] tables named {:?}", + path.display(), + profile.name + )); + } + } + + let attributes: Vec<&str> = profiles + .iter() + .filter_map(|profile| profile.fact("filesystem", "store_directory_attributes")) + .collect(); + if attributes.len() != profiles.len() { return Err(format!( - "{} has no [profile.filesystem].store_directory_attributes; contract \ - review 2026-07-24-B adds it and this binary depends on it", + "{} has a profile with no [profile.filesystem].store_directory_attributes; \ + contract review 2026-07-24-B adds it and this binary depends on it", path.display() )); } // A profile that silently permits two different on-disk configurations for // the files carrying the throughput is not a frozen profile (scope 9.2). - if attributes.iter().any(|value| value != &attributes[0]) { + if attributes.iter().any(|value| *value != attributes[0]) { return Err(format!( "{} declares disagreeing store_directory_attributes across profiles: \ {attributes:?}", path.display() )); } + let store_directory_attributes = attributes[0].to_string(); + + let directories: Vec> = profiles + .iter() + .filter_map(|profile| profile.fact("filesystem", "store_directories")) + .map(parse_string_array) + .collect(); + if directories.len() != profiles.len() { + return Err(format!( + "{} has a profile with no [profile.filesystem].store_directories", + path.display() + )); + } + if directories.iter().any(|value| *value != directories[0]) { + return Err(format!( + "{} declares disagreeing store_directories across profiles: {directories:?}", + path.display() + )); + } + Ok(FrozenProfile { - store_directory_attributes: attributes[0].clone(), - store_directories: toml_scan::string_array(&text, "store_directories") - .ok_or("store_directories")?, + store_directory_attributes, + store_directories: directories[0].clone(), + profiles, }) } +/// `["a", "b"]` as written in the frozen file. +fn parse_string_array(raw: &str) -> Vec { + raw.trim() + .strip_prefix('[') + .and_then(|inner| inner.strip_suffix(']')) + .map(|inner| { + inner + .split(',') + .map(|entry| entry.trim().trim_matches('"').to_string()) + .filter(|entry| !entry.is_empty()) + .collect() + }) + .unwrap_or_default() +} + // --------------------------------------------------------------------------- // Precheck 1 — free space // --------------------------------------------------------------------------- @@ -627,10 +826,22 @@ fn digest_hex(bytes: &[u8]) -> String { hex::encode(blake3::hash(bytes).as_bytes()) } -fn digest_file(path: &Path) -> String { +/// The digest of a file the bundle attests to. +/// +/// A read failure is a refusal, not `digest_hex(b"")`. The empty digest is a +/// well-formed 64-hex-character value that a reader cannot distinguish from +/// the digest of a file that was actually read, so returning it made an +/// unreadable input indistinguishable from an attested one — the same defect +/// class as a run that ends quietly, in the reporting half of the emitter. +fn digest_file(path: &Path) -> Result { match std::fs::read(path) { - Ok(bytes) => digest_hex(&bytes), - Err(_) => digest_hex(b""), + Ok(bytes) => Ok(digest_hex(&bytes)), + Err(error) => Err(format!( + "refusing to emit a bundle: {} could not be read to digest it ({error}). A \ + bundle field that names a digest must name the digest of something this run \ + read.", + path.display() + )), } } @@ -651,6 +862,17 @@ fn read_first_line(path: &str) -> String { .unwrap_or_default() } +/// A reading, or nothing. An empty string is what these helpers return when the +/// host would not answer, and treating that as an observed value is how a fact +/// with no reading would compare equal to a profile that pins the empty string. +fn non_empty(value: String) -> Option { + if value.is_empty() { + None + } else { + Some(value) + } +} + fn or_unknown(value: String, what: &str) -> String { if value.is_empty() { format!("unknown ({what} not readable on this host)") @@ -659,6 +881,976 @@ fn or_unknown(value: String, what: &str) -> String { } } +// --------------------------------------------------------------------------- +// Run conditions (contract review 2026-07-28-C, scope 6.6 item 5b) +// --------------------------------------------------------------------------- +// +// The ten members of `run_conditions` are **declarations of what this run +// did**, and every one of them is derived here from something the run observed +// or configured. A hardcoded block would be the prose caveat with a different +// syntax, which is what the review rejected — so the types below are +// observations, and `RunConditions::derive` is the only place a schema +// enumeration value is produced. +// +// Charter item 6 applies throughout: the schema's enumerations are closed sets +// and the mapping onto them is closed too. There is no catch-all arm and no +// fallback binding anywhere in this section; an observation that maps to no +// named value is a refusal that names it. + +/// How the store root the run measured was created. +/// +/// `segment_initialize_root` has no variant here any more, and its absence is +/// the deliverable: while `StoreEngine::open` refused startup state 1 the +/// submit run seeded its own root and declared it, which is a benchmark +/// measuring the production path over a store production had not built. B1 +/// landed state 1, `run_engine` builds through `open`, and a variant nothing +/// constructs would be a declaration this file could make without having done +/// anything. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum Initialization { + /// `StoreEngine::open` created the root, observed rather than assumed: no + /// `FORMAT` marker before the call, one after. + StoreEngineOpen, + /// `ShardDrive::create`, the Wave A journal seam. + ShardDriveCreate, +} + +/// The entry point the measured transactions actually went through. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum MutationPath { + StoreEngineSubmit, + JournalDrive, +} + +/// What `StoreEngine::checkpoint` answered when this run called it. +/// +/// Classified from a real call rather than from this file's opinion of what +/// the store implements. Both variants are constructed by +/// [`probe_checkpointing`]; anything else it sees is a refusal, because the +/// schema's four values name no third answer. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum CheckpointProbe { + /// `StoreEngine::checkpoint` refused `NotImplemented`, so no checkpoint was + /// taken and none could have been. + RefusedNotImplemented, + /// `StoreEngine::checkpoint` returned a lease. The measured window still + /// contains no checkpoint — the harness takes none inside it — so the + /// truthful value is `enabled_not_reached`, and reaching one is the + /// emitter's next change rather than a relabelling of this one. + EnabledNotReached, +} + +/// Whether the index reached a steady state, from what is on the device. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum IndexObservation { + /// What `shards/*/indexes` held after the run, alongside the group + /// publications the run performed. + /// + /// `validated_runs` counts only entries that were read back and parsed as + /// an `IndexRun` against this root's own uuid. Counting directory entries + /// was the defect: a temporary file, a truncated file, or an unrelated file + /// dropped in the directory would each have moved `index_maintenance` to + /// `runs_sealed`, and a declaration that the index reached a steady state + /// must not be satisfiable by a stray file. + StoreRoot { + validated_runs: u64, + /// Entries under those directories that did not parse as an index run, + /// with why. Non-empty is a refusal, not a lower count: an entry the + /// emitter cannot validate is a directory it cannot describe. + unvalidatable: Vec, + groups: u64, + /// Index deltas the run published and did not seal into a run, as the + /// store reports them. + /// + /// `None` while nothing seals and the store exposes no reading of the + /// outstanding backlog. A steady state is *bounded maintenance*, not + /// the presence of files, so a run that cannot show the backlog cannot + /// earn `runs_sealed` — see [`RunConditions::derive`], which refuses by + /// name rather than inferring it. Recorded as an interface request: + /// `StoreEngine` needs a counter of sealed runs and outstanding deltas + /// in the same shape as `DurabilityCounters`. + unsealed_delta_backlog: Option, + }, + /// The measured path is the journal seam: nothing below `engine.rs` touches + /// an index at all. + NoIndexInPath, +} + +/// What the run reconciled each acknowledged operation against. +/// +/// `exact_receipts_reconciled` and `canonical_receipt_digest_reconciled` have +/// no variants here for the same reason `store_engine_open` has none above: no +/// code in this file performs either, and a variant that nothing constructs is +/// a claim waiting to be made by accident. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum ReceiptComparison { + /// The reconciliation loop accepts any `TransactionStatus::Committed(_)` + /// without comparing the payload, and the `receipt_digest` it journals is + /// `blake3(operation_id)` rather than a digest of the receipt. + AnyCommittedStatusAccepted, + /// The journal seam produces no receipts to reconcile. + NoReceiptsInPath, +} + +/// How blob, tree, and commit identifier uniqueness was established. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum UniquenessCheck { + /// Every `blob_ids`, `tree_ids`, and `commit_ids` entry from every record + /// `ExternalAckJournal::recover` returned, unioned into one set per kind + /// across all records at once, with no repeat. The counts are the set + /// sizes, so a reader can see the check had something to check. + GlobalAcrossAckRecords { + blob_ids: u64, + tree_ids: u64, + commit_ids: u64, + }, + /// Not performed. On the journal seam the identifiers in an acknowledgment + /// record are fabricated by this harness rather than produced by the store, + /// so checking them would establish a property of the harness. + NotPerformed, +} + +/// What a measured run observed about itself, whichever path produced it. +/// +/// Everything here is recorded at the site that performed the thing it +/// describes, so a path that stops doing one of them stops being able to +/// declare it. +pub struct RunFacts { + pub initialization: Initialization, + pub mutation: MutationPath, + pub checkpoint: CheckpointProbe, + pub index: IndexObservation, + pub configured_max_index_runs: u32, + pub default_max_index_runs: u32, + pub receipts: ReceiptComparison, + pub objects_new_counted: bool, + pub uniqueness: UniquenessCheck, +} + +/// The facts a run observed, from which the ten declarations are derived. +pub struct RunObservations { + pub initialization: Initialization, + pub mutation: MutationPath, + pub checkpoint: CheckpointProbe, + pub index: IndexObservation, + /// `StoreOptions::max_index_runs` as the run configured it, read back from + /// the options struct the store was opened with. + pub configured_max_index_runs: u32, + /// The same field on an otherwise untouched `StoreOptions`, so the + /// comparison below is against the library's default rather than against a + /// number this file writes twice. + pub default_max_index_runs: u32, + pub receipts: ReceiptComparison, + /// `counts.objects_new` was summed from the store's own receipts rather + /// than multiplied out of the transaction count. + pub objects_new_counted: bool, + pub uniqueness: UniquenessCheck, + /// The filesystem carrying the store root, read from `/proc/mounts`. + pub store_root_filesystem: String, + pub store_root_is_tmpfs: bool, +} + +impl RunObservations { + /// What the run observed about itself, plus what the host observed about + /// where it ran. + pub fn new(facts: RunFacts, mount: MountFacts) -> Self { + Self { + initialization: facts.initialization, + mutation: facts.mutation, + checkpoint: facts.checkpoint, + index: facts.index, + configured_max_index_runs: facts.configured_max_index_runs, + default_max_index_runs: facts.default_max_index_runs, + receipts: facts.receipts, + objects_new_counted: facts.objects_new_counted, + uniqueness: facts.uniqueness, + store_root_filesystem: mount.filesystem, + store_root_is_tmpfs: mount.tmpfs, + } + } +} + +/// The ten declarations, each already mapped onto its schema enumeration. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RunConditions { + pub initialization_path: &'static str, + pub mutation_path: &'static str, + pub checkpointing: &'static str, + pub index_maintenance: &'static str, + pub index_run_ceiling: &'static str, + pub receipt_reconciliation: &'static str, + pub objects_new_source: &'static str, + pub commit_id_uniqueness: &'static str, + pub build_profile: &'static str, + pub environment_fidelity: &'static str, +} + +impl RunConditions { + /// Derive all ten from what the run observed. + /// + /// `attributes_verified` is whether the store-directory attribute precheck + /// ran and matched; `hardware_profile` is the value this bundle is about to + /// emit as `hardware.profile`, which is itself derived from whether any + /// hardware field is a placeholder rather than a measurement. + pub fn derive( + observations: &RunObservations, + attributes_verified: bool, + hardware_profile: &str, + ) -> Result { + let index_maintenance = match &observations.index { + IndexObservation::StoreRoot { + validated_runs, + unvalidatable, + groups, + unsealed_delta_backlog, + } => { + // An entry the emitter could not read back as an index run is a + // refusal by name and never a lower count. Folding it into + // "zero sealed runs" would let a corrupt or half-written run + // read as a run that was never sealed, and folding it into the + // count would let a stray file declare a steady state. + if !unvalidatable.is_empty() { + return Err(format!( + "refusing to emit a bundle: {} entr(ies) under shards/*/indexes did \ + not read back as an IndexRun for this root: {}. \ + run_conditions.index_maintenance declares whether the index reached \ + a steady state, and a directory this emitter cannot describe is not \ + a state it may declare.", + unvalidatable.len(), + unvalidatable.join("; ") + )); + } + if *validated_runs > 0 { + // Files are not maintenance. `runs_sealed` says the index + // reached a steady state, which is a statement about the + // *backlog* — deltas published and not yet sealed — and not + // about how many files are on the device. Nothing seals + // today, so this arm is unreachable today; it exists so the + // day sealing lands the declaration is earned from a + // reading rather than inferred from a directory listing. + match unsealed_delta_backlog { + Some(0) => "runs_sealed", + Some(backlog) => { + return Err(format!( + "refusing to emit a bundle: {validated_runs} index run(s) were \ + sealed and {backlog} published delta(s) are still unsealed. \ + run_conditions.index_maintenance names a steady state and a \ + run that retained every delta, and this is neither: sealing \ + ran and did not keep up. Naming either would describe a \ + backlog as its opposite." + )); + } + None => { + return Err(format!( + "refusing to emit a bundle: {validated_runs} validated index \ + run(s) are on the device, but the store exposes no reading of \ + how many published deltas remain unsealed, so this run cannot \ + show that index maintenance was bounded. `runs_sealed` would \ + be inferred from the presence of files rather than earned \ + from a measurement. Interface request to B1: an outstanding \ + index-delta counter alongside DurabilityCounters." + )); + } + } + } else if *groups > 0 { + "deltas_retained_in_memory" + } else { + return Err( + "refusing to emit a bundle: the run published no group and sealed \ + no index run, so there is no index maintenance to declare. \ + run_conditions.index_maintenance has no value for a run that did \ + nothing, and inventing one would be the caveat this block \ + replaced." + .to_string(), + ); + } + } + IndexObservation::NoIndexInPath => "no_index_in_path", + }; + + // Not `ENGINE_MAX_INDEX_RUNS` compared against a restated 64: the + // configured value comes from the options the store opened with and the + // default comes from `StoreOptions` itself, so the day either moves, + // this declaration moves with it. + let index_run_ceiling = + if observations.configured_max_index_runs == observations.default_max_index_runs { + "store_default" + } else if observations.configured_max_index_runs > observations.default_max_index_runs { + "raised_because_index_sealing_unimplemented" + } else { + return Err(format!( + "refusing to emit a bundle: the run configured max_index_runs = {} below \ + the store default of {}. run_conditions.index_run_ceiling names a default \ + and a raise and nothing else, and a lowered ceiling is neither — it is a \ + run that refuses earlier than the store would, which is a condition the \ + schema cannot express and this emitter will not disguise as one it can.", + observations.configured_max_index_runs, observations.default_max_index_runs + )); + }; + + // `cfg!(debug_assertions)` is the whole of what a compiled binary can + // see of its own profile: `opt-level` is not exposed to `cfg`, so a + // release build with `debug-assertions = true` is indistinguishable + // from a debug build from in here and is reported as `debug`. That is + // the safe direction — it can only disqualify a run from + // `reference_profile`, never admit one — and it is why + // `release_with_debug_assertions` is a value this emitter never + // produces. + let build_profile = if cfg!(debug_assertions) { + "debug" + } else { + "release" + }; + + // `reference_profile` re-pins persistent_data_mount, tmpfs, + // build_profile, and a named hardware profile, so it is asserted only + // when every one of those was observed to hold — including the + // store-directory attribute precheck actually having run. + let environment_fidelity = if build_profile == "release" + && attributes_verified + && !observations.store_root_is_tmpfs + && hardware_profile != "diagnostic" + { + "reference_profile" + } else { + "diagnostic" + }; + + Ok(Self { + initialization_path: match observations.initialization { + Initialization::StoreEngineOpen => "store_engine_open", + Initialization::ShardDriveCreate => "shard_drive_create", + }, + mutation_path: match observations.mutation { + MutationPath::StoreEngineSubmit => "store_engine_submit", + MutationPath::JournalDrive => "journal_drive", + }, + checkpointing: match observations.checkpoint { + CheckpointProbe::RefusedNotImplemented => "unimplemented", + CheckpointProbe::EnabledNotReached => "enabled_not_reached", + }, + index_maintenance, + index_run_ceiling, + receipt_reconciliation: match observations.receipts { + ReceiptComparison::AnyCommittedStatusAccepted => { + "acceptance_of_any_committed_status" + } + ReceiptComparison::NoReceiptsInPath => "no_receipts_in_path", + }, + objects_new_source: if observations.objects_new_counted { + "summed_from_receipts" + } else { + "derived_from_transaction_count" + }, + commit_id_uniqueness: match observations.uniqueness { + UniquenessCheck::GlobalAcrossAckRecords { .. } => { + "checked_globally_across_ack_records" + } + UniquenessCheck::NotPerformed => "not_checked", + }, + build_profile, + environment_fidelity, + }) + } + + fn to_json(&self) -> J { + J::Object(vec![ + ("initialization_path".into(), jstr(self.initialization_path)), + ("mutation_path".into(), jstr(self.mutation_path)), + ("checkpointing".into(), jstr(self.checkpointing)), + ("index_maintenance".into(), jstr(self.index_maintenance)), + ("index_run_ceiling".into(), jstr(self.index_run_ceiling)), + ( + "receipt_reconciliation".into(), + jstr(self.receipt_reconciliation), + ), + ("objects_new_source".into(), jstr(self.objects_new_source)), + ( + "commit_id_uniqueness".into(), + jstr(self.commit_id_uniqueness), + ), + ("build_profile".into(), jstr(self.build_profile)), + ( + "environment_fidelity".into(), + jstr(self.environment_fidelity), + ), + ]) + } + + /// The ten members in schema order, for the harness's own stdout. + pub fn pairs(&self) -> [(&'static str, &'static str); 10] { + [ + ("initialization_path", self.initialization_path), + ("mutation_path", self.mutation_path), + ("checkpointing", self.checkpointing), + ("index_maintenance", self.index_maintenance), + ("index_run_ceiling", self.index_run_ceiling), + ("receipt_reconciliation", self.receipt_reconciliation), + ("objects_new_source", self.objects_new_source), + ("commit_id_uniqueness", self.commit_id_uniqueness), + ("build_profile", self.build_profile), + ("environment_fidelity", self.environment_fidelity), + ] + } +} + +/// What is actually under every shard's `indexes` directory, validated. +/// +/// The observation behind `index_maintenance`. The previous version counted +/// *directory entries*: a temporary file, a truncated run, or an unrelated file +/// each raised the count, and one of them was enough to flip the declaration to +/// `runs_sealed`. A declaration that the index reached a steady state must not +/// be satisfiable by a stray file, so every entry is read back and parsed as an +/// `IndexRun` against this root's own uuid, and anything that does not parse is +/// returned as an entry the emitter could not validate rather than silently +/// counted or silently dropped. +struct IndexRunScan { + validated_runs: u64, + unvalidatable: Vec, +} + +fn scan_index_runs(root: &Path, shard_count: u16) -> IndexRunScan { + use levcs_store::index::IndexRun; + use levcs_store::segment::{read_format, RootLayout}; + + let layout = RootLayout::new(root); + let mut scan = IndexRunScan { + validated_runs: 0, + unvalidatable: Vec::new(), + }; + + // Without the root's own uuid no entry can be validated *as this root's*, + // and an index run carrying another root's uuid is exactly the kind of + // stray file this scan exists to reject. A root whose FORMAT cannot be read + // is reported as one unvalidatable entry rather than as an empty directory. + let root_uuid = match read_format(&layout) { + Ok(marker) => marker.root_uuid, + Err(error) => { + scan.unvalidatable.push(format!( + "{}: FORMAT could not be read ({error}), so no index run under this root \ + can be validated as belonging to it", + layout.format_path().display() + )); + return scan; + } + }; + + for shard in 0..shard_count { + let directory = layout.shard(shard).indexes(); + let entries = match std::fs::read_dir(&directory) { + Ok(entries) => entries, + // An absent `indexes` directory is an empty one: `initialize_root` + // creates it, so its absence is itself a finding — but it cannot + // make a sealed run appear, and an unreadable directory is recorded + // as unvalidatable rather than as evidence of emptiness. + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => { + scan.unvalidatable + .push(format!("{}: {error}", directory.display())); + continue; + } + }; + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + Err(error) => { + scan.unvalidatable + .push(format!("{}: {error}", directory.display())); + continue; + } + }; + let path = entry.path(); + let bytes = match std::fs::read(&path) { + Ok(bytes) => bytes, + Err(error) => { + scan.unvalidatable + .push(format!("{}: {error}", path.display())); + continue; + } + }; + match IndexRun::from_vec(bytes, &root_uuid) { + Ok(_) => scan.validated_runs += 1, + Err(error) => scan + .unvalidatable + .push(format!("{}: {error:?}", path.display())), + } + } + } + + scan +} + +/// Ask the store what it does about checkpoints, and classify the answer. +/// +/// Called on the engine the run measured wherever there is one, and on a +/// throwaway root otherwise, so both paths declare `checkpointing` from a real +/// call to the real entry point rather than from a constant per path. +fn probe_checkpointing(engine: &levcs_store::StoreEngine) -> Result { + use levcs_store::types::StoreError; + match engine.checkpoint() { + Ok(_) => Ok(CheckpointProbe::EnabledNotReached), + Err(StoreError::NotImplemented(_)) => Ok(CheckpointProbe::RefusedNotImplemented), + Err(other) => Err(format!( + "refusing to emit a bundle: StoreEngine::checkpoint refused with {other}. \ + run_conditions.checkpointing names four states and none of them is this \ + one, so the run has nothing truthful to declare." + )), + } +} + +/// The same probe for a path with no engine of its own. +fn probe_checkpointing_on_fresh_root(directory: &Path) -> Result { + use levcs_store::segment::{initialize_root, RootLayout}; + use levcs_store::types::DurabilityCounters; + + let _ = std::fs::remove_dir_all(directory); + initialize_root( + &RootLayout::new(directory), + 1, + [0x9f; 16], + engine_now_micros(), + &DurabilityCounters::default(), + ) + .map_err(|e| format!("checkpoint probe: initialize_root: {e}"))?; + let mut options = levcs_store::StoreOptions::new(directory); + options.shard_count = 1; + // A store that cannot sequence a transaction is not a store to ask about + // checkpoints; the engine refuses to open without a signer. + options.signer = Some(std::sync::Arc::new(MeasuringSigner::new())); + let probe = match levcs_store::StoreEngine::open(options) { + Ok(engine) => { + let probe = probe_checkpointing(&engine); + drop(engine); + probe + } + Err(error) => Err(format!("checkpoint probe: {error}")), + }; + // Cleaned up on both outcomes: a probe root left behind next to the run's + // own root is a directory the next run trips over. + let _ = std::fs::remove_dir_all(directory); + probe +} + +/// The filesystem the store root actually sits on. +pub struct MountFacts { + pub filesystem: String, + pub tmpfs: bool, +} + +/// Read the mount carrying `path` out of `/proc/mounts`. +/// +/// `deployment.tmpfs` and `deployment.persistent_data_mount` were constants +/// until contract review 2026-07-28-C relaxed them; emitting them as constants +/// now would keep the defect the relaxation exists to fix, since a run on a +/// non-persistent mount would still describe itself as persistent. +fn detect_mount(path: &Path) -> Result { + let target = std::fs::canonicalize(path) + .map_err(|e| format!("resolving the store root {}: {e}", path.display()))?; + let mounts = std::fs::read_to_string("/proc/mounts") + .map_err(|e| format!("reading /proc/mounts: {e}"))?; + + let unescape = |field: &str| field.replace("\\040", " ").replace("\\011", "\t"); + let mut best: Option<(usize, String)> = None; + for line in mounts.lines() { + let mut fields = line.split_whitespace(); + let (Some(_device), Some(point), Some(kind)) = + (fields.next(), fields.next(), fields.next()) + else { + continue; + }; + let point = PathBuf::from(unescape(point)); + if target.starts_with(&point) { + let depth = point.components().count(); + if best.as_ref().map_or(true, |(best, _)| depth > *best) { + best = Some((depth, unescape(kind))); + } + } + } + + match best { + Some((_, filesystem)) => Ok(MountFacts { + tmpfs: matches!(filesystem.as_str(), "tmpfs" | "ramfs"), + filesystem, + }), + None => Err(format!( + "no mount in /proc/mounts contains {}; the bundle cannot declare whether \ + the run was on a persistent filesystem, and declaring it anyway is what \ + deployment.tmpfs was relaxed to stop.", + target.display() + )), + } +} + +/// Markers this emitter writes where it has recorded a placeholder instead of a +/// measurement. +/// +/// A bundle carrying any of them has not verified the frozen hardware profile, +/// whatever its other fields say. Retained as a cross-check on the comparison +/// below rather than as the derivation itself: naming a profile while still +/// writing "recorded by the deployed-node harness" into a `hardware.*` field +/// would mean the comparison and the emitted fields disagree about the same +/// host, and that is a refusal. +const HARDWARE_PLACEHOLDER_MARKERS: [&str; 4] = [ + "unknown (", + "unverified", + "recorded by the deployed-node harness", + "none (in-process P2)", +]; + +/// The two frozen profile names the schema's `hardware.profile` enumeration +/// admits alongside `diagnostic`. +/// +/// A profile in `bench/reference-hardware.toml` whose name is not one of these +/// is a refusal rather than a name emitted on faith: the schema's enumeration +/// is a closed set, and a third frozen profile is a lead-owned schema change +/// before it is a bundle value. +const NAMED_HARDWARE_PROFILES: [&str; 2] = ["minimum-30k", "release-60k"]; + +/// One fact the frozen profile pins, and what this run read for it. +/// +/// `observed: None` is the deployed-node harness's half. It is deliberately not +/// a hole the emitter fills with a default: a fact with no reading eliminates +/// every profile that pins it, so an unobservable fact can only ever move the +/// derivation toward `diagnostic` and never toward a name. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HostFact { + /// `[profile.
]` as the frozen file spells it. + pub table: &'static str, + pub key: &'static str, + pub observed: Option, +} + +impl HostFact { + /// `table.key`, spelled as the frozen file spells it. Test-only since the + /// comparison was inverted: the derivation now names each fact from the + /// profile side, because that is the side that decides which facts exist. + #[cfg(test)] + fn name(&self) -> String { + format!("{}.{}", self.table, self.key) + } +} + +/// What `store-bench` can read about the host it is running on, named exactly +/// as `bench/reference-hardware.toml` names it. +/// +/// Every entry is a reading or an explicit absence. The absences are the +/// interface request to the deployed-node harness — and they are *evidence to +/// compare*, never a name to accept: a harness that supplied +/// `hardware.profile = "release-60k"` would be supplying a claim this emitter +/// has no way to check, which is the failure mode the split design exists to +/// prevent. +pub fn observe_host_facts(store_root_filesystem: &str) -> Vec { + let cpuinfo = |field: &str| -> Option { + let text = std::fs::read_to_string("/proc/cpuinfo").ok()?; + text.lines() + .find(|line| line.trim_start().starts_with(field)) + .and_then(|line| line.split_once(':')) + .and_then(|(_, value)| non_empty(value.trim().to_string())) + }; + let threads = std::fs::read_to_string("/proc/cpuinfo") + .ok() + .map(|text| { + text.lines() + .filter(|line| line.trim_start().starts_with("processor")) + .count() + }) + .filter(|count| *count > 0) + .map(|count| count.to_string()); + let numa_nodes = std::fs::read_dir("/sys/devices/system/node") + .ok() + .map(|entries| { + entries + .filter_map(|entry| entry.ok()) + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with("node")) + }) + .count() + }) + .filter(|count| *count > 0) + .map(|count| count.to_string()); + let swap_enabled = std::fs::read_to_string("/proc/swaps") + .ok() + .map(|text| (text.lines().count() > 1).to_string()); + + vec![ + HostFact { + table: "cpu", + key: "model", + observed: cpuinfo("model name"), + }, + HostFact { + table: "cpu", + key: "physical_cores", + observed: cpuinfo("cpu cores"), + }, + HostFact { + table: "cpu", + key: "threads", + observed: threads, + }, + HostFact { + table: "cpu", + key: "architecture", + observed: Some(std::env::consts::ARCH.to_string()), + }, + HostFact { + table: "cpu", + key: "governor", + observed: non_empty(read_first_line( + "/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor", + )), + }, + HostFact { + table: "cpu", + key: "numa_nodes", + observed: numa_nodes, + }, + HostFact { + table: "cpu", + key: "microcode", + observed: cpuinfo("microcode"), + }, + HostFact { + table: "memory", + key: "swap_enabled", + observed: swap_enabled, + }, + // `MemTotal` is what the kernel manages, not what is installed: + // firmware reservations make the two differ by hundreds of megabytes, + // so an exact comparison against `installed_gib` cannot be made from + // it. The reading belongs to the deployed-node harness. + HostFact { + table: "memory", + key: "installed_gib", + observed: None, + }, + HostFact { + table: "filesystem", + key: "type", + observed: Some(store_root_filesystem.to_string()), + }, + HostFact { + table: "software", + key: "kernel", + observed: non_empty(command_output("uname", &["-sr"])), + }, + // Everything below needs a privileged reading of the device or the + // link. The two frozen profiles are identical *except* for their + // `[profile.network]` tables, so without these the comparison cannot + // distinguish `minimum-30k` from `release-60k` even on the reference + // host — which is why they are listed rather than dropped. + HostFact { + table: "nvme", + key: "model", + observed: None, + }, + HostFact { + table: "nvme", + key: "firmware", + observed: None, + }, + HostFact { + table: "nvme", + key: "scheduler", + observed: None, + }, + HostFact { + table: "network", + key: "nic", + observed: None, + }, + HostFact { + table: "network", + key: "driver", + observed: None, + }, + HostFact { + table: "network", + key: "link_mbps", + observed: None, + }, + ] +} + +/// Why `hardware.profile` came out the way it did, reported alongside the value +/// so a reader can see what was compared rather than only what was concluded. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HardwareProfileDerivation { + pub name: &'static str, + /// Facts a frozen profile pins that this run could not read — whether the + /// emitter tried and failed or does not model the fact at all, which are + /// the same thing from a bundle's point of view. Each one eliminates every + /// profile pinning it; this list is the deployed-node harness's work order. + pub unobserved: Vec, + /// Facts read whose value disagrees with every frozen profile. + pub mismatched: Vec, +} + +/// `hardware.profile`, derived by comparing observed host facts against the +/// complete `[[profile]]` tables of `bench/reference-hardware.toml`. +/// +/// The comparison is exact and per fact, and it is driven by **the frozen +/// profile**, not by what this emitter happens to observe. A profile is a +/// candidate only when every fact it pins was read *and* compared equal; a +/// fact with no reading eliminates the profile rather than being skipped, +/// because skipping it would let a host be named on a partial match. Zero +/// candidates is `diagnostic` — the run is not on the reference host, or has +/// not shown that it is. Two candidates is a refusal: the facts that would +/// separate them were not observed, and picking one would be the guess this +/// function exists to eliminate. +/// +/// The direction matters and was wrong. Iterating the *supplied* facts made a +/// pinned fact this emitter does not model invisible rather than unobserved: +/// `cpu.turbo`, `nvme.power_loss_protection`, `filesystem.barriers`, +/// `network.mtu` and the rest were pinned by both profiles and compared by +/// nothing, so a host could match every modelled fact, differ on an unmodelled +/// one, and be named anyway. Iterating the profile closes that mechanically — +/// a pinned fact with no `HostFact` entry at all is indistinguishable from one +/// whose entry reads `None`, because in both cases the run did not observe it +/// — and it makes the comparison set self-maintaining: a fact added to +/// `bench/reference-hardware.toml` tightens the check on the next run instead +/// of being ignored until someone remembers to model it. +fn hardware_profile_of( + profiles: &[ReferenceProfile], + facts: &[HostFact], + emitted_fields: &[(String, J)], +) -> Result { + let mut unobserved: Vec = Vec::new(); + let mut matched_somewhere: Vec = Vec::new(); + let mut compared: Vec = Vec::new(); + let mut candidates: Vec<&ReferenceProfile> = Vec::new(); + + for profile in profiles { + let mut candidate = true; + let mut pinned_facts = 0u32; + for (table, key, pinned) in &profile.facts { + if table.is_empty() { + // `name` and `purpose` sit above every `[profile.
]`. + // They identify the profile rather than pin a property of a + // host, and there is nothing on a host to read them from. + continue; + } + pinned_facts += 1; + let name = format!("{table}.{key}"); + let observed = facts + .iter() + .find(|fact| fact.table == table.as_str() && fact.key == key.as_str()) + .and_then(|fact| fact.observed.as_deref()); + match observed { + None => { + candidate = false; + if !unobserved.contains(&name) { + unobserved.push(name); + } + } + Some(observed) => { + if !compared.contains(&name) { + compared.push(name.clone()); + } + if observed == pinned.as_str() { + if !matched_somewhere.contains(&name) { + matched_somewhere.push(name); + } + } else { + candidate = false; + } + } + } + } + // A profile that pins nothing is matched by every host, including one + // that observed nothing at all. Refused rather than treated as a + // vacuous candidate, because the name it would produce would rest on + // no comparison whatsoever. + if pinned_facts == 0 { + return Err(format!( + "refusing to derive hardware.profile: the frozen profile {:?} pins no \ + [profile.
] fact, so every host would match it and the name would \ + rest on no comparison at all.", + profile.name + )); + } + if candidate { + candidates.push(profile); + } + } + + let mismatched: Vec = compared + .into_iter() + .filter(|name| !matched_somewhere.contains(name)) + .collect(); + + if candidates.len() > 1 { + return Err(format!( + "refusing to emit a bundle: the observed host facts match {} frozen profiles \ + ({}). The facts that separate them were not observed, and naming one of them \ + would be a guess wearing a frozen profile's name.", + candidates.len(), + candidates + .iter() + .map(|profile| profile.name.as_str()) + .collect::>() + .join(", ") + )); + } + + let Some(profile) = candidates.first() else { + return Ok(HardwareProfileDerivation { + name: "diagnostic", + unobserved, + mismatched, + }); + }; + + // The frozen file names the profile; the schema enumerates what a bundle + // may carry. A name in one and not the other is a refusal, never a value + // emitted because the file said so. + let name = NAMED_HARDWARE_PROFILES + .iter() + .find(|known| **known == profile.name) + .ok_or_else(|| { + format!( + "refusing to emit a bundle: the host matches the frozen profile {:?}, which \ + is not one of the names bench/result-schema.json admits for \ + hardware.profile ({}). A third frozen profile is a schema change before it \ + is a bundle value.", + profile.name, + NAMED_HARDWARE_PROFILES.join(", ") + ) + })?; + + // The comparison said this host is the reference host; the fields about to + // be emitted still say otherwise. Two statements about one host that + // disagree is a refusal, not a value to pick between. + let placeholders: Vec<&str> = emitted_fields + .iter() + .filter_map(|(field, value)| match value { + J::Str(text) => HARDWARE_PLACEHOLDER_MARKERS + .iter() + .any(|marker| text.contains(marker)) + .then_some(field.as_str()), + J::Bool(_) | J::Int(_) | J::Float(_) | J::Array(_) | J::Object(_) => None, + }) + .collect(); + if !placeholders.is_empty() { + return Err(format!( + "refusing to emit a bundle: the host facts match the frozen profile {name}, but \ + the hardware fields this bundle would carry are still placeholders \ + ({}). A bundle cannot name a frozen profile in one field and record \ + \"not measured here\" in the next.", + placeholders.join(", ") + )); + } + + Ok(HardwareProfileDerivation { + name, + unobserved, + mismatched, + }) +} + // --------------------------------------------------------------------------- // The bundle // --------------------------------------------------------------------------- @@ -728,6 +1920,11 @@ pub struct BundleInputs { /// is forbidden to perform. pub acknowledged_sequences_reconciled: bool, pub skeleton: bool, + /// What the run observed, from which `run_conditions` is derived. Carried + /// as observations rather than as the ten declarations so the derivation + /// itself is under test: a fixture that handed the emitter ten finished + /// strings would test the JSON writer and nothing else. + pub observations: RunObservations, } /// Whether the run met its gate. @@ -763,13 +1960,38 @@ pub const STORAGE_PRIMITIVE_P99_CEILING_MICROS: u64 = 50_000; /// `preliminary` while quietly exceeding the gate ceiling is precisely the /// unrepresentable-failure problem in a different costume. `preliminary` means /// "no ceiling missed, but this run did not measure the gate's workload". -pub fn storage_primitive_outcome(inputs: &BundleInputs) -> Outcome { +/// +/// A run that did not meet the reference environment cannot be a `pass` at any +/// gate — the schema says so in one rule, and this says it before the schema +/// has to, so the emitter never has to be told by a validator that it wrote a +/// claim it had not earned. +/// +/// # Every condition a `pass` costs, not only the binding one +/// +/// `bench/result-schema.json` requires four more declarations of a passing +/// `storage_primitive` bundle: the root built by `StoreEngine::open`, the +/// transactions through `StoreEngine::submit`, `checkpointing: "exercised"`, +/// and `index_maintenance: "runs_sealed"`. Until hardware recognition worked, +/// `environment_fidelity` was the only thing standing between this function and +/// a `pass`, so the other four were never reached — and the moment recognition +/// starts working, an emitter that checked only fidelity would produce a `pass` +/// the schema then rejects. A bundle refused by its own validator is a harness +/// that learned what it had claimed from a validator, which is the sequence +/// this file exists to avoid. Every condition is checked here, at the same +/// altitude, so the emitter's answer and the schema's answer cannot diverge. +pub fn storage_primitive_outcome(inputs: &BundleInputs, conditions: &RunConditions) -> Outcome { if inputs.latency_p99 > STORAGE_PRIMITIVE_P99_CEILING_MICROS || inputs.windows_meeting_target_percent < 95.0 { return Outcome::Fail; } - if inputs.skeleton { + if inputs.skeleton + || conditions.environment_fidelity != "reference_profile" + || conditions.initialization_path != "store_engine_open" + || conditions.mutation_path != "store_engine_submit" + || conditions.checkpointing != "exercised" + || conditions.index_maintenance != "runs_sealed" + { return Outcome::Preliminary; } Outcome::Pass @@ -828,11 +2050,56 @@ fn verdicts_for(gate: &str, value: &str) -> J { ) } +/// The bundle bytes, for callers that need nothing else. pub fn build_bundle( repo_root: &Path, workload: &FrozenWorkload, inputs: &BundleInputs, ) -> Result { + assemble_bundle(repo_root, workload, inputs).map(|assembled| assembled.json) +} + +/// A bundle and the derivations the harness also has to report. +pub struct AssembledBundle { + pub json: String, + pub outcome: Outcome, + pub conditions: RunConditions, + /// Why `hardware.profile` came out the way it did. Reported on the + /// harness's own stdout so the facts the deployed-node harness still owes + /// are a list a reader can act on rather than a sentence in a doc comment. + pub hardware: HardwareProfileDerivation, +} + +pub fn assemble_bundle( + repo_root: &Path, + workload: &FrozenWorkload, + inputs: &BundleInputs, +) -> Result { + // A run that measured nothing may not make a claim about what it measured. + // + // Every one of `setup_traffic_excluded`, `unique_blob_tree_commit_ids`, and + // `objects_new_equals_three_per_commit` is *vacuously* true over an empty + // set, which is exactly why none of them may be earned that way: a + // uniqueness claim over zero identifiers and a three-objects-per-commit + // claim over zero commits are statements about nothing, presented in the + // same field a real run uses. The schema is no help here — it validates a + // zero-commit bundle cheerfully, because `counts` is `nonnegative` and the + // claims are `const true` — so this is the only place the refusal can live. + // + // A refusal by name, not a `false`: the schema pins those claims to + // `const true` on the submit path, so emitting `false` is unavailable, and + // emitting `true` would be the untrue statement this check exists to stop. + if inputs.counted_commits == 0 || inputs.acknowledged_requests == 0 { + return Err(format!( + "refusing to emit a bundle: the measured interval contains {} counted commit(s) \ + and {} acknowledged request(s). verification.setup_traffic_excluded, \ + unique_blob_tree_commit_ids, and objects_new_equals_three_per_commit are all \ + vacuously true over an empty set, and a claim that cannot fail is not a check. \ + The schema pins them to true rather than permitting false, so a zero-work run \ + is refused by name instead of being published with claims it did not earn.", + inputs.counted_commits, inputs.acknowledged_requests + )); + } if inputs.acknowledged_loss != 0 { return Err(format!( "refusing to emit a bundle: {} acknowledged operations are absent \ @@ -880,7 +2147,7 @@ pub fn build_bundle( ), ( "cargo_lock_digest".into(), - jstr(digest_file(&repo_root.join("Cargo.lock"))), + jstr(digest_file(&repo_root.join("Cargo.lock"))?), ), ( "rustc".into(), @@ -895,9 +2162,12 @@ pub fn build_bundle( ), ]); - let binary_digest = std::env::current_exe() - .map(|path| digest_file(&path)) - .unwrap_or_else(|_| digest_hex(b"")); + // The binary that produced the numbers, refused rather than blanked: an + // unreadable executable leaves `artifacts.binary_digest` naming nothing. + let binary_digest = digest_file( + &std::env::current_exe() + .map_err(|error| format!("refusing to emit a bundle: current_exe: {error}"))?, + )?; let artifacts = J::Object(vec![ ("binary_digest".into(), jstr(binary_digest)), @@ -905,13 +2175,13 @@ pub fn build_bundle( "config_digest".into(), jstr(digest_file( &repo_root.join("bench/reference-hardware.toml"), - )), + )?), ), ( "workload_digest".into(), jstr(digest_file( &repo_root.join("bench/workloads/small-commit.toml"), - )), + )?), ), ( "corpus_digest".into(), @@ -954,8 +2224,7 @@ pub fn build_bundle( ("generator".into(), jstr(workload.generator.clone())), ]); - let hardware = J::Object(vec![ - ("profile".into(), jstr("diagnostic")), + let mut hardware_fields: Vec<(String, J)> = vec![ ( "cpu".into(), jstr(or_unknown( @@ -984,12 +2253,16 @@ pub fn build_bundle( "microcode revision", )), ), - ( - "ram_bytes".into(), - jint(read_total_ram_bytes().max(1) as i128), - ), + ("ram_bytes".into(), jint(read_total_ram_bytes()? as i128)), ("swap_events".into(), jint(0i128)), - ("filesystem".into(), jstr("btrfs")), + // Read back from `/proc/mounts` for the store root this run used, not + // the frozen profile's `btrfs` restated. A bundle that names the + // filesystem it was supposed to run on tells a reader nothing about the + // one it ran on. + ( + "filesystem".into(), + jstr(inputs.observations.store_root_filesystem.clone()), + ), ( "mount_options".into(), J::Array(vec![jstr(inputs.store_directory_attributes.clone())]), @@ -1011,11 +2284,63 @@ pub fn build_bundle( "kernel".into(), jstr(or_unknown(command_output("uname", &["-sr"]), "kernel")), ), - ]); + ]; + + // Derived by comparing what this run read off the host against the complete + // `[[profile]]` tables of the frozen file, and derived before + // `run_conditions` because `environment_fidelity` is conditioned on it: a + // bundle may only claim the reference environment if it named a frozen + // hardware profile, and it may only name one if the facts compared equal. + let frozen = load_frozen_profile(&repo_root.join("bench/reference-hardware.toml"))?; + let host_facts = observe_host_facts(&inputs.observations.store_root_filesystem); + let hardware_derivation = hardware_profile_of(&frozen.profiles, &host_facts, &hardware_fields)?; + let hardware_profile = hardware_derivation.name; + hardware_fields.insert(0, ("profile".into(), jstr(hardware_profile))); + let hardware = J::Object(hardware_fields); + + let conditions = RunConditions::derive( + &inputs.observations, + inputs.store_directory_attributes_verified, + hardware_profile, + )?; + + // Scope 6.6 item 5c. Both sides counted: `objects_new` was summed from the + // store's own receipts and `counted_commits` was counted as commits + // acknowledged, so this comparison can fail — which is the entire + // difference between the claim and the tautology it was excluded for. A + // disagreement is a refusal and never a `false`: the schema pins the claim + // to `const true`, and emitting the flag as `false` would be recording a + // failed invariant as a negative result. + if conditions.objects_new_source == "summed_from_receipts" { + let expected = inputs + .counted_commits + .checked_mul(OBJECTS_PER_COMMIT) + .ok_or_else(|| "counted_commits * 3 overflows u64".to_string())?; + if inputs.objects_new != expected { + return Err(format!( + "refusing to emit a bundle: the receipts reported {} new objects across \ + {} counted commits, and {OBJECTS_PER_COMMIT} per commit would be {expected}. \ + The store stages exactly one blob, one tree, and one commit per \ + transaction, so a disagreement is a finding about the store or about \ + this harness's counting — not a flag to emit as false.", + inputs.objects_new, inputs.counted_commits + )); + } + } let deployment = J::Object(vec![ - ("persistent_data_mount".into(), J::Bool(true)), - ("tmpfs".into(), J::Bool(false)), + // Observed, not asserted. A tmpfs run is representable and + // mechanically disqualified as of contract review 2026-07-28-C; it was + // previously neither, which is why the number that prompted the review + // could live only in a paragraph. + ( + "persistent_data_mount".into(), + J::Bool(!inputs.observations.store_root_is_tmpfs), + ), + ( + "tmpfs".into(), + J::Bool(inputs.observations.store_root_is_tmpfs), + ), ("overlay".into(), J::Bool(false)), ("remote_storage".into(), J::Bool(false)), ("durability_enabled".into(), J::Bool(true)), @@ -1127,6 +2452,14 @@ pub fn build_bundle( ( "configured_ceilings".into(), J::Object(vec![ + // The `u32` from the `StoreOptions` the store was opened with, + // carried through the run rather than `ENGINE_MAX_INDEX_RUNS` + // written out a second time. It is the ceiling this workload + // genuinely reaches: `submit` refuses `NotImplemented` at it. + ( + "max_index_runs".into(), + jint(i128::from(inputs.observations.configured_max_index_runs)), + ), ( "writer_group_transactions".into(), J::Float(workload.max_writer_group_transactions as f64), @@ -1185,16 +2518,24 @@ pub fn build_bundle( ("torn_transactions".into(), jint(0i128)), ]); - // Five neighbours are absent, not false. `unique_blob_tree_commit_ids`, - // `objects_new_equals_three_per_commit`, `blobs_recomputed`, - // `operation_receipts_reconciled`, and `metadata_complete` are claims about - // an object graph that does not exist below `engine.rs`, and contract - // review 2026-07-24-B (amended) forbids them at `storage_primitive` rather - // than merely permitting their omission. `false` would say the check was - // applicable and failed, which is a different untrue statement; omission is - // the only encoding that says "not applicable here", the same treatment + // Three neighbours are absent, not false. `blobs_recomputed`, + // `metadata_complete`, and `operation_receipts_reconciled` are claims this + // run did not earn, and the schema forbids them here rather than merely + // permitting their omission. `false` would say the check was applicable and + // failed, which is a different untrue statement; omission is the only + // encoding that says "not performed here", the same treatment // `storage.store_directory_attributes_verified` gets. - let verification = J::Object(vec![ + // + // `operation_receipts_reconciled` is the one to read twice. It is approved + // in principle (contract review 2026-07-28-C) and **not earned**: the + // reconciliation below accepts any `TransactionStatus::Committed(_)` + // without comparing the payload, and the `receipt_digest` the harness + // journals is `blake3(operation_id)` rather than a digest of the receipt. + // So `receipt_reconciliation` declares `acceptance_of_any_committed_status` + // and the schema forbids the claim on that declaration. Earning it means + // reconciling the exact receipt or a frozen canonical receipt digest, and + // then the schema *requires* the claim rather than permitting it. + let mut verification_fields = vec![ ("setup_traffic_excluded".into(), J::Bool(true)), // False, unconditionally, and not a placeholder. Proving a commit is in // the recovered *closure* of its acknowledged ref requires walking a @@ -1207,9 +2548,21 @@ pub fn build_bundle( // adopted. Guarded by the refusal above, so this constant is a report // of a completed reconciliation and not a decoration. ("acknowledged_sequences_reconciled".into(), J::Bool(true)), - ]); + ]; - let outcome = storage_primitive_outcome(inputs); + // The two claims contract review 2026-07-28-C granted, each emitted only + // where its own declaration says the check was performed. The `if`s are not + // belt and braces over the schema: they are what makes the claim follow + // from the check rather than from which path this code took. + if conditions.commit_id_uniqueness == "checked_globally_across_ack_records" { + verification_fields.push(("unique_blob_tree_commit_ids".into(), J::Bool(true))); + } + if conditions.objects_new_source == "summed_from_receipts" { + verification_fields.push(("objects_new_equals_three_per_commit".into(), J::Bool(true))); + } + let verification = J::Object(verification_fields); + + let outcome = storage_primitive_outcome(inputs, &conditions); let mut storage_fields = vec![ ( @@ -1257,6 +2610,9 @@ pub fn build_bundle( ("workload".into(), workload_value), ("hardware".into(), hardware), ("deployment".into(), deployment), + // The conditions the run was obtained under, as values a consumer + // checks. Every claim below is conditioned on one of them. + ("run_conditions".into(), conditions.to_json()), ("measurement".into(), measurement), ("counts".into(), counts), ("bytes".into(), bytes), @@ -1289,20 +2645,32 @@ pub fn build_bundle( ("storage".into(), storage), ]); - Ok(json::to_string(&bundle)) + Ok(AssembledBundle { + json: json::to_string(&bundle), + outcome, + conditions, + hardware: hardware_derivation, + }) } -fn read_total_ram_bytes() -> u64 { - std::fs::read_to_string("/proc/meminfo") - .ok() - .and_then(|text| { - text.lines() - .find(|line| line.starts_with("MemTotal:")) - .and_then(|line| line.split_whitespace().nth(1)) - .and_then(|value| value.parse::().ok()) - }) +/// Installed memory, or a refusal. +/// +/// The `unwrap_or(0)` that stood here was emitted as `ram_bytes` after a +/// `.max(1)`, so a host whose `/proc/meminfo` could not be read published one +/// byte of RAM as an observation. A value nobody read is not an observation. +fn read_total_ram_bytes() -> Result { + let text = std::fs::read_to_string("/proc/meminfo") + .map_err(|error| format!("refusing to emit a bundle: /proc/meminfo: {error}"))?; + text.lines() + .find(|line| line.starts_with("MemTotal:")) + .and_then(|line| line.split_whitespace().nth(1)) + .and_then(|value| value.parse::().ok()) .map(|kib| kib * 1024) - .unwrap_or(0) + .ok_or_else(|| { + "refusing to emit a bundle: /proc/meminfo carries no parsable MemTotal, so \ + resources.ram_bytes has no reading behind it." + .to_string() + }) } // --------------------------------------------------------------------------- @@ -1352,15 +2720,112 @@ struct SkeletonRun { /// graph traversal happened or could have. acknowledged_sequences_reconciled: bool, ack_journal_digest: String, + facts: RunFacts, +} + +/// The character device whose every write fails with `ENOSPC`. +/// +/// The armed fault below writes to it rather than fabricating an +/// `io::Error`, so the error the refusal path handles is the kernel's and not +/// the harness's idea of one. +const FULL_DEVICE: &str = "/dev/full"; + +/// A deterministic acknowledgment-journal write failure, armed by name. +/// +/// The failure this exists to exercise is not hypothetical: `ENOSPC` and +/// `EDQUOT` on the journal's filesystem both surface here, and a run on tmpfs +/// meets them routinely. Racing a real full device is not a test, so the fault +/// is armed at a chosen append instead — **after** at least one commit, because +/// that is the case the zero-work guard cannot see. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +enum AckJournalFault { + #[default] + None, + /// The append with this zero-based index is issued against `/dev/full` + /// instead of the journal, and its error is propagated as the journal's. + EnospcOnAppend(u64), +} + +/// The acknowledgment journal as the run loop uses it, plus the armed fault. +/// +/// A wrapper rather than a branch at the call site: the fault has to fire +/// inside the same call the run loop already makes, or the regression would be +/// exercising a path the emitter does not take. +struct AckSink { + journal: ExternalAckJournal, + fault: AckJournalFault, + appended: u64, + /// Opened when the fault is armed, not when it fires, so a host where the + /// fault cannot be induced says so before the run rather than after it. + full_device: Option, +} + +impl AckSink { + fn open(path: &Path, fault: AckJournalFault) -> Result { + let journal = ExternalAckJournal::open(path).map_err(|e| format!("ack journal: {e}"))?; + let full_device = match fault { + AckJournalFault::None => None, + AckJournalFault::EnospcOnAppend(_) => Some( + std::fs::OpenOptions::new() + .write(true) + .open(FULL_DEVICE) + .map_err(|error| { + format!( + "refusing to arm the acknowledgment-journal fault: \ + {FULL_DEVICE} could not be opened for writing ({error}). The \ + injected failure is a real kernel ENOSPC from that device \ + rather than an io::Error this harness invented, so a host \ + that cannot supply one cannot run the injection at all." + ) + })?, + ), + }; + Ok(Self { + journal, + fault, + appended: 0, + full_device, + }) + } + + fn append_durable( + &mut self, + record: &AckRecord, + ) -> Result<(), levcs_protocol::oracle::AckJournalError> { + use std::io::Write as _; + + if self.fault == AckJournalFault::EnospcOnAppend(self.appended) { + let device = self + .full_device + .as_mut() + .expect("an armed fault opened /dev/full before the run"); + return Err(match device.write_all(&[0u8]) { + Err(error) => error.into(), + // Named rather than folded into the fault: a device that + // accepted the write did not produce the failure this run + // claims to be injecting, and reporting it as one would make + // the regression pass on evidence it never had. + Ok(()) => std::io::Error::other(format!( + "{FULL_DEVICE} accepted a write, so the armed acknowledgment-journal \ + fault produced no kernel error to propagate" + )) + .into(), + }); + } + self.journal.append_durable(record)?; + self.appended += 1; + Ok(()) + } } fn run_skeleton( root: &Path, ack_path: &Path, + ack_fault: AckJournalFault, group_len: usize, seconds: u64, ) -> Result { - let mut ack = ExternalAckJournal::open(ack_path).map_err(|e| format!("ack journal: {e}"))?; + let mut ack = AckSink::open(ack_path, ack_fault)?; let mut acknowledged = 0u64; let mut drive = ShardDrive::create(root, 0, 1).map_err(|e| format!("create: {e}"))?; let mut namespace_bytes = [0u8; 32]; @@ -1428,8 +2893,21 @@ fn run_skeleton( // below vacuous, because a lost acknowledgment would also be a missing // record. for sequence in &sequences { + // The same incomplete-accounting refusal the submit path makes, in + // the same words: the group's fence is already inside the counters + // this run would report, so a frame that cannot be acknowledged + // leaves the run unable to account for work it already measured. ack.append_durable(&skeleton_ack_record(*sequence)) - .map_err(|e| format!("ack append: {e}"))?; + .map_err(|error| { + format!( + "refusing to emit a bundle: incomplete accounting. Shard \ + sequence {sequence} was appended and fenced, but appending its \ + acknowledgment to the external journal failed: {error} \ + ({error:?}). The fence is already inside the counters this run \ + reports and the transaction cannot be counted, so every total \ + it could emit would describe a workload that did not happen." + ) + })?; acknowledged += 1; } } @@ -1460,7 +2938,30 @@ fn run_skeleton( let torn_transactions = sequences.missing_sequences; let repeated_adoptions = sequences.repeated_adoptions(); + // The seam has no engine of its own, so the checkpoint question is put to a + // real `StoreEngine` on a throwaway root next to this run's. Declaring + // `unimplemented` here without asking would be this file's opinion of what + // `engine.rs` does, which is exactly what the declaration is not for. + let checkpoint = probe_checkpointing_on_fresh_root(&root.with_extension("checkpoint-probe"))?; + Ok(SkeletonRun { + facts: RunFacts { + initialization: Initialization::ShardDriveCreate, + mutation: MutationPath::JournalDrive, + checkpoint, + index: IndexObservation::NoIndexInPath, + // Nothing below `engine.rs` consults `max_index_runs`, so the seam + // runs under the store default: the value is read off an untouched + // `StoreOptions` rather than written as 64. + configured_max_index_runs: levcs_store::StoreOptions::new(root).max_index_runs, + default_max_index_runs: levcs_store::StoreOptions::new(root).max_index_runs, + receipts: ReceiptComparison::NoReceiptsInPath, + objects_new_counted: false, + // The identifiers in these acknowledgment records are fabricated by + // `skeleton_ack_record`, not produced by a store. Unioning them + // would establish a property of this harness's hash domains. + uniqueness: UniquenessCheck::NotPerformed, + }, groups: latencies.len() as u64, transactions: ordinal, latencies_micros: latencies, @@ -1474,7 +2975,7 @@ fn run_skeleton( acknowledged_sequences_reconciled: acknowledged_loss == 0 && sequences.is_strictly_contiguous() && records.len() as u64 == acknowledged, - ack_journal_digest: digest_file(ack_path), + ack_journal_digest: digest_file(ack_path)?, }) } @@ -1518,12 +3019,9 @@ fn skeleton_ack_record(sequence: u64) -> AckRecord { // // # What this run can and cannot claim, stated before the code // -// Three of B1's unimplemented deliverables bound it, and every one of them is a -// bound on the *bundle*, not merely on this file: +// Two of B1's unimplemented deliverables bound it, and both are a bound on the +// *bundle*, not merely on this file: // -// * `StoreEngine::open` refuses startup state 1, so the root is created by -// `segment::initialize_root`. The measured path is production; the path -// that made the store it measures is not. // * `submit` refuses `NotImplemented` after `max_index_runs` group // publications, because sealing the in-memory index delta into an // `IndexRun` is unimplemented. The ceiling is raised here so the run can @@ -1536,6 +3034,14 @@ fn skeleton_ack_record(sequence: u64) -> AckRecord { // checkpointing disabled. This one was. That alone makes the // `storage_primitive` gate unearnable today, whatever the rate says. // +// A third bound stood here until B1 landed startup state 1: `StoreEngine::open` +// refused to build an absent root, so this run seeded one with +// `segment::initialize_root` and measured the production path over a store +// production had not built. That is charter item 8 in its most literal form, +// and it is now closed — the root below is created by `open` itself, and the +// bundle's `initialization_path` reports that as an observation of the `FORMAT` +// marker rather than as a label. +// // What genuinely improves over Wave A: the signer is real Ed25519 and its cost // is measured rather than reported as a zero; every commit carries the // canonical three objects and a typed ref CAS, so `objects_new` is counted from @@ -1684,6 +3190,51 @@ fn engine_evidence() -> levcs_protocol::v2::TransactionEvidenceV1 { } } +/// Durability fences the store has performed, from the store's own counters. +/// +/// One reader for the baseline and for the total, so the two figures cannot +/// come from two different notions of what a fence is. +fn sum_fences(engine: &levcs_store::StoreEngine, shard_count: u16) -> Result { + let mut total = 0u64; + for shard in 0..shard_count { + // A shard whose counters cannot be read used to contribute zero, which + // is the same defect class as the acknowledgment failure above: the + // number the bundle publishes silently loses a term, and the reader + // cannot tell a shard that fenced nothing from a shard that was not + // asked. Both the baseline and the total come through here, so a quiet + // zero could also make the measured interval look smaller than it was. + let counters = engine.durability_counters(shard).ok_or_else(|| { + format!( + "refusing to measure: the store returned no durability counters for shard \ + {shard} of {shard_count}. The fence total is what bounds every durability \ + claim in the bundle, and a shard counted as zero fences would understate \ + it without saying so." + ) + })?; + total += counters.fdatasync; + } + Ok(total) +} + +/// Durability and signing work performed *before* the measured interval opened. +/// +/// Repositories are created through the same `StoreEngine::submit` path the +/// measurement uses, so creating them fences and signs. Both counters therefore +/// have to be snapshotted once creation has completed and subtracted, or every +/// bundle reports setup fences and setup signatures while asserting +/// `verification.setup_traffic_excluded: true` — a bundle making a false +/// statement about its own methodology, which is worse than a wrong number +/// because a wrong number invites scrutiny and a false exclusion claim deflects +/// it. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub struct SetupTraffic { + /// `DurabilityCounters::fdatasync`, summed across shards, at the instant + /// the last repository creation returned. + pub fences: u64, + /// Signing samples the `MeasuringSigner` had recorded at the same instant. + pub signings: u64, +} + /// What one engine-driven run measured. struct EngineRun { latencies_micros: Vec, @@ -1695,43 +3246,72 @@ struct EngineRun { acknowledged_loss: u64, torn_transactions: u64, repeated_adoptions: u64, + /// Fences performed **inside the measured interval**: the store's counter at + /// the end, less its value at the instant repository creation completed. fences: u64, /// Group publications, counted as fences: `journal::append_group_and_fence` /// performs exactly one per group and A1's acceptance pins that. groups: u64, signing_micros_p50: f64, + /// Signing samples taken inside the measured interval, on the same basis. signings: u64, + /// What the two counters read at the instant the measured interval opened, + /// and what they read when it closed. Carried out of the run so the + /// exclusion can be asserted against `DurabilityCounters` rather than + /// against the emitter's intention to have excluded it. + setup: SetupTraffic, + total_fences: u64, + total_signings: u64, refused: u64, first_refusal: Option, acknowledged_sequences_reconciled: bool, ack_journal_digest: String, - lock_release_attempts: u32, + facts: RunFacts, } #[allow(clippy::too_many_lines)] fn run_engine( root: &Path, ack_path: &Path, + ack_fault: AckJournalFault, group_len: usize, seconds: u64, shard_count: u16, submitters_per_shard: usize, ) -> Result { - use levcs_store::segment::{initialize_root, RootLayout}; + use levcs_store::segment::RootLayout; use levcs_store::transaction::StagedObject; - use levcs_store::types::{DurabilityCounters, OperationId, PrivilegedConstruction, StoreError}; + use levcs_store::types::{OperationId, PrivilegedConstruction, StoreError}; use levcs_store::{StoreEngine, ValidatedTransaction}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; - initialize_root( - &RootLayout::new(root), - shard_count, - [0x9e; 16], - engine_now_micros(), - &DurabilityCounters::default(), - ) - .map_err(|e| format!("initialize_root: {e}"))?; + // A run with no submitter measures nothing, and a bundle from it asserts + // `setup_traffic_excluded`, `unique_blob_tree_commit_ids`, and + // `objects_new_equals_three_per_commit` over an empty set, where each is + // vacuously true. `assemble_bundle` refuses such a bundle; this refuses + // before spending the run, and refuses at the entry point rather than at + // the flag parser so an in-process caller gets the same answer the CLI + // does. + if submitters_per_shard == 0 { + return Err( + "refusing to measure: --submitters-per-shard 0 spawns no submitter, so the \ + measured interval contains no transaction at all. The repositories would \ + still be created, so the run would report their fences and their signatures \ + as if they were measured work, and the bundle would assert claims that are \ + vacuously true over zero commits. The schema does not permit false for those \ + claims, so this is refused by name." + .to_string(), + ); + } + if shard_count == 0 { + return Err( + "refusing to measure: --shards 0 creates no repository and routes no \ + transaction, which is the same zero-work run --submitters-per-shard 0 \ + produces, reached through the other flag." + .to_string(), + ); + } let signer = Arc::new(MeasuringSigner::new()); let build_options = || { @@ -1746,14 +3326,57 @@ fn run_engine( options }; - let engine = StoreEngine::open(build_options()).map_err(|e| format!("open: {e}"))?; + // Read back from the options this store is opened with, and compared + // against the library's own default rather than against a 64 written here. + // `resources.configured_ceilings.max_index_runs` and + // `run_conditions.index_run_ceiling` both come from these two values, so + // neither can drift from what the run configured. + let opened_with = build_options(); + let configured_max_index_runs = opened_with.max_index_runs; + let default_max_index_runs = levcs_store::StoreOptions::new(root).max_index_runs; + + // The store that is about to be measured is now **built by the entry point + // a consumer calls**. It used to be seeded with `segment::initialize_root`, + // because `StoreEngine::open` refused startup state 1, and the benchmark + // measured the production path over a root production had not built — + // charter item 8, disclosed in this file, in the crash-matrix fixture, and + // in the bundle's own `initialization_path`. B1 landed state 1 and the + // weakening is closed here rather than re-worded. + // + // `initialization_path` is then **observed**, not labelled: `FORMAT` is + // absent before the call and present after, so the declaration reports what + // this open did. A root that already carried `FORMAT` was built by + // something this run cannot name, and that is a refusal below rather than a + // guess. + let layout = RootLayout::new(root); + let format_before_open = layout.format_path().exists(); + + let engine = StoreEngine::open(opened_with).map_err(|e| format!("open: {e}"))?; + + let initialization = if format_before_open { + return Err(format!( + "refusing to measure {}: it already carries a FORMAT marker, so this run did \ + not build the store it is about to measure and cannot say what did. \ + run_conditions.initialization_path names the three paths that create a \ + root, and \"whatever was here already\" is not one of them.", + root.display() + )); + } else if layout.format_path().exists() { + Initialization::StoreEngineOpen + } else { + return Err( + "StoreEngine::open returned without writing FORMAT to an absent root, so \ + the store this run is about to measure was not built by the production \ + entry point after all." + .to_string(), + ); + }; let namespaces: Vec = (0..shard_count) .map(|shard| engine_namespace(shard, shard_count)) .collect(); - let ack = - Mutex::new(ExternalAckJournal::open(ack_path).map_err(|e| format!("ack journal: {e}"))?); + let ack = Mutex::new(AckSink::open(ack_path, ack_fault)?); let acknowledged = AtomicU64::new(0); // One repository per shard, created before the measured window. @@ -1788,9 +3411,38 @@ fn run_engine( } } + // **The baselines, after creation has completed and before anything is + // measured.** Both counters are cumulative over the life of the engine, so + // reading them only at the end means reading them from zero — from before + // the repositories existed — and every bundle then reports the creation + // fences and the creation signatures inside the measured interval while + // asserting `verification.setup_traffic_excluded: true`. + // + // Snapshotted here, subtracted below, so the measured interval contains + // only measured work. `fdatasync` is read from the store's own + // `DurabilityCounters` and the signing count from the signer's own sample + // vector, which is what makes the exclusion checkable against a counter + // rather than against this comment. + let setup = SetupTraffic { + fences: sum_fences(&engine, shard_count)?, + signings: signer + .micros + .lock() + .unwrap_or_else(|p| p.into_inner()) + .len() as u64, + }; + let stop = AtomicBool::new(false); let refused = AtomicU64::new(0); let first_refusal: Mutex> = Mutex::new(None); + // A failure that leaves the run unable to account for work it already + // measured. Separate from `refused` on purpose: a refused submit is a + // transaction the store never committed, and this is a transaction it + // *did* commit whose acknowledgment the harness could not record. The + // first is a countable event; the second is the end of the run, because + // the fence and the signature are already inside the totals and the + // transaction can never be. + let unaccounted: Mutex> = Mutex::new(None); let results: Mutex> = Mutex::new(Vec::new()); let latencies: Mutex> = Mutex::new(Vec::new()); let objects_new = AtomicU64::new(0); @@ -1809,6 +3461,7 @@ fn run_engine( let stop = &stop; let refused = &refused; let first_refusal = &first_refusal; + let unaccounted = &unaccounted; let results = &results; let latencies = &latencies; let objects_new = &objects_new; @@ -1917,7 +3570,44 @@ fn run_engine( }; { let mut journal = ack.lock().unwrap_or_else(|p| p.into_inner()); - if journal.append_durable(&record).is_err() { + // Not a counter and not a `false`. The + // store committed this transaction and + // fenced and signed for it, so those costs + // are already inside the totals; the + // harness cannot record the acknowledgment, + // so the transaction can never be inside + // them. Ending the run here without saying + // so — which is what this did — omitted a + // committed transaction from the bundle + // while still counting its fence and its + // signature, and silently shortened the + // measured interval. `EDQUOT` on a full + // tmpfs is the ordinary way to get here. + if let Err(error) = journal.append_durable(&record) { + let mut slot = + unaccounted.lock().unwrap_or_else(|p| p.into_inner()); + if slot.is_none() { + *slot = Some(format!( + "refusing to emit a bundle: incomplete \ + accounting. StoreEngine::submit committed \ + operation {} in namespace {} at \ + repo_sequence {} and returned its receipt, \ + but appending that acknowledgment to the \ + external journal failed: {error} \ + ({error:?}). The commit's durability fence \ + and its signature are already inside the \ + counters this run reports and the \ + transaction itself can never be, so every \ + total this run could emit describes a \ + workload that did not happen. This is not \ + a refused submit and not a shortened run: \ + the harness cannot account for what it \ + measured, so it emits nothing.", + hex::encode(operation_id), + hex::encode(namespace.as_bytes()), + receipt.repo_sequence, + )); + } stop.store(true, Ordering::Relaxed); return; } @@ -1971,14 +3661,18 @@ fn run_engine( }); let elapsed = started.elapsed(); - let fences: u64 = (0..shard_count) - .map(|shard| { - engine - .durability_counters(shard) - .map(|counters| counters.fdatasync) - .unwrap_or(0) - }) - .sum(); + + // Before anything is read off a counter, let alone reported. A run that + // could not account for a committed transaction has no totals worth + // computing, and the emitter's `refused` guard is deliberately not the + // thing that catches this: `refused` counts submits the store declined, + // and folding an accounting failure into it would report a commit that + // happened as one that did not. + if let Some(failure) = unaccounted.into_inner().unwrap_or_else(|p| p.into_inner()) { + return Err(failure); + } + + let total_fences = sum_fences(&engine, shard_count)?; let latencies = latencies.into_inner().unwrap_or_else(|p| p.into_inner()); let results = results.into_inner().unwrap_or_else(|p| p.into_inner()); @@ -1989,34 +3683,62 @@ fn run_engine( let acknowledged = acknowledged.load(Ordering::Relaxed); let objects_new = objects_new.load(Ordering::Relaxed); let raw_bytes = raw_bytes.load(Ordering::Relaxed); - let mut signing = signer + let all_signing = signer .micros .lock() .unwrap_or_else(|p| p.into_inner()) .clone(); + let total_signings = all_signing.len() as u64; + + // The measured interval is what the totals hold *beyond* the baselines. A + // counter that went backwards is a store or a harness finding and not a + // number to publish, so the subtraction refuses rather than saturating. + let fences = total_fences.checked_sub(setup.fences).ok_or_else(|| { + format!( + "refusing to measure: the durability fence counter read {total_fences} at the \ + end of the measured interval and {} at its start. A counter that went \ + backwards cannot bound what the interval contained.", + setup.fences + ) + })?; + let signings = total_signings.checked_sub(setup.signings).ok_or_else(|| { + format!( + "refusing to measure: {total_signings} signing samples were recorded in total \ + and {} before the measured interval opened.", + setup.signings + ) + })?; + // Samples are appended in signing order, so the measured interval's samples + // are exactly the tail past the baseline. Sorted *after* the split, because + // sorting first would make the baseline index meaningless. + let mut signing = all_signing[setup.signings as usize..].to_vec(); signing.sort_unstable(); - let signings = signing.len() as u64; let signing_p50 = percentile(&signing, 0.50) as f64; drop(ack); drop(engine); - // The root lock is not always free when `StoreEngine::drop` returns; see - // the same finding recorded in `tests/support/engine_matrix.rs`. Bounded - // and reported, never silent. - let lock_wait_started = Instant::now(); - let mut lock_release_attempts = 0u32; - let reopened = loop { - lock_release_attempts += 1; - match StoreEngine::open(build_options()) { - Ok(engine) => break engine, - Err(StoreError::AlreadyLocked) - if lock_wait_started.elapsed() < Duration::from_secs(30) => - { - std::thread::sleep(Duration::from_micros(200)); - } - Err(other) => return Err(format!("reopen through production recovery: {other}")), + // One attempt, no budget, no sleep. Commit e050b6d fixed the root cause — + // `flock` lives on the open file description, and `lock_root` now returns + // an RAII `RootLock` that issues an explicit `LOCK_UN` in `Drop` — and the + // bounded wait that stood here existed only to compensate for that defect. + // Scope 3.1 says `AlreadyLocked` is a refusal and never a wait, so a + // harness that waited on it was asserting something the store does not + // promise, and leaving the wait in place would hide a recurrence behind a + // second attempt that succeeded. + let reopened = match StoreEngine::open(build_options()) { + Ok(engine) => engine, + Err(StoreError::AlreadyLocked) => { + return Err( + "the root lock was still held on the first StoreEngine::open after the \ + measured engine was dropped. Scope 3.1 makes AlreadyLocked a refusal \ + and never a wait, and commit e050b6d released the lock explicitly in \ + RootLock::drop, so this is a recurrence of that defect rather than a \ + slow reclaim to sleep through." + .to_string(), + ) } + Err(other) => return Err(format!("reopen through production recovery: {other}")), }; // Reconciliation, through the production status read rather than through a @@ -2044,6 +3766,11 @@ fn run_engine( } } } + + // What the store answers about checkpoints, asked of the store rather than + // asserted about it. Asked after the reconciliation so a checkpoint the day + // this stops refusing cannot move what the reconciliation read. + let checkpoint = probe_checkpointing(&reopened)?; drop(reopened); // Per-repository sequence integrity, through the one shared checker. @@ -2063,7 +3790,44 @@ fn run_engine( let records = ExternalAckJournal::recover(ack_path).map_err(|e| format!("ack recover: {e}"))?; + // Scope 6.6 item 5d: uniqueness established **globally**, across every + // recovered acknowledgment record at once. Per-record checking cannot see a + // collision between two records, and the distinctness of the generator's + // seed domains is an argument about likelihood rather than an observation; + // the schema names both of those as separate values precisely so a harness + // that did either has something truthful to record instead of this claim. + let uniqueness = check_global_uniqueness(&records)?; + + // What is on the device under `shards/*/indexes`, read back and validated + // as index runs rather than counted as directory entries. + let index_scan = scan_index_runs(root, shard_count); + Ok(EngineRun { + facts: RunFacts { + initialization, + mutation: MutationPath::StoreEngineSubmit, + checkpoint, + index: IndexObservation::StoreRoot { + validated_runs: index_scan.validated_runs, + unvalidatable: index_scan.unvalidatable, + groups: fences, + // Nothing seals, and `StoreEngine` exposes no reading of how + // many published deltas are outstanding. `None` is that + // absence, and it is what makes `runs_sealed` unearnable until + // the reading exists rather than inferable from a file count. + unsealed_delta_backlog: None, + }, + configured_max_index_runs, + default_max_index_runs, + // The reconciliation loop above accepts any `Committed(_)` without + // comparing the payload, and the `receipt_digest` written into + // every `AckRecord` is `blake3(operation_id)`. Recorded here at the + // site that decides it, so earning the claim is a change to the + // loop and to this value together rather than to this value alone. + receipts: ReceiptComparison::AnyCommittedStatusAccepted, + objects_new_counted: true, + uniqueness, + }, groups: fences, transactions: acknowledged, latencies_micros: latencies, @@ -2077,14 +3841,57 @@ fn run_engine( fences, signing_micros_p50: signing_p50, signings, + setup, + total_fences, + total_signings, refused, first_refusal, acknowledged_sequences_reconciled: acknowledged_loss == 0 && torn_transactions == 0 && repeated_adoptions == 0 && records.len() as u64 == acknowledged, - ack_journal_digest: digest_file(ack_path), - lock_release_attempts, + ack_journal_digest: digest_file(ack_path)?, + }) +} + +/// Every blob, tree, and commit identifier in every recovered acknowledgment +/// record, unioned into one set per kind. +/// +/// A repeat is a **refusal**, not a `false`: the schema pins +/// `unique_blob_tree_commit_ids` to `const true`, so a run that saw a collision +/// has no way to report it as a failed check and must not report it as a +/// passed one either. The bundle is what would be wrong. +fn check_global_uniqueness(records: &[AckRecord]) -> Result { + use std::collections::HashSet; + + /// One kind, unioned across every record before anything is concluded. + fn union( + kind: &str, + records: &[AckRecord], + select: impl Fn(&AckRecord) -> &[ObjectId], + ) -> Result { + let mut seen: HashSet = HashSet::new(); + for record in records { + for id in select(record) { + if !seen.insert(*id) { + return Err(format!( + "refusing to emit a bundle: the {kind} identifier {} appears in \ + more than one place across the recovered acknowledgment \ + records. Every counted commit must introduce distinct objects, \ + and a collision is a finding about the run rather than a \ + verification flag to emit as false.", + hex::encode(id.0) + )); + } + } + } + Ok(seen.len() as u64) + } + + Ok(UniquenessCheck::GlobalAcrossAckRecords { + blob_ids: union("blob", records, |record| &record.blob_ids)?, + tree_ids: union("tree", records, |record| &record.tree_ids)?, + commit_ids: union("commit", records, |record| &record.commit_ids)?, }) } @@ -2106,7 +3913,6 @@ struct MeasuredRun { /// this gate on the drive path: a derived figure asserted against its own /// derivation is a tautology. objects_new: u64, - objects_new_counted: bool, bytes: u64, elapsed: Duration, acknowledged: u64, @@ -2116,6 +3922,11 @@ struct MeasuredRun { fences: u64, signing_micros_p50: f64, signings: u64, + /// Fences and signatures performed before the measured interval opened, and + /// the totals they were subtracted from. + setup: SetupTraffic, + total_fences: u64, + total_signings: u64, acknowledged_sequences_reconciled: bool, ack_journal_digest: String, path: &'static str, @@ -2124,9 +3935,8 @@ struct MeasuredRun { /// from the one it names. refused: u64, first_refusal: Option, - /// Attempts the post-run reopen needed to acquire the root lock. `1` is the - /// expected reading; see the note on `run_engine`. - lock_release_attempts: u32, + /// What this run observed about itself, recorded where it happened. + facts: RunFacts, } impl From for MeasuredRun { @@ -2135,7 +3945,7 @@ impl From for MeasuredRun { groups: run.groups, transactions: run.transactions, objects_new: run.transactions * OBJECTS_PER_COMMIT, - objects_new_counted: false, + facts: run.facts, bytes: run.bytes, elapsed: run.elapsed, acknowledged: run.acknowledged, @@ -2145,13 +3955,19 @@ impl From for MeasuredRun { fences: run.fences, signing_micros_p50: 0.0, signings: 0, + // The journal seam creates no repository and signs nothing, so + // there is no setup traffic to exclude and the totals are the + // measured figures. Recorded as an observed zero rather than left + // out, so the same assertion covers both paths. + setup: SetupTraffic::default(), + total_fences: run.fences, + total_signings: 0, acknowledged_sequences_reconciled: run.acknowledged_sequences_reconciled, ack_journal_digest: run.ack_journal_digest, latencies_micros: run.latencies_micros, path: "drive", refused: 0, first_refusal: None, - lock_release_attempts: 1, } } } @@ -2162,7 +3978,7 @@ impl From for MeasuredRun { groups: run.groups, transactions: run.transactions, objects_new: run.objects_new, - objects_new_counted: true, + facts: run.facts, bytes: run.raw_bytes, elapsed: run.elapsed, acknowledged: run.acknowledged, @@ -2172,13 +3988,15 @@ impl From for MeasuredRun { fences: run.fences, signing_micros_p50: run.signing_micros_p50, signings: run.signings, + setup: run.setup, + total_fences: run.total_fences, + total_signings: run.total_signings, acknowledged_sequences_reconciled: run.acknowledged_sequences_reconciled, ack_journal_digest: run.ack_journal_digest, latencies_micros: run.latencies_micros, path: "submit", refused: run.refused, first_refusal: run.first_refusal, - lock_release_attempts: run.lock_release_attempts, } } } @@ -2303,8 +4121,15 @@ store-bench [flags] emit-skeleton --root P --out P --allow-unsigned [--repo-root P] [--seconds N] [--group-len N] [--skip-attribute-check] [--path submit|drive] [--shards N] [--submitters-per-shard N] + [--fail-ack-append-after N] run --root P --out-dir P (P2; blocked, see below) +--fail-ack-append-after N is fault injection, and a run that uses it emits no +bundle by construction: the Nth acknowledgment-journal append is issued against +/dev/full so the kernel's ENOSPC drives the emitter's incomplete-accounting +refusal. It exists so that refusal is exercised deterministically rather than +by racing a full device. + --path submit is the default and goes through StoreEngine::submit. --path drive is the Wave A journal seam, kept so the two measurements can be compared rather than confused; it signs nothing, creates no object, and issues no receipt. @@ -2474,13 +4299,26 @@ fn emit_skeleton( // promised. Same device here, different subtree — the strongest isolation // an unprivileged in-process harness can give it, and the reason the // deployed campaigns of §10 put it on another host. + // Fault injection, off unless named. There is no default index and no + // "0 means off": the flag's presence arms it and its absence does not, so + // an armed run is always something a caller asked for by name. + let ack_fault = match flags.get("fail-ack-append-after") { + None => AckJournalFault::None, + Some(_) => { + AckJournalFault::EnospcOnAppend(flags.number::("fail-ack-append-after", 0)?) + } + }; + let path = flags.get("path").unwrap_or("submit").to_string(); let run: MeasuredRun = if path == "submit" { let shards = flags.number::("shards", 4)?; let submitters = flags.number::("submitters-per-shard", group_len.max(1))?; - run_engine(&root, &ack_path, group_len, seconds, shards, submitters)?.into() + run_engine( + &root, &ack_path, ack_fault, group_len, seconds, shards, submitters, + )? + .into() } else if path == "drive" { - run_skeleton(&root, &ack_path, group_len, seconds)?.into() + run_skeleton(&root, &ack_path, ack_fault, group_len, seconds)?.into() } else { return Err(format!( "--path must be submit or drive, got {path:?}. `submit` is the production \ @@ -2519,6 +4357,12 @@ fn emit_skeleton( } }; + // Where the run actually ran, read back from `/proc/mounts`. The two + // deployment fields it feeds were unconditional constants until contract + // review 2026-07-28-C, which is how a tmpfs run could describe itself as + // persistent without anything noticing. + let mount = detect_mount(&root)?; + // The trim-settle interval between repetitions (scope 8.2). A one-shot // skeleton has no second repetition, so the interval is observed as zero // and recorded as such rather than omitted. @@ -2618,11 +4462,12 @@ fn emit_skeleton( free_bytes_required: requirement.required_bytes, acknowledged_sequences_reconciled: run.acknowledged_sequences_reconciled, skeleton: true, + observations: RunObservations::new(run.facts, mount), }; - let outcome = storage_primitive_outcome(&inputs); - let bundle = build_bundle(repo_root, workload, &inputs)?; - std::fs::write(&out, bundle).map_err(|e| format!("writing {}: {e}", out.display()))?; + let assembled = assemble_bundle(repo_root, workload, &inputs)?; + let outcome = assembled.outcome; + std::fs::write(&out, assembled.json).map_err(|e| format!("writing {}: {e}", out.display()))?; println!("bundle_schema=1"); println!("bundle_path={}", out.display()); @@ -2630,11 +4475,51 @@ fn emit_skeleton( println!("bundle_promotable=false"); println!("bundle_skeleton=true"); println!("bundle_path_driven={}", run.path); - println!("objects_new={}", run.objects_new); - println!("objects_new_counted={}", run.objects_new_counted); + println!("objects_new={}", inputs.objects_new); + println!( + "objects_new_counted={}", + inputs.observations.objects_new_counted + ); + // The ten declarations, on stdout as well as in the bundle, so a reviewer + // can check each value against what the run did without reading the JSON. + for (member, value) in assembled.conditions.pairs() { + println!("run_condition:{member}={value}"); + } + if let UniquenessCheck::GlobalAcrossAckRecords { + blob_ids, + tree_ids, + commit_ids, + } = inputs.observations.uniqueness + { + println!("unique_ids:blob={blob_ids} tree={tree_ids} commit={commit_ids}"); + } + println!( + "configured_max_index_runs={}", + inputs.observations.configured_max_index_runs + ); + println!( + "store_root_filesystem={} tmpfs={}", + inputs.observations.store_root_filesystem, inputs.observations.store_root_is_tmpfs + ); println!("evidence_signings={}", run.signings); println!("evidence_signing_micros_p50={:.1}", run.signing_micros_p50); - println!("lock_release_attempts={}", run.lock_release_attempts); + // The exclusion, as three numbers a reader can subtract rather than as a + // `true` in the bundle they have to take on trust. + println!("setup_fences_excluded={}", run.setup.fences); + println!("setup_signings_excluded={}", run.setup.signings); + println!("total_fences_including_setup={}", run.total_fences); + println!("total_signings_including_setup={}", run.total_signings); + // What the hardware-profile comparison could not compare. This is the + // deployed-node harness's work order, and it is why `hardware.profile` is + // what it is. + println!( + "hardware_profile_unobserved={}", + assembled.hardware.unobserved.join(",") + ); + println!( + "hardware_profile_mismatched={}", + assembled.hardware.mismatched.join(",") + ); println!("bundle_outcome={}", outcome.name()); println!("groups={}", run.groups); println!("transactions={}", run.transactions); @@ -2737,6 +4622,287 @@ mod tests { ); } + #[test] + fn the_frozen_profiles_are_read_as_whole_tables_rather_than_as_loose_keys() { + // Naming `hardware.profile` means deciding *which* profile a host is, + // and that cannot be done by a flat "find every occurrence of this key" + // scan: such a scan cannot say which profile a value belongs to. The + // reader parses the `[[profile]]` array-of-tables, and this asserts it + // against the frozen file rather than against a restatement. + let frozen = load_frozen_profile(&repo_root().join("bench/reference-hardware.toml")) + .expect("profile"); + assert_eq!(frozen.profiles.len(), 2); + let names: Vec<&str> = frozen + .profiles + .iter() + .map(|profile| profile.name.as_str()) + .collect(); + assert_eq!(names, vec!["minimum-30k", "release-60k"]); + + let minimum = &frozen.profiles[0]; + assert_eq!(minimum.fact("cpu", "model"), Some("AMD Ryzen 7 9800X3D")); + assert_eq!(minimum.fact("cpu", "physical_cores"), Some("8")); + assert_eq!(minimum.fact("filesystem", "type"), Some("btrfs")); + assert_eq!(minimum.fact("network", "link_mbps"), Some("1000")); + assert_eq!( + frozen.profiles[1].fact("network", "link_mbps"), + Some("10000"), + "the two profiles differ only in their network table, which is why an \ + unobserved NIC leaves them indistinguishable" + ); + assert_eq!( + minimum.fact("cpu", "nonexistent_key"), + None, + "an unpinned fact is not a constraint and must not be compared as one" + ); + } + + #[test] + fn the_hardware_profile_is_compared_against_the_frozen_tables_rather_than_accepted() { + // The split design: the deployed-node harness gathers privileged host + // facts; `store-bench` compares them against the complete frozen + // profiles and derives the name. A harness that supplied a profile + // *label* would be supplying an unverified claim wearing a verified + // field's name, so a label is never an input here — only facts are. + let frozen = load_frozen_profile(&repo_root().join("bench/reference-hardware.toml")) + .expect("profile"); + + // 1. Today: several facts have no reading at all, so no profile is a + // candidate and the honest answer is `diagnostic`. The facts that + // prevented a match are reported rather than swallowed. + let today = observe_host_facts("btrfs"); + let derived = hardware_profile_of(&frozen.profiles, &today, &[]).expect("derivation"); + assert_eq!(derived.name, "diagnostic"); + for owed in [ + "nvme.model", + "nvme.firmware", + "network.nic", + "network.driver", + ] { + assert!( + derived.unobserved.iter().any(|name| name == owed), + "{owed} is the deployed-node harness's half and must be reported as owed: \ + {:?}", + derived.unobserved + ); + } + + // The facts a deployed-node harness would have supplied, taken from the + // frozen table itself so the test compares the derivation and not a + // second copy of the reference host. + let facts_of = |profile: &ReferenceProfile| -> Vec { + profile + .facts + .iter() + .filter(|(table, _, _)| !table.is_empty()) + .map(|(table, key, value)| HostFact { + table: Box::leak(table.clone().into_boxed_str()), + key: Box::leak(key.clone().into_boxed_str()), + observed: Some(value.clone()), + }) + .collect() + }; + + // 2. Every fact the profile pins, observed and equal: the name follows + // from the comparison. This is what the split design buys — the + // emitter derives the name, and never accepts one. + let complete = facts_of(&frozen.profiles[0]); + assert_eq!( + hardware_profile_of(&frozen.profiles, &complete, &[]) + .expect("derivation") + .name, + "minimum-30k" + ); + assert_eq!( + hardware_profile_of(&frozen.profiles, &facts_of(&frozen.profiles[1]), &[]) + .expect("derivation") + .name, + "release-60k", + "the two frozen profiles are separated by their network facts, and the \ + derivation must separate them by comparing those facts" + ); + + // ...and the same set with one fact unreadable is not a name. One + // unobservable fact eliminates the profile that pins it, because + // skipping it would name a host on a partial match. + let mut missing = complete.clone(); + missing[0].observed = None; + let missing_one = hardware_profile_of(&frozen.profiles, &missing, &[]).expect("derivation"); + assert_eq!(missing_one.name, "diagnostic"); + assert!( + missing_one.unobserved.contains(&missing[0].name()), + "the fact that cost the name must be reported: {:?}", + missing_one.unobserved + ); + + // 3. A disagreeing fact is a mismatch, not a near miss: the host is not + // the reference host and the bundle says `diagnostic`. + let mut wrong = complete.clone(); + wrong[0].observed = Some("Some Other CPU".to_string()); + let mismatched = hardware_profile_of(&frozen.profiles, &wrong, &[]).expect("derivation"); + assert_eq!(mismatched.name, "diagnostic"); + assert!( + mismatched.mismatched.contains(&wrong[0].name()), + "a fact read and disagreeing must be reported: {:?}", + mismatched.mismatched + ); + + // 4. Facts that match more than one frozen profile are a refusal, never + // a pick. + // + // Under the inverted comparison the *current* frozen file cannot + // reach this state — its two profiles pin the same keys with + // different network values, so observing everything names exactly + // one and observing less than everything names none. The refusal is + // still reachable, and still required, for two profiles that pin + // different key sets: the second below pins a strict subset of the + // first's facts, so a host that observes both is matched by both. + // Both names are in `NAMED_HARDWARE_PROFILES` so the ambiguity is + // what refuses, not the enumeration. + let broad = ReferenceProfile { + name: "minimum-30k".to_string(), + purpose: "pins two facts".to_string(), + facts: vec![ + ( + "cpu".to_string(), + "architecture".to_string(), + "x86_64".to_string(), + ), + ("cpu".to_string(), "numa_nodes".to_string(), "1".to_string()), + ], + }; + let narrow = ReferenceProfile { + name: "release-60k".to_string(), + purpose: "pins one of them".to_string(), + facts: vec![( + "cpu".to_string(), + "architecture".to_string(), + "x86_64".to_string(), + )], + }; + let both = vec![ + HostFact { + table: "cpu", + key: "architecture", + observed: Some("x86_64".to_string()), + }, + HostFact { + table: "cpu", + key: "numa_nodes", + observed: Some("1".to_string()), + }, + ]; + let error = hardware_profile_of(&[broad, narrow], &both, &[]) + .expect_err("two candidates is not a name"); + assert!(error.contains("match 2 frozen profiles"), "{error}"); + + // ...and the gap the inversion closes: a fact the frozen profile pins + // and this emitter does not model must be *unobserved*, not invisible. + // Iterating the supplied facts made a host that matched every modelled + // fact a candidate while `cpu.turbo`, `nvme.power_loss_protection`, + // `filesystem.barriers` and `network.mtu` went uncompared. + let unmodelled = hardware_profile_of(&frozen.profiles, &today, &[]).expect("derivation"); + for pinned_but_unmodelled in [ + "cpu.turbo", + "memory.cgroup_limit_gib", + "nvme.power_loss_protection", + "nvme.write_cache", + "filesystem.barriers", + "filesystem.mount_options", + "network.mtu", + "software.proxy", + ] { + assert!( + unmodelled + .unobserved + .iter() + .any(|name| name == pinned_but_unmodelled), + "{pinned_but_unmodelled} is pinned by the frozen profile and read by \ + nothing, so it must eliminate the profile rather than pass silently: \ + {:?}", + unmodelled.unobserved + ); + } + // Every pinned fact of every profile is accounted for: observed and + // compared, or reported unobserved. Nothing pinned goes unmentioned, + // which is the property the old direction could not have. + for profile in &frozen.profiles { + for (table, key, _) in &profile.facts { + if table.is_empty() { + continue; + } + let name = format!("{table}.{key}"); + let observed = observe_host_facts("btrfs").into_iter().any(|fact| { + fact.table == table.as_str() + && fact.key == key.as_str() + && fact.observed.is_some() + }); + assert!( + observed || unmodelled.unobserved.contains(&name), + "{name} is pinned by {:?} and neither observed nor reported unobserved", + profile.name + ); + } + } + + // A profile that pins nothing would be matched by every host, so it is + // refused rather than treated as a candidate that happened to pass. + let empty = ReferenceProfile { + name: "minimum-30k".to_string(), + purpose: "pins nothing".to_string(), + facts: vec![( + "".to_string(), + "name".to_string(), + "minimum-30k".to_string(), + )], + }; + let error = hardware_profile_of(&[empty], &complete, &[]) + .expect_err("a profile that pins nothing is not a profile"); + assert!(error.contains("pins no"), "{error}"); + + // 5. A named profile alongside a hardware field still reading + // "recorded by the deployed-node harness" is two statements about + // one host that disagree, and that is a refusal rather than a value + // to pick between. + let error = hardware_profile_of( + &frozen.profiles, + &complete, + &[( + "nvme".to_string(), + jstr("recorded by the deployed-node harness"), + )], + ) + .expect_err("a named profile may not sit beside a placeholder field"); + assert!(error.contains("still placeholders"), "{error}"); + } + + #[test] + fn a_frozen_profile_name_the_schema_does_not_enumerate_is_refused() { + // Charter item 6 across the file boundary: `bench/reference-hardware.toml` + // names profiles and `bench/result-schema.json` enumerates what a bundle + // may carry. A third frozen profile is a lead-owned schema change before + // it is a bundle value, and emitting its name on the file's say-so would + // be exactly the "accept a label" failure the split design forbids. + let invented = ReferenceProfile { + name: "maximum-90k".to_string(), + purpose: "not in the schema".to_string(), + facts: vec![( + "cpu".to_string(), + "architecture".to_string(), + "x86_64".to_string(), + )], + }; + let facts = vec![HostFact { + table: "cpu", + key: "architecture", + observed: Some("x86_64".to_string()), + }]; + let error = hardware_profile_of(&[invented], &facts, &[]) + .expect_err("a name the schema does not admit is not a name"); + assert!(error.contains("maximum-90k"), "{error}"); + assert!(error.contains("schema change"), "{error}"); + } + #[test] fn the_bundle_uses_the_frozen_seed_and_generator_rather_than_a_copy() { let workload = load_frozen_workload(&repo_root().join("bench/workloads/small-commit.toml")) @@ -2760,6 +4926,7 @@ mod tests { let profile = FrozenProfile { store_directory_attributes: "nodatacow".to_string(), store_directories: vec!["shards/*/active".to_string()], + profiles: Vec::new(), }; let error = check_store_directory_attributes(directory.path(), &profile) .expect_err("an empty expansion must refuse"); @@ -2774,6 +4941,7 @@ mod tests { let profile = FrozenProfile { store_directory_attributes: "nodatacow".to_string(), store_directories: vec!["shards/*/active".to_string()], + profiles: Vec::new(), }; match check_store_directory_attributes(directory.path(), &profile) { Ok(observed) => { @@ -2862,13 +5030,617 @@ sys.exit(1 if errors else 0) fn the_emitted_bundle_validates_against_the_frozen_schema() { let workload = load_frozen_workload(&repo_root().join("bench/workloads/small-commit.toml")) .expect("workload"); - let bundle = build_bundle(&repo_root(), &workload, &skeleton_inputs()).expect("bundle"); - let errors = schema_errors(&bundle); - assert!( - errors.is_empty(), - "the emitted bundle does not satisfy bench/result-schema.json:\n{}", - errors.join("\n") + // Both paths, because the schema's branch conditionals are keyed on + // `run_conditions.mutation_path` and a suite that validated one of them + // would leave the other's rules unexercised — which is where a + // journal-seam bundle asserting what the seam cannot observe would + // appear. + for (path, inputs) in [("drive", skeleton_inputs()), ("submit", submit_inputs())] { + let bundle = build_bundle(&repo_root(), &workload, &inputs).expect("bundle"); + let errors = schema_errors(&bundle); + assert!( + errors.is_empty(), + "the emitted {path} bundle does not satisfy bench/result-schema.json:\n{}", + errors.join("\n") + ); + } + } + + #[test] + fn the_ten_run_conditions_are_derived_from_what_the_run_observed() { + // Each value against the observation it comes from. The point of the + // block is that it is a derivation: change an observation and the + // declaration follows, which a hardcoded block would not do. + let drive = RunConditions::derive(&drive_observations(), true, "diagnostic") + .expect("the drive path declares"); + assert_eq!(drive.initialization_path, "shard_drive_create"); + assert_eq!(drive.mutation_path, "journal_drive"); + assert_eq!(drive.checkpointing, "unimplemented"); + assert_eq!(drive.index_maintenance, "no_index_in_path"); + assert_eq!(drive.index_run_ceiling, "store_default"); + assert_eq!(drive.receipt_reconciliation, "no_receipts_in_path"); + assert_eq!(drive.objects_new_source, "derived_from_transaction_count"); + assert_eq!(drive.commit_id_uniqueness, "not_checked"); + assert_eq!(drive.environment_fidelity, "diagnostic"); + + let submit = RunConditions::derive(&submit_observations(), true, "diagnostic") + .expect("the submit path declares"); + assert_eq!(submit.initialization_path, "store_engine_open"); + assert_eq!(submit.mutation_path, "store_engine_submit"); + assert_eq!( + submit.index_run_ceiling, "raised_because_index_sealing_unimplemented", + "the submit run raises max_index_runs above the store default, and the \ + declaration must follow from that comparison rather than from the path" ); + assert_eq!(submit.index_maintenance, "deltas_retained_in_memory"); + assert_eq!( + submit.receipt_reconciliation, "acceptance_of_any_committed_status", + "the reconciliation accepts any Committed(_) without comparing the payload, \ + and the receipt_digest journalled is blake3(operation_id)" + ); + assert_eq!(submit.objects_new_source, "summed_from_receipts"); + assert_eq!( + submit.commit_id_uniqueness, + "checked_globally_across_ack_records" + ); + + // The one member that follows from how this binary was built, and the + // one that follows from four conditions at once. + let expected_profile = if cfg!(debug_assertions) { + "debug" + } else { + "release" + }; + assert_eq!(drive.build_profile, expected_profile); + assert_eq!( + RunConditions::derive(&drive_observations(), true, "minimum-30k") + .expect("derive") + .environment_fidelity, + if cfg!(debug_assertions) { + "diagnostic" + } else { + "reference_profile" + }, + "reference-profile fidelity requires a release build, a verified attribute \ + precheck, a persistent mount, and a named hardware profile — all four" + ); + let mut on_tmpfs = drive_observations(); + on_tmpfs.store_root_is_tmpfs = true; + assert_eq!( + RunConditions::derive(&on_tmpfs, true, "minimum-30k") + .expect("derive") + .environment_fidelity, + "diagnostic" + ); + assert_eq!( + RunConditions::derive(&drive_observations(), false, "minimum-30k") + .expect("derive") + .environment_fidelity, + "diagnostic", + "a run that skipped the store-directory attribute precheck has not met the \ + reference environment, whatever else it met" + ); + } + + #[test] + fn an_observation_with_no_named_value_is_refused_rather_than_guessed() { + // Charter item 6 at the emitter: the schema's enumerations are closed + // sets, so the mapping onto them is closed too. A lowered ceiling has + // no named value, and inventing one would be the caveat this block + // replaced. + let mut lowered = submit_observations(); + lowered.configured_max_index_runs = lowered.default_max_index_runs - 1; + let error = RunConditions::derive(&lowered, true, "diagnostic") + .expect_err("a lowered ceiling has no named value"); + assert!(error.contains("index_run_ceiling"), "{error}"); + + // A run that published no group and sealed no index run has no index + // maintenance to declare. + let mut idle = submit_observations(); + idle.index = IndexObservation::StoreRoot { + validated_runs: 0, + unvalidatable: Vec::new(), + groups: 0, + unsealed_delta_backlog: None, + }; + let error = RunConditions::derive(&idle, true, "diagnostic") + .expect_err("an idle run declares nothing"); + assert!(error.contains("index_maintenance"), "{error}"); + } + + #[test] + fn index_steady_state_is_not_satisfiable_by_a_stray_file() { + // The defect: `index_maintenance` was derived from a count of *directory + // entries* under `shards/*/indexes`, so one temporary file, one + // truncated run, or one unrelated file was enough to declare that the + // index had reached a steady state. A declaration about maintenance must + // not be earnable by a file existing. + + // 1. An entry that did not read back as an index run is a refusal by + // name, not a lower count and not a silent skip. + let mut stray = submit_observations(); + stray.index = IndexObservation::StoreRoot { + validated_runs: 0, + unvalidatable: vec!["shards/00/indexes/.tmp-4711: not an IndexRun".to_string()], + groups: 12, + unsealed_delta_backlog: None, + }; + let error = RunConditions::derive(&stray, true, "diagnostic") + .expect_err("an entry the emitter cannot validate is not a state it may declare"); + assert!( + error.contains("did not read back as an IndexRun"), + "{error}" + ); + assert!( + error.contains(".tmp-4711"), + "the refusal must name it: {error}" + ); + + // 2. Validated runs alone are still not a steady state. Files are not + // maintenance: without a reading of the outstanding delta backlog, + // `runs_sealed` would be inferred from a directory listing, which is + // the same defect the count had. + let mut files_only = submit_observations(); + files_only.index = IndexObservation::StoreRoot { + validated_runs: 8, + unvalidatable: Vec::new(), + groups: 12, + unsealed_delta_backlog: None, + }; + let error = RunConditions::derive(&files_only, true, "diagnostic") + .expect_err("presence of files is not evidence of bounded maintenance"); + assert!(error.contains("remain unsealed"), "{error}"); + + // 3. Sealing that ran and did not keep up is neither named value, and + // charter item 6 makes that a refusal rather than the nearer of the + // two. + let mut behind = submit_observations(); + behind.index = IndexObservation::StoreRoot { + validated_runs: 8, + unvalidatable: Vec::new(), + groups: 12, + unsealed_delta_backlog: Some(3), + }; + let error = RunConditions::derive(&behind, true, "diagnostic") + .expect_err("a backlog is not a steady state and not a retained-every-delta run"); + assert!(error.contains("did not keep up"), "{error}"); + + // 4. And the one shape that earns it: validated runs with a backlog + // observed to be zero. Unreachable today — nothing seals and nothing + // reports the backlog — and written so the day it lands the + // declaration is already conditioned on the reading. + let mut steady = submit_observations(); + steady.index = IndexObservation::StoreRoot { + validated_runs: 8, + unvalidatable: Vec::new(), + groups: 12, + unsealed_delta_backlog: Some(0), + }; + assert_eq!( + RunConditions::derive(&steady, true, "diagnostic") + .expect("a bounded backlog over validated runs is a steady state") + .index_maintenance, + "runs_sealed" + ); + + // 5. Today's honest value is unchanged: groups published, nothing + // sealed, nothing unvalidatable. + assert_eq!( + RunConditions::derive(&submit_observations(), true, "diagnostic") + .expect("today's run declares") + .index_maintenance, + "deltas_retained_in_memory" + ); + } + + #[test] + fn a_directory_entry_that_is_not_an_index_run_is_reported_as_unvalidatable() { + // The scan itself, against a real root, because the derivation above + // can only refuse what the scan hands it. A file dropped into a shard's + // `indexes` directory used to be counted as a sealed run. + let directory = tempfile::tempdir().expect("tempdir"); + let root = directory.path().join("root"); + let layout = levcs_store::segment::RootLayout::new(&root); + levcs_store::segment::initialize_root( + &layout, + 1, + [0x4b; 16], + engine_now_micros(), + &levcs_store::types::DurabilityCounters::default(), + ) + .expect("initialize_root"); + + let clean = scan_index_runs(&root, 1); + assert_eq!(clean.validated_runs, 0); + assert!(clean.unvalidatable.is_empty(), "{:?}", clean.unvalidatable); + + std::fs::write(layout.shard(0).indexes().join(".tmp-writer"), b"not a run") + .expect("write stray file"); + let strayed = scan_index_runs(&root, 1); + assert_eq!( + strayed.validated_runs, 0, + "a file that is not an index run must not count as one" + ); + assert_eq!( + strayed.unvalidatable.len(), + 1, + "and it must be reported rather than dropped: {:?}", + strayed.unvalidatable + ); + assert!(strayed.unvalidatable[0].contains(".tmp-writer")); + } + + #[test] + fn a_disagreeing_objects_new_is_refused_rather_than_emitted_as_false() { + // Scope 6.6 item 5c. The two sides are counted separately — receipts + // summed on one side, commits counted on the other — so this comparison + // can fail, and when it does the bundle is what is wrong. `false` is + // not available: the schema pins the claim to `const true`, and + // recording a failed invariant as a negative result is how a harness + // publishes a defect as a measurement. + let workload = load_frozen_workload(&repo_root().join("bench/workloads/small-commit.toml")) + .expect("workload"); + let mut inputs = submit_inputs(); + inputs.objects_new = inputs.counted_commits * OBJECTS_PER_COMMIT + 1; + let error = build_bundle(&repo_root(), &workload, &inputs) + .expect_err("a disagreement must refuse rather than emit"); + assert!(error.contains("new objects across"), "{error}"); + + // And the drive path, which derives `objects_new` from the transaction + // count, is not subject to the comparison at all — it may not assert + // the claim, which is what makes the tautology unreachable rather than + // merely discouraged. + let mut derived = skeleton_inputs(); + derived.objects_new = derived.counted_commits * OBJECTS_PER_COMMIT + 1; + let bundle = build_bundle(&repo_root(), &workload, &derived) + .expect("the drive path asserts nothing about objects_new"); + assert!(!bundle.contains("objects_new_equals_three_per_commit")); + } + + #[test] + fn repository_creation_is_outside_the_fences_and_signatures_the_bundle_reports() { + // The P1 defect: repositories are created *before* the measured + // interval, through the same `StoreEngine::submit` path the measurement + // uses, so creating them fences and signs. Both counters were read only + // at the end of the run — that is, from a baseline of zero, taken + // before the repositories existed — so every bundle reported the + // creation fences and the creation signatures inside the measured + // interval while asserting `verification.setup_traffic_excluded: true`. + // + // Charter item 7: this asserts against the store's own + // `DurabilityCounters` and the signer's own sample vector, not against + // the emitter's intention to have excluded anything. + let directory = tempfile::tempdir().expect("tempdir"); + let root = directory.path().join("root"); + let ack = directory.path().join("ack-journal"); + let run = run_engine(&root, &ack, AckJournalFault::None, 4, 1, 2, 1) + .expect("an engine-driven run"); + + // The exclusion is not vacuous: creating the repositories really did + // fence and really did sign, so there is something to exclude. Without + // this the three assertions below would pass on a run that had no setup + // traffic at all. + assert!( + run.setup.fences > 0, + "creating {} repositories through StoreEngine::submit must fence, or this \ + test cannot tell an exclusion from an absence", + 2 + ); + assert!( + run.setup.signings > 0, + "creating repositories through StoreEngine::submit signs every event" + ); + + // The strict inequality is the assertion that fails without the fix. + // Before it, `fences` *was* the total, so this read `total > total`. + assert!( + run.total_fences > run.fences, + "the fence counter read {} in total and {} for the measured interval; setup \ + traffic is being counted as measured work", + run.total_fences, + run.fences + ); + assert!( + run.total_signings > run.signings, + "the signer recorded {} samples in total and {} inside the measured interval", + run.total_signings, + run.signings + ); + + // And the reported figures are exactly the totals less the baselines, + // so the subtraction is the whole of the exclusion. + assert_eq!( + run.fences, + run.total_fences - run.setup.fences, + "storage.fences must be what the counter gained across the measured interval" + ); + assert_eq!( + run.signings, + run.total_signings - run.setup.signings, + "evidence_signings must count only signatures taken inside the interval" + ); + + // A run that measured something, so the counts above are about a real + // interval rather than about an empty one. + assert!( + run.transactions > 0, + "the measured interval must contain work" + ); + + // Through the shape the emitter actually consumes, at the altitude a + // consumer calls (charter item 8): `MeasuredRun` is what becomes + // `BundleInputs.fences` and `BundleInputs.evidence_signings`. + let (fences, signings) = (run.fences, run.signings); + let measured: MeasuredRun = run.into(); + assert_eq!(measured.fences, fences); + assert_eq!(measured.signings, signings); + assert_eq!( + measured.fences, + measured.total_fences - measured.setup.fences + ); + } + + #[test] + fn a_zero_submitter_run_is_refused_rather_than_measured() { + // The lead's reproduction, first half. `--submitters-per-shard 0` + // spawns no submitter, so the measured interval contains no + // transaction — but the repositories are still created, so the run + // reported their fences and their signatures as measured work. + let directory = tempfile::tempdir().expect("tempdir"); + let error = run_engine( + &directory.path().join("root"), + &directory.path().join("ack-journal"), + AckJournalFault::None, + 4, + 1, + 2, + 0, + ) + .err() + .expect("a run with no submitter measures nothing"); + assert!(error.contains("submitters-per-shard 0"), "{error}"); + + // The same zero-work run through the other flag. + let error = run_engine( + &directory.path().join("root-b"), + &directory.path().join("ack-journal-b"), + AckJournalFault::None, + 4, + 1, + 0, + 1, + ) + .err() + .expect("no shard is no repository and no transaction"); + assert!(error.contains("--shards 0"), "{error}"); + } + + #[test] + fn an_acknowledgment_that_could_not_be_journaled_refuses_instead_of_shortening_the_run() { + // The P1 defect. `append_durable` failing set `stop` and returned: it + // did not increment `refused`, so the fatal-error guard in + // `emit_skeleton` never saw it, and it happened *after* a successful + // commit, so the zero-work guard did not see it either. The bundle then + // omitted a committed transaction while still counting its durability + // fence and its Ed25519 signature, and reported a measured interval + // that had been cut short — numbers describing a workload that did not + // happen, under an accounting identity the bundle asserts is exact. + // + // `EDQUOT` on a full tmpfs is the ordinary way to reach it, which is + // why the fault is armed rather than raced: the failure has to be the + // same failure every time this test runs. + // One invocation, parameterized only by the fault, so the faulted run + // and its control differ in exactly one flag. + fn invocation(root: &Path, out: &Path, fault: Option) -> Flags { + let mut raw = vec![ + "emit-skeleton".to_string(), + "--repo-root".to_string(), + repo_root().display().to_string(), + "--root".to_string(), + root.display().to_string(), + "--out".to_string(), + out.display().to_string(), + "--path".to_string(), + "submit".to_string(), + "--seconds".to_string(), + "2".to_string(), + "--group-len".to_string(), + "4".to_string(), + "--shards".to_string(), + "1".to_string(), + "--submitters-per-shard".to_string(), + "1".to_string(), + "--allow-unsigned".to_string(), + // The tempdir is not a nodatacow btrfs subtree, and this test + // is about the accounting refusal rather than about the + // attribute precheck that would otherwise refuse first. + "--skip-attribute-check".to_string(), + ]; + if let Some(index) = fault { + raw.push("--fail-ack-append-after".to_string()); + raw.push(index.to_string()); + } + Flags::parse(raw.into_iter()).expect("flags").1 + } + + let directory = tempfile::tempdir().expect("tempdir"); + let out = directory.path().join("bundle.json"); + let root = directory.path().join("root"); + + // Fail the *second* append, so a transaction was committed, + // acknowledged, fenced and signed before the failure. That is exactly + // the state neither surviving guard could see. + // + // Charter item 8: through `dispatch`, the entry point the CLI and + // scripts/verify-store-recovery.sh both call, not through `run_engine` + // — the defect was that the refusal never reached the emitter, so a + // test below the emitter could not have caught it. + let error = dispatch("emit-skeleton", &invocation(&root, &out, Some(1))) + .expect_err("a commit whose acknowledgment could not be journaled voids the run"); + assert!( + error.contains("incomplete accounting"), + "the refusal must name what is wrong — the run cannot account for a \ + transaction it measured — rather than reporting a refused submit: {error}" + ); + // The original I/O error, preserved rather than summarized. `code: 28` + // is ENOSPC as the kernel reported it through /dev/full; `EDQUOT` would + // arrive here as 122 by the same route. + assert!( + error.contains("code: 28"), + "the underlying I/O error must survive into the refusal: {error}" + ); + assert!( + error.contains("repo_sequence"), + "the refusal must say which committed transaction went unaccounted: {error}" + ); + // And no bundle. A refusal that still wrote one is not a refusal — this + // is the assertion the defect failed: before the fix the run reached + // `assemble_bundle` and wrote a short, self-consistent-looking file. + assert!( + !out.exists(), + "a run that cannot account for a committed transaction must emit nothing, \ + and {} exists", + out.display() + ); + + // The negative control. Without the armed fault the identical + // invocation emits a bundle, so the refusal above is caused by the + // journal failure and not by the flags, the tempdir, or the two-second + // budget. A fresh root, because the faulted run left one behind and + // `run_engine` refuses a root it did not build. + let control = tempfile::tempdir().expect("tempdir"); + let control_out = control.path().join("bundle.json"); + dispatch( + "emit-skeleton", + &invocation(&control.path().join("root"), &control_out, None), + ) + .expect("an unfaulted run emits a bundle"); + assert!( + control_out.exists(), + "the negative control must produce the bundle the faulted run must not" + ); + } + + #[test] + fn the_journal_seam_refuses_an_unjournalable_acknowledgment_the_same_way() { + // The same failure on the Wave A drive path. It was already fatal there + // — the append is behind a `?` — but it reported itself as "ack append: + // ..." rather than as the accounting failure it is, and the two paths + // must answer the same question the same way or the seam becomes the + // place the weaker answer survives. + let directory = tempfile::tempdir().expect("tempdir"); + let error = run_skeleton( + &directory.path().join("root"), + &directory.path().join("ack-journal"), + AckJournalFault::EnospcOnAppend(1), + 4, + 2, + ) + .err() + .expect("a fenced group whose acknowledgment cannot be journaled voids the run"); + assert!(error.contains("incomplete accounting"), "{error}"); + assert!(error.contains("code: 28"), "{error}"); + } + + #[test] + fn a_zero_work_run_cannot_earn_a_claim_that_is_vacuous_over_an_empty_set() { + // The lead's reproduction, second half, and the reason the refusal + // cannot live in the schema: a zero-commit bundle is **schema-valid**. + // `counts` is `nonnegative`, and the three claims are `const true`, so + // a run that measured nothing satisfies every rule in + // `bench/result-schema.json` while asserting `setup_traffic_excluded`, + // `unique_blob_tree_commit_ids`, and `objects_new_equals_three_per_commit` + // over zero commits — each vacuously true, each presented in the same + // field a real run uses. + let workload = load_frozen_workload(&repo_root().join("bench/workloads/small-commit.toml")) + .expect("workload"); + + // First: the schema really does accept it. Recorded deliberately, as + // the negative control for the refusal below — an emitter-side check + // whose job the schema already did would be decoration. + let real = build_bundle(&repo_root(), &workload, &submit_inputs()).expect("bundle"); + let vacuous = real + .replace("\"counted_commits\": 3", "\"counted_commits\": 0") + .replace("\"objects_new\": 9", "\"objects_new\": 0") + .replace( + "\"acknowledged_requests\": 3", + "\"acknowledged_requests\": 0", + ) + .replace("\"offered_requests\": 3", "\"offered_requests\": 0") + .replace("\"accepted_requests\": 3", "\"accepted_requests\": 0"); + assert_ne!(vacuous, real, "the mutation must have applied"); + assert!( + schema_errors(&vacuous).is_empty(), + "the schema accepts a zero-commit bundle asserting all three claims, which is \ + exactly why the emitter must refuse to produce one:\n{}", + schema_errors(&vacuous).join("\n") + ); + + // Second: the emitter refuses, by name, on both paths. A `false` is not + // available — the schema pins the claims to `const true` — and emitting + // one would be a different untrue statement. + for mut inputs in [submit_inputs(), skeleton_inputs()] { + inputs.counted_commits = 0; + inputs.objects_new = 0; + let error = build_bundle(&repo_root(), &workload, &inputs) + .expect_err("a claim over an empty set is not a claim this run earned"); + assert!(error.contains("vacuously true"), "{error}"); + } + + // A measured, acknowledged commit is what the claims are conditioned + // on, so acknowledging nothing is refused even where commits were + // counted. + let mut unacknowledged = submit_inputs(); + unacknowledged.acknowledged_requests = 0; + let error = build_bundle(&repo_root(), &workload, &unacknowledged) + .expect_err("nothing acknowledged is nothing measured"); + assert!(error.contains("acknowledged request"), "{error}"); + } + + #[test] + fn uniqueness_is_checked_across_every_record_rather_than_within_each() { + // Two records that are each internally distinct and still share a + // commit id. A per-record check passes this input; the claim's whole + // condition is that this one does not. + let mut first = skeleton_ack_record(1); + let mut second = skeleton_ack_record(2); + let shared = ObjectId([0x5c; 32]); + first.commit_ids = vec![shared]; + second.commit_ids = vec![shared]; + let error = check_global_uniqueness(&[first, second]) + .expect_err("a collision between two records must be refused"); + assert!(error.contains("commit identifier"), "{error}"); + + let clean = check_global_uniqueness(&[skeleton_ack_record(1), skeleton_ack_record(2)]) + .expect("distinct records"); + assert_eq!( + clean, + UniquenessCheck::GlobalAcrossAckRecords { + blob_ids: 2, + tree_ids: 2, + commit_ids: 2, + }, + "the counts are the set sizes, so a reader can see the check had something \ + to check" + ); + } + + #[test] + fn the_receipt_claim_is_never_emitted_while_no_receipt_is_reconciled() { + // Scope 6.6 item 5e, asserted on the artifact rather than left to the + // schema: the claim is approved in principle and not earned, and the + // reason it is not earned lives in this file. + let workload = load_frozen_workload(&repo_root().join("bench/workloads/small-commit.toml")) + .expect("workload"); + for inputs in [skeleton_inputs(), submit_inputs()] { + let bundle = build_bundle(&repo_root(), &workload, &inputs).expect("bundle"); + assert!( + !bundle.contains("operation_receipts_reconciled"), + "store-bench accepts any Committed(_) status and journals \ + blake3(operation_id) as its receipt digest, so it may not assert \ + receipt reconciliation:\n{bundle}" + ); + } } #[test] @@ -2883,7 +5655,12 @@ sys.exit(1 if errors else 0) let mut inputs = skeleton_inputs(); inputs.skeleton = false; inputs.latency_p99 = STORAGE_PRIMITIVE_P99_CEILING_MICROS + 1; - assert_eq!(storage_primitive_outcome(&inputs), Outcome::Fail); + let conditions = + RunConditions::derive(&inputs.observations, true, "diagnostic").expect("conditions"); + assert_eq!( + storage_primitive_outcome(&inputs, &conditions), + Outcome::Fail + ); let bundle = build_bundle(&repo_root(), &workload, &inputs).expect("bundle"); assert!(bundle.contains("\"outcome\": \"fail\""), "{bundle}"); let errors = schema_errors(&bundle); @@ -2893,19 +5670,117 @@ sys.exit(1 if errors else 0) let mut inputs = skeleton_inputs(); inputs.skeleton = false; inputs.windows_meeting_target_percent = 80.0; - assert_eq!(storage_primitive_outcome(&inputs), Outcome::Fail); + assert_eq!( + storage_primitive_outcome(&inputs, &conditions), + Outcome::Fail + ); let errors = schema_errors(&build_bundle(&repo_root(), &workload, &inputs).expect("bundle")); assert!(errors.is_empty(), "{}", errors.join("\n")); - // And a run that met both is a pass under the same conditional. + // A run that missed no ceiling is still not a pass, and the reason is + // contract review 2026-07-28-C rather than a ceiling: `outcome: "pass"` + // requires `environment_fidelity: "reference_profile"` at every gate, + // and this emitter records placeholders for hardware it never measured. + // The run is therefore representable and mechanically disqualified — + // which is the state the review was after, and which the *previous* + // version of this arm asserted the opposite of. let mut inputs = skeleton_inputs(); inputs.skeleton = false; - assert_eq!(storage_primitive_outcome(&inputs), Outcome::Pass); + assert_eq!( + storage_primitive_outcome(&inputs, &conditions), + Outcome::Preliminary + ); let bundle = build_bundle(&repo_root(), &workload, &inputs).expect("bundle"); - assert!(bundle.contains("\"outcome\": \"pass\"")); + assert!(bundle.contains("\"outcome\": \"preliminary\""), "{bundle}"); + assert!( + bundle.contains("\"environment_fidelity\": \"diagnostic\""), + "{bundle}" + ); let errors = schema_errors(&bundle); assert!(errors.is_empty(), "{}", errors.join("\n")); + + // A pass remains reachable in principle, and the conditional that + // gates it is exercised rather than assumed. **Every** condition the + // schema requires of a passing storage_primitive bundle has to hold, + // not only the fidelity one: the root through `StoreEngine::open`, the + // transactions through `StoreEngine::submit`, a checkpoint exercised, + // and the index in a sealed steady state. + let reference = RunConditions { + environment_fidelity: "reference_profile", + build_profile: "release", + initialization_path: "store_engine_open", + mutation_path: "store_engine_submit", + checkpointing: "exercised", + index_maintenance: "runs_sealed", + ..conditions.clone() + }; + assert_eq!( + storage_primitive_outcome(&inputs, &reference), + Outcome::Pass, + "nothing about recording a diagnostic run may change what a pass costs" + ); + } + + #[test] + fn the_checkpoint_and_index_conditions_bound_a_pass_before_the_schema_has_to() { + // The trap this closes: `environment_fidelity` was the only condition + // between the emitter and a `pass`, and it is about to stop being the + // binding one. The moment hardware recognition names a frozen profile, + // an outcome derived from fidelity alone would emit `pass` on a run + // that took no checkpoint and sealed no index run — and the schema + // would then reject the bundle. A harness that learns what it claimed + // from its own validator has already published the claim. + let mut inputs = skeleton_inputs(); + inputs.skeleton = false; + + // Everything a pass costs, held true at once. + let passing = RunConditions { + initialization_path: "store_engine_open", + mutation_path: "store_engine_submit", + checkpointing: "exercised", + index_maintenance: "runs_sealed", + index_run_ceiling: "store_default", + receipt_reconciliation: "acceptance_of_any_committed_status", + objects_new_source: "summed_from_receipts", + commit_id_uniqueness: "checked_globally_across_ack_records", + build_profile: "release", + environment_fidelity: "reference_profile", + }; + assert_eq!(storage_primitive_outcome(&inputs, &passing), Outcome::Pass); + + // Each condition alone, moved to the value today's run genuinely has. + // Every one of them must cost the pass, and the reason each one is + // named separately is that a suite asserting only the conjunction + // cannot tell which member is load-bearing. + for (member, today) in [ + ("checkpointing", "unimplemented"), + ("index_maintenance", "deltas_retained_in_memory"), + ("initialization_path", "shard_drive_create"), + ("mutation_path", "journal_drive"), + ] { + let mut degraded = passing.clone(); + match member { + "checkpointing" => degraded.checkpointing = today, + "index_maintenance" => degraded.index_maintenance = today, + "initialization_path" => degraded.initialization_path = today, + "mutation_path" => degraded.mutation_path = today, + other => panic!("unnamed member {other}"), + } + assert_eq!( + storage_primitive_outcome(&inputs, °raded), + Outcome::Preliminary, + "run_conditions.{member} = {today:?} is a condition bench/result-schema.json \ + requires of a passing storage_primitive bundle, so the emitter must reach \ + the same answer the validator would" + ); + } + + // And a ceiling miss still outranks all of it: a run that exceeded the + // gate ceiling is a `fail`, never a `preliminary` that quietly did. + let mut missed = inputs; + missed.latency_p99 = STORAGE_PRIMITIVE_P99_CEILING_MICROS + 1; + assert_eq!(storage_primitive_outcome(&missed, &passing), Outcome::Fail); } #[test] @@ -2918,6 +5793,15 @@ sys.exit(1 if errors else 0) let workload = load_frozen_workload(&repo_root().join("bench/workloads/small-commit.toml")) .expect("workload"); let bundle = build_bundle(&repo_root(), &workload, &skeleton_inputs()).expect("bundle"); + let submit = build_bundle(&repo_root(), &workload, &submit_inputs()).expect("bundle"); + + /// A mutation must both apply and be rejected. A mutation that silently + /// did not apply is a negative control that proves nothing, which is + /// the failure mode this whole test exists to prevent one level up. + fn refuses(original: &str, mutated: String, why: &str) { + assert_ne!(mutated, original, "the mutation did not apply: {why}"); + assert!(!schema_errors(&mutated).is_empty(), "{why}"); + } // 1. The conditional: commits_in_recovered_closure must be false at // storage_primitive, because the store cannot traverse a graph. @@ -2990,6 +5874,141 @@ sys.exit(1 if errors else 0) why the assertion above is on the emitted bytes and not on a \ validator error" ); + + // -- one mutation per field contract review 2026-07-28-C made required. + // + // A required field the negative control never removes is a field the + // suite cannot notice the loss of, which is the same class of defect as + // a test asserting a count it merely observed. Each of the two new + // required fields is removed here, and each conditional the block + // introduced is contradicted here, on the path whose rules it keys on. + + // 6. The whole `run_conditions` block, gone. + refuses( + &bundle, + bundle.replace("\"run_conditions\":", "\"run_conditions_removed\":"), + "a bundle with no run_conditions block must be rejected: every claim in it \ + is conditioned on a declaration, and a bundle that declares nothing has \ + gone back to caveats living outside it", + ); + + // 7. `resources.configured_ceilings.max_index_runs`, gone. It was + // reachable only through free-form additionalProperties before the + // review, which is how an emitter could omit the one ceiling this + // workload actually reaches and stay valid. + refuses( + &bundle, + bundle.replace("\"max_index_runs\":", "\"max_index_runs_removed\":"), + "a bundle omitting the configured max_index_runs must be rejected", + ); + + // 8. The ceiling cross-check, in both directions. A declared store + // default may not record a raised value... + refuses( + &submit, + submit.replace( + "\"index_run_ceiling\": \"raised_because_index_sealing_unimplemented\"", + "\"index_run_ceiling\": \"store_default\"", + ), + "declaring the store default while recording a raised ceiling must be \ + rejected in that direction", + ); + // ...and a declared raise may not record the default. + refuses( + &bundle, + bundle.replace( + "\"index_run_ceiling\": \"store_default\"", + "\"index_run_ceiling\": \"raised_because_index_sealing_unimplemented\"", + ), + "declaring a raise while recording the default must be rejected in the \ + other direction", + ); + + // 9. The receipt claim, asserted while the declaration says any + // Committed status was accepted. + refuses( + &submit, + submit.replace( + "\"acknowledged_sequences_reconciled\": true", + "\"acknowledged_sequences_reconciled\": true,\n \ + \"operation_receipts_reconciled\": true", + ), + "accepting any Committed status is not reconciling a receipt, and a bundle \ + that declares the first while claiming the second must be rejected", + ); + + // 10. The two claims the submit path earned, each removed. They are + // required there — a missing result, not an inapplicable one. + refuses( + &submit, + submit.replace( + "\"unique_blob_tree_commit_ids\": true", + "\"unique_blob_tree_commit_ids_removed\": true", + ), + "the submit path must assert unique_blob_tree_commit_ids", + ); + refuses( + &submit, + submit.replace( + "\"objects_new_equals_three_per_commit\": true", + "\"objects_new_equals_three_per_commit_removed\": true", + ), + "the submit path must assert objects_new_equals_three_per_commit", + ); + + // 11. The same two claims asserted on the journal seam, which has + // neither the objects nor the receipts to earn them. This is the + // hole the branch conditional exists to close: permitting the + // claims on both paths would have been a worse defect than the one + // the review fixed, arriving disguised as the fix. + refuses( + &bundle, + bundle.replace( + "\"acknowledged_sequences_reconciled\": true", + "\"acknowledged_sequences_reconciled\": true,\n \ + \"unique_blob_tree_commit_ids\": true", + ), + "the journal seam declares commit_id_uniqueness: not_checked and may not \ + claim global uniqueness", + ); + refuses( + &bundle, + bundle.replace( + "\"acknowledged_sequences_reconciled\": true", + "\"acknowledged_sequences_reconciled\": true,\n \ + \"objects_new_equals_three_per_commit\": true", + ), + "a run that derived objects_new from its transaction count may not assert \ + the claim: the assertion could not fail", + ); + + // 12. A diagnostic run promoted to a pass, and a diagnostic run + // claiming the reference profile it did not meet. + refuses( + &bundle, + bundle.replace("\"outcome\": \"preliminary\"", "\"outcome\": \"pass\""), + "a run declaring diagnostic fidelity may not be a pass at any gate", + ); + refuses( + &bundle, + bundle.replace( + "\"environment_fidelity\": \"diagnostic\"", + "\"environment_fidelity\": \"reference_profile\"", + ), + "reference-profile fidelity re-pins a release build, and this bundle \ + declares the profile it was built with", + ); + + // 13. The seam declaring what only the engine can do. + refuses( + &bundle, + bundle.replace( + "\"mutation_path\": \"journal_drive\"", + "\"mutation_path\": \"store_engine_submit\"", + ), + "a bundle whose other declarations are the seam's may not call its \ + mutation path production submit", + ); } #[test] @@ -3069,8 +6088,79 @@ sys.exit(1 if errors else 0) assert_eq!(known, "2026-01-26T00:00:00Z"); } + /// The observations a `--path drive` run makes. + /// + /// Written as observations rather than as ten finished strings on purpose: + /// what these tests have to cover is the derivation, and a fixture that + /// handed the emitter the answers would test the JSON writer instead. + fn drive_observations() -> RunObservations { + RunObservations::new( + RunFacts { + initialization: Initialization::ShardDriveCreate, + mutation: MutationPath::JournalDrive, + checkpoint: CheckpointProbe::RefusedNotImplemented, + index: IndexObservation::NoIndexInPath, + // Read off `StoreOptions` rather than written as 64 here for + // the same reason the emitter does it: a test that restates the + // default cannot notice the default moving. + configured_max_index_runs: default_max_index_runs(), + default_max_index_runs: default_max_index_runs(), + receipts: ReceiptComparison::NoReceiptsInPath, + objects_new_counted: false, + uniqueness: UniquenessCheck::NotPerformed, + }, + MountFacts { + filesystem: "btrfs".into(), + tmpfs: false, + }, + ) + } + + /// The observations a `--path submit` run makes today. + fn submit_observations() -> RunObservations { + RunObservations::new( + RunFacts { + initialization: Initialization::StoreEngineOpen, + mutation: MutationPath::StoreEngineSubmit, + checkpoint: CheckpointProbe::RefusedNotImplemented, + index: IndexObservation::StoreRoot { + validated_runs: 0, + unvalidatable: Vec::new(), + groups: 1, + unsealed_delta_backlog: None, + }, + configured_max_index_runs: ENGINE_MAX_INDEX_RUNS, + default_max_index_runs: default_max_index_runs(), + receipts: ReceiptComparison::AnyCommittedStatusAccepted, + objects_new_counted: true, + uniqueness: UniquenessCheck::GlobalAcrossAckRecords { + blob_ids: 3, + tree_ids: 3, + commit_ids: 3, + }, + }, + MountFacts { + filesystem: "btrfs".into(), + tmpfs: false, + }, + ) + } + + fn default_max_index_runs() -> u32 { + levcs_store::StoreOptions::new("/nonexistent/store-bench-default-probe").max_index_runs + } + + fn submit_inputs() -> BundleInputs { + BundleInputs { + run_id: "engine-wave-b-test".into(), + observations: submit_observations(), + ..skeleton_inputs() + } + } + fn skeleton_inputs() -> BundleInputs { BundleInputs { + observations: drive_observations(), run_id: "skeleton-wave-a-test".into(), repetition: 1, warmup_seconds: 0, diff --git a/doc/instance-throughput-rewrite-plan.md b/doc/instance-throughput-rewrite-plan.md index 0ad792e..7c3afa4 100644 --- a/doc/instance-throughput-rewrite-plan.md +++ b/doc/instance-throughput-rewrite-plan.md @@ -1217,6 +1217,183 @@ shard tree outside the root. `read_format` follows a link at `FORMAT`, read-only priority. The first is a correctness defect in the locking discipline and should be scheduled on its own, not folded into a later pass. +##### Contract review 2026-07-28-C + +B4 re-pointed `store-bench` at a real `StoreEngine::submit` and found that four verification +claims `bench/result-schema.json` declares not-applicable at `storage_primitive` have become +genuinely earnable. It **requested rather than emitted** them, which is what §6.6 item 4 asks +for. All four are granted, three with conditions that are landed as schema constraints rather +than as prose, and one — `operation_receipts_reconciled` — is granted **in principle and is +not earned by the emitter as it stands.** The schema and its contract tests land here; the +emitter is B4's follow-up and is specified in scope §6.6. + +**1. Machine-readable run conditions, ranked first because the other three depend on it.** +There was nowhere in a bundle to record that the root was seeded outside `StoreEngine::open`, +that `max_index_runs` was raised, or that no checkpoint was taken. Every one of those is +true of the run that produces today's numbers, and every one of them lived only in a harness +comment and a human report. A bundle whose caveats live outside it reads as unconditional to +everyone who receives it, and the people most likely to receive it without the report are the +ones furthest from the harness. + +`run_conditions` is a new required top-level block with ten members: `initialization_path`, +`mutation_path`, `checkpointing`, `index_maintenance`, `index_run_ceiling`, +`receipt_reconciliation`, `objects_new_source`, `commit_id_uniqueness`, `build_profile`, and +`environment_fidelity`. Every member is a closed enumeration; there is no free-text member and +no catch-all value, and a contract test asserts both properties over the block rather than +over a list this file also wrote. + +**A prose caveat field was rejected outright, and the reason is the whole design.** A string +is something a consumer reads; these are things a consumer checks. The block is not a place to +put disclosures beside the claims — it is what the claims are *conditioned on*. +`objects_new_equals_three_per_commit` is forbidden when `objects_new_source` says the count was +derived from the transaction total. `unique_blob_tree_commit_ids` is forbidden when +`commit_id_uniqueness` says the check was per-record or inferred. `operation_receipts_reconciled` +is forbidden when `receipt_reconciliation` says any `Committed` status was accepted, and +*required* when it says otherwise. A harness can only declare what it did, and the declaration +decides what it may claim. That is the difference between a caveat and a condition. + +Two of scope §7's exit clauses were prose until now and are mechanical here: a passing +`storage_primitive` run must declare `checkpointing: "exercised"` — §7 requires the P2 runs not +to have been achieved with checkpointing disabled — and `index_maintenance: "runs_sealed"`, +because a run holding every index delta in memory with a lookup fan-out that grows for its +whole duration is not the steady state a P2 number describes. It must also declare +`initialization_path: "store_engine_open"` and `mutation_path: "store_engine_submit"`. The +harness satisfies **two of the four** — B1's startup state 1 landed, so the submit path both +creates its root and mutates it through the production entry points. It cannot satisfy +`checkpointing` or `index_maintenance`, so it still cannot emit a passing P2 bundle, which is +correct and was previously only an assertion in a comment. + +That the count moved from zero to two is worth stating rather than silently editing: the four +conditions are not decoration on a number, they are the number's preconditions, and knowing +which remain unmet says exactly how far the P2 criterion is from being earned. The two that +remain are the two that make a P2 figure a steady-state measurement rather than a burst. + +**2. `objects_new_equals_three_per_commit`, granted for the production submit path.** The +schema said it was absent at `storage_primitive` "which creates no objects". That stopped being +true when the measured path became `submit`: the path stages the canonical three objects per +commit and the harness sums `receipt.objects_new` from the store's own receipts. The condition +is that the claim compares an **independently summed receipt total against a separately counted +`3 × counted_commits`**. Both sides deriving from the transaction count is what the claim was +originally excluded for — an assertion that cannot fail is not a check — and +`objects_new_source` is what makes that exclusion survive the grant, at this gate and at every +other. + +**3. `operation_receipts_reconciled`, granted in principle and not earned.** +`store-bench.rs:2033` accepts any `TransactionStatus::Committed(_)` without comparing its +payload, and the `receipt_digest` the harness journals is `blake3(operation_id)` — a digest of +the operation, not of the receipt. So the run reads back that *something* committed, which is +already what `acknowledged_sequences_reconciled` reports, and calling it receipt reconciliation +would be a second name for the same evidence. The claim is landed as expressible and correctly +constrained: an emitter that reconciles exact receipts, or a frozen canonical receipt digest, +declares it and **must** then assert the claim; an emitter that does not declares +`acceptance_of_any_committed_status` and **cannot**. Landing it now means closing it is an +emitter change rather than a second schema amendment, and the schema description says in as +many words that the current emitter does not earn it. + +**4. `unique_blob_tree_commit_ids`, granted for the production submit path**, on condition that +uniqueness is established **globally across every recovered acknowledgment record**. Per-record +uniqueness is not uniqueness — two records may each be internally distinct and still share a +commit id — and distinct generator seed domains make a collision unlikely rather than absent, +which is an argument about probability rather than an observation. `inferred_from_seed_domains` +is therefore a *named* value of `commit_id_uniqueness` rather than something folded into the +passing one: a harness that reasoned that way has a truthful thing to record and is refused the +claim, which is a better outcome than having to choose between a lie and silence. + +**The branch conditional, which is the part that is easy to get wrong.** These are not global +loosenings. A schema that merely *permitted* the three claims on both paths would hand the +Wave A journal seam a way to assert what nothing below `engine.rs` can observe — and that is a +worse defect than the one being fixed, because it arrives disguised as the fix. Two rules key +on `gate == "storage_primitive"` and `run_conditions.mutation_path`: + +- **`journal_drive`** forbids all three claims outright *and* pins the four provenance + declarations to the only values the seam can truthfully make (`no_index_in_path`, + `no_receipts_in_path`, `derived_from_transaction_count`, `not_checked`). Forbidding the + claims alone would have left the same hole one field over: a drive-path bundle could declare + exact receipt reconciliation it has no receipts to perform, and nothing would have noticed. +- **`store_engine_submit`** requires `unique_blob_tree_commit_ids` and + `objects_new_equals_three_per_commit`, both `const true`. Omission is a missing result here, + not an inapplicable one. + +`blobs_recomputed`, `metadata_complete`, and `commits_in_recovered_closure` stay unavailable on +**both** paths and stay in the gate-wide rule, because they need the graph traversal plan §5.1 +forbids the store from performing. The gate-wide forbidden set is now exactly the claims no +path can earn, and a contract test asserts its size so a future claim cannot be quietly parked +there. + +**`max_index_runs` is now a named required member of `resources.configured_ceilings`.** It was +reachable only through that block's free-form `additionalProperties`, so an emitter could omit +the one ceiling this workload actually reaches — `submit` refuses `NotImplemented` at it — and +the bundle stayed valid. It must be the value the run configured, read back from the options the +store opened with. The schema cannot see the process's options, so the bite is a cross-check: +`index_run_ceiling: "store_default"` bounds the recorded value at 64 and +`"raised_because_index_sealing_unimplemented"` floors it at 65, and a bundle that declares one +while recording the other is invalid in both directions. A contract test asserts that +`StoreOptions::default`'s 64 is still 64, so the schema's bound and the library cannot drift in +silence. What remains open is a bundle that lies about both consistently, which no schema +closes, and it is stated here rather than left to be discovered. + +**The truthful-environment ruling: yes, and here is why.** `deployment.persistent_data_mount` +and `deployment.tmpfs` were unconditional `const true` / `const false`, so B4's debug/tmpfs +diagnostic run at 8,052/s **could not be encoded at all**. It was not disqualified — it was +unrepresentable, which is a strictly worse state: the number existed, it was informative, and +the only places it could live were a console and a paragraph. That is the same failure the +`outcome` field was added to fix in review 2026-07-24-B, one block over, and the same argument +applies. A schema that can only express successful runs is not a record of what was measured. + +The two fields relax to `type: boolean` and are **re-pinned** by +`environment_fidelity: "reference_profile"`, which additionally requires a `release` build and +a named hardware profile. `outcome: "pass"` requires `reference_profile` **at every gate**, so +nothing a claim used to cost has changed — a `gate="storage_primitive"` bundle claiming P2 +still requires the real environment, by a rule that is one implication instead of two consts. +Going the other way, `"diagnostic"` is not merely a label: outcome is bounded to `fail` or +`preliminary` and no verdict may be `pass`. Recording a diagnostic run is worth nothing unless +the record also refuses to let it be read as a result. + +What was **not** relaxed, deliberately: `overlay`, `remote_storage`, and `durability_enabled` +keep their unconditional consts. A run with durability disabled is not a slower measurement of +the same thing, it is a measurement of something else, and there is no diagnostic value in a +fence-free number that would justify making it expressible. + +**Expected collateral: the emitter no longer produces a valid bundle, and the gate is red until +B4's follow-up.** Exactly two fields are missing, on both paths: + +``` +[]: 'run_conditions' is a required property +['resources', 'configured_ceilings']: 'max_index_runs' is a required property +``` + +`scripts/check-phase1.sh` does not run `scripts/verify-store-recovery.sh`, so the expectation +was that the gate would stay green while bundle emission broke. **It does not, and the reason +is worth recording:** the gate runs `store-bench`'s own unit tests, and since review +2026-07-24-B three of them validate the emitted bundle against `bench/result-schema.json` +rather than against a list of substrings. So the emitter's schema conformance is inside the +gate, which is exactly the property that review was after — the drift is reported by the gate +instead of by a script nobody ran. `the_emitted_bundle_validates_against_the_frozen_schema` and +`a_failing_run_is_representable_rather_than_suppressed` fail with the two errors above; +`the_schema_check_can_actually_fail` fails on its final assertion for the same reason and not a +second one, because it asserts that a coordinated-omission mutation leaves a *clean* bundle and +the base bundle is no longer clean. `scripts/verify-store-recovery.sh --cycles 2` was run +directly rather than assumed and reports `matrix=pass`, `cycles_completed=2`, +`acknowledged_loss=0`, `torn_transactions=0`, `repeated_adoptions=0`, `bundle=schema-invalid`, +`VERIFY_EXIT=1`. + +All three failures are in `store-bench.rs`, which is B4's file, and the fix is scope §6.6 item +5 rather than an edit here. This is the sequencing of 2026-07-28-A repeated deliberately: the +gate is transiently red between the contract and the package's pass, and landing the contract +first is what keeps B4 from implementing against a surface that is about to move. It is +recorded rather than worked around, because a lead who edits the emitter to keep the gate green +has moved a package's work into a review and left no one able to see that it happened. + +One measurement from that run belongs on the record, because it is what makes amendment 2 more +than an argument: the submit path reported `objects_new=861` against `transactions=287`, summed +from 287 independent receipts. Three per commit, counted rather than multiplied. + +Amended: `bench/result-schema.json`, `crates/levcs-protocol/tests/phase0_benchmark_contracts.rs`, +and scope §6.6 and §7. `bench/reference-hardware.toml` required no change: the frozen profiles +describe the reference environment, and `environment_fidelity` records which runs met it — +putting a diagnostic profile in the frozen file would have made a non-comparable configuration +part of what "frozen" means. + ### Phase 1 — storage engine spine Lead first defines sealed transaction/frame/snapshot interfaces and file ownership. That deliverable (D0) landed on 2026-07-24 as `crates/levcs-store`: the frozen public API compiling against `StoreError::NotImplemented`, the file-ownership split, strict configuration validation, the single durability syscall funnel with its counters and fault hooks, the failpoint registry in enforced one-to-one correspondence with `oracle::AppendFailpoint`, and the journal-level drive seam that lets the crash harness run in Wave A. The enforced gate is `scripts/check-phase1.sh`, which runs `check-phase0.sh` first so the Phase 0 freeze stays enforced. That work is scoped in `doc/phase1-storage-spine-scope.md`, which realizes this section as a file-ownership matrix, a frozen `levcs-store` API, a physical format and durability/recovery specification, per-package deliverables and acceptance criteria, the Wave A adversarial review charter, and the capacity analysis for P2 on the frozen reference hardware. This plan remains authoritative; that document is the Phase 1 realization of it and lists the decisions that must be resolved before Wave A starts. diff --git a/doc/phase1-storage-spine-scope.md b/doc/phase1-storage-spine-scope.md index 98add60..b36cd77 100644 --- a/doc/phase1-storage-spine-scope.md +++ b/doc/phase1-storage-spine-scope.md @@ -1190,6 +1190,36 @@ enforced somewhere and not on the path that runs. Here, an invariant was documen than required. In both cases the gate reported green because nothing made the omission expressible as a failure. +### 5.1 Resource exhaustion is indistinguishable from a concurrency flake by symptom + +Recorded from a Wave B incident, because the wrong diagnosis was already written down before +the evidence arrived and only measurement caught it. + +Verifying a commit in a second worktree put an 11 GiB `target` directory on this machine's +`/tmp`, which is tmpfs. The store's tests build roots under `TMPDIR`, and the engine +preallocates per shard, so the filesystem filled. Seventeen `engine::tests` failures appeared +at once. **Every one passed in isolation and failed under the full suite** — the exact +signature §5's `recovery_eio` record describes, and the reason is structural rather than +coincidental: whichever tests run last are the ones that find the filesystem full, and which +tests run last depends on scheduling. Re-running the main tree reproduced it 5 of 5, which +looked like confirmation of an intrinsic flake in newly added startup tests. It was not. The +panic carried `Io(Os { code: 122, kind: QuotaExceeded })`, and after the worktree was removed +the same suite passed 5 of 5 unchanged. + +**The rule this yields.** Before classifying clustered failures as a concurrency flake, +preserve and read the **errno**, and capture free blocks, free inodes, and any quota state for +the filesystem the test roots live on. `ENOSPC`, `EDQUOT`, and `EMFILE` all present as +unrelated-looking failures that vanish in isolation, and all three are cheap to rule out and +expensive to misdiagnose: the flake conclusion sends someone hunting a race that does not +exist, and — worse — it invites the rerun-until-green habit §5 exists to forbid, which would +have "resolved" this incident while leaving the disk full. + +The corollary for harnesses: an I/O error must reach a report with its errno intact. A path +that folds one into a boolean, a count, or a generic message destroys the only evidence that +distinguishes these two diagnoses. That is the same requirement as the emitter's +incomplete-accounting refusal in §6.6 — a failure the harness cannot account for must be +reported as itself, not compressed into a symptom. + ## 6. Wave B work packages ### 6.0 Preconditions @@ -1767,6 +1797,98 @@ Owns the crash driver, the benchmark, the matrix, and the recovery script. `commits_in_recovered_closure` and the object-graph flags forbidden at this gate should be re-examined. If any becomes genuinely earnable, that is a schema amendment and a contract review — **request it, do not emit it.** `bench/result-schema.json` is lead-owned. + *Requested, reviewed, and granted as contract review 2026-07-28-C; the schema and its + contract tests have landed and item 5 below is what B4 must emit against them.* + +#### Consequence of contract review 2026-07-28-C: the emitter contract + +`bench/result-schema.json` now requires a `run_conditions` block and a named +`resources.configured_ceilings.max_index_runs`, and requires two verification claims on the +submit path. The emitter does not produce any of them, so `store-bench emit-skeleton` currently +writes a bundle that fails validation on exactly two fields: + +``` +[]: 'run_conditions' is a required property +['resources', 'configured_ceilings']: 'max_index_runs' is a required property +``` + +`scripts/verify-store-recovery.sh --cycles 2` reports `bundle=schema-invalid`, `VERIFY_EXIT=1` +with everything else green (`matrix=pass`, `acknowledged_loss=0`, `torn_transactions=0`). +**`scripts/check-phase1.sh` is red as well**, which is not what landing a schema alone would +normally do: the gate runs `store-bench`'s unit tests, and three of them validate the emitted +bundle against the schema rather than against substrings, so schema conformance is inside the +gate. That is the property review 2026-07-24-B was after, working. All three failures are in +`store-bench.rs` and none is a defect in the store; they are expected collateral of landing the +contract first and they close with item 5. + +5. **Emit the run conditions, the index-run ceiling, and the two earned claims.** All of it in + `store-bench.rs`; no other file is involved. Nothing here may be a constant this file + restates — every value must come from what the run configured or observed. + + **a. `resources.configured_ceilings.max_index_runs`** — the `u32` from the `StoreOptions` + the store was opened with, not `ENGINE_MAX_INDEX_RUNS` written out a second time. Emit it as + an integer. + + **b. A `run_conditions` object** with all ten members. The truthful values today are: + + | member | submit path | drive path | + |---|---|---| + | `initialization_path` | `store_engine_open` (see **f**) | `shard_drive_create` | + | `mutation_path` | `store_engine_submit` | `journal_drive` | + | `checkpointing` | `unimplemented` | `unimplemented` | + | `index_maintenance` | `deltas_retained_in_memory` | `no_index_in_path` | + | `index_run_ceiling` | `raised_because_index_sealing_unimplemented` | `store_default` | + | `receipt_reconciliation` | `acceptance_of_any_committed_status` | `no_receipts_in_path` | + | `objects_new_source` | `summed_from_receipts` | `derived_from_transaction_count` | + | `commit_id_uniqueness` | `checked_globally_across_ack_records` (after **d**) | `not_checked` | + | `build_profile` | from `cfg!(debug_assertions)` | same | + | `environment_fidelity` | `diagnostic` unless every reference condition holds | same | + + These are **declarations of what the run did**, not configuration. Each must be derived from + the `MeasuredRun` and the options rather than hardcoded per path where a derivation exists: + `index_run_ceiling` follows from comparing the configured ceiling to the store default, + `objects_new_source` is the existing `MeasuredRun::objects_new_counted`, `build_profile` + follows from `cfg!`, and `environment_fidelity` follows from the profile the run verified. + A hardcoded `run_conditions` block is the prose caveat with a different syntax. + + **c. Both earned claims on the submit path**, and neither on the drive path. + `objects_new_equals_three_per_commit` may be emitted only when the summed + `receipt.objects_new` total equals a **separately counted** `3 × counted_commits` — count the + commits, do not reuse the summed total to produce the expected value, and refuse to emit the + bundle when they disagree rather than emitting the flag as `false`, which the schema does not + permit and which would be a different untrue statement. + + **d. A global uniqueness check** over every recovered ACK record, across all records at once: + collect every `blob_ids`, `tree_ids`, and `commit_ids` entry from + `ExternalAckJournal::recover` into one set per kind and require no repeat. Per-record checking + and any argument from the distinctness of the generator seed domains are both explicitly + insufficient, and `commit_id_uniqueness` has named values for both so a harness that did + either has something truthful to record. + + **e. Do not emit `operation_receipts_reconciled`.** The schema forbids it while + `receipt_reconciliation` is `acceptance_of_any_committed_status`, which is the honest + declaration for `store-bench.rs:2033` — it accepts any `Committed(_)` without comparing the + payload, and the `AckRecord.receipt_digest` it writes is `blake3(operation_id)` rather than a + digest of the receipt. Earning it is separate work: reconcile the exact receipt, or freeze a + canonical receipt digest and reconcile that, then declare the matching value and assert the + claim. The schema will then *require* the claim rather than permit it. + + **f. Build every measured root through `StoreEngine::open`.** Added after B1 landed startup + state 1. The benchmark previously seeded its root with `segment::initialize_root` and + disclosed the fact in three places, because seeding a store off the production path in order + to measure the production path is the charter item 8 smell and a disclosure is not a fix. + With state 1 implemented the smell is closable rather than merely recordable, so it is + closed: the seeding helpers and the `ROOT_SEEDED_BY_NON_PRODUCTION_PATH` constant are retired, + and `initialization_path` is **observed** — the `FORMAT` marker is absent before the call and + present after — rather than asserted. Deriving it from an observation is what keeps the + declaration honest if the seeding ever regresses. + + *Accept:* `bash scripts/verify-store-recovery.sh --cycles 2` reports `bundle=schema-valid` + on both `--path submit` and `--path drive`; the emitter's own + `the_emitted_bundle_validates_against_the_frozen_schema` passes; and + `the_schema_check_can_actually_fail` gains a mutation for each newly required field, since a + required field the negative control never removes is a field the suite cannot notice the loss + of. #### Carry-forward: the SIGKILL cycles still drive the journal seam @@ -1781,11 +1903,17 @@ one step of a publication and not the step where the status root, the sequencer, acknowledgment, and the checkpoint install are at risk. Every ordering hazard that only exists between those is untested by this script, at any cycle count. -Moving the cycles onto production submit is **blocked on `StoreEngine::open` startup state -1**: the child process cannot create a store root through the production entry point, which -still refuses that state by name (B1 deliverable 1). The same block is what forces -`engine_matrix.rs` to seed roots through `segment::initialize_root`, and it is disclosed -there as `ROOT_SEEDED_BY_NON_PRODUCTION_PATH`. +Moving the cycles onto production submit was blocked on `StoreEngine::open` startup state 1, +because the child process could not create a store root through the production entry point. +**That block is gone**: B1 landed state 1, and every root B4 measures — the benchmark's and +the Wave B rows' — is now built by `StoreEngine::open`, so `engine_matrix.rs`'s +`ROOT_SEEDED_BY_NON_PRODUCTION_PATH` disclosure and its `segment::initialize_root` seeding +are retired. + +The move is therefore **deferred, not blocked** — a separate B4 assignment that has not been +made rather than one that cannot be done. That distinction matters here: a blocked item waits +for someone else, and a deferred one waits only for a decision, so this is the entry that +should be picked up first when the acknowledged-crash-recovery criterion is next worked. Consequences, stated so no later reader has to reconstruct them: @@ -1914,6 +2042,24 @@ Also required before the phase closes, from §13's stop conditions: the result b report index bytes/object and checkpoint lookup fan-out, and the P2 runs must not have been achieved with checkpointing disabled. +**Both of those last clauses are now mechanical rather than prose** (contract review +2026-07-28-C). A `gate="storage_primitive"` bundle with `outcome="pass"` must declare +`run_conditions.checkpointing = "exercised"` and `run_conditions.index_maintenance = +"runs_sealed"`, alongside `initialization_path = "store_engine_open"` and `mutation_path = +"store_engine_submit"`. A reader no longer has to take the P2 row of the table above on trust: +the conditions the number was obtained under travel inside the bundle as values, and a bundle +that met none of them cannot encode a pass. + +The harness as it stands satisfies **two of the four**: since B1 landed startup state 1, the +submit path both creates its root through `StoreEngine::open` and mutates it through +`StoreEngine::submit`. It satisfies neither `checkpointing` nor `index_maintenance`, and those +are the two that decide whether a P2 figure describes a steady state or a burst — a run holding +every index delta in memory, with a lookup fan-out that grows for its whole duration and no +checkpoint ever taken, is measuring a system that has not yet reached the condition the number +is supposed to characterize. Two of four is the accurate reading of how much of the P2 exit +criterion is currently earned, and the remaining two are the expensive ones — the same +disclosure the SIGKILL carry-forward above makes about the crash-recovery row. + ## 8. Capacity analysis for P2 on the frozen reference hardware `bench/reference-hardware.toml` freezes both profiles as a Ryzen 7 9800X3D (8 physical diff --git a/scripts/verify-store-recovery.sh b/scripts/verify-store-recovery.sh index 6f6db4a..d93749c 100755 --- a/scripts/verify-store-recovery.sh +++ b/scripts/verify-store-recovery.sh @@ -237,15 +237,15 @@ done # --------------------------------------------------------------------------- bundle_status="skipped" +zero_work_status="skipped" +unaccounted_status="skipped" if [ "$run_bundle" = "1" ]; then echo "== skeleton result bundle ==" >&2 if cargo build -q -p levcs-store \ --features bench-harness,store-internals,store-privileged \ --bin store-bench 2>/dev/null; then bundle_parent="$work/bundle" - bundle_root="$bundle_parent/root" - bundle_out="$work/storage-primitive-skeleton.json" - rm -rf "$bundle_parent" "$bundle_out" + rm -rf "$bundle_parent" # The frozen profile requires nodatacow on the journal and segment # directories, and store-bench refuses on mismatch rather than recording @@ -257,8 +257,21 @@ if [ "$run_bundle" = "1" ]; then mkdir -p "$bundle_parent" chattr +C "$bundle_parent" 2>/dev/null || true + # Both paths, because the schema's verification rules branch on + # `run_conditions.mutation_path`: the production submit path is *required* + # to assert the two claims contract review 2026-07-28-C granted, and the + # journal seam is forbidden from asserting any of the three. A run that + # validated one path would leave the other's rules unexercised, and the + # unexercised one is where a seam bundle claiming what the seam cannot + # observe would appear. + bundle_status="schema-valid" + for bundle_variant in submit drive; do + bundle_root="$bundle_parent/$bundle_variant-root" + bundle_out="$work/storage-primitive-skeleton-$bundle_variant.json" + rm -rf "$bundle_root" "$bundle_out" + if "$repo_root/target/debug/store-bench" emit-skeleton \ - --root "$bundle_root" --out "$bundle_out" \ + --root "$bundle_root" --out "$bundle_out" --path "$bundle_variant" \ --allow-unsigned --seconds 2 --group-len 16 >&2; then if python3 - "$repo_root/bench/result-schema.json" "$bundle_out" >&2 <<'PY' import json, sys @@ -278,7 +291,9 @@ for error in errors: sys.exit(1 if errors else 0) PY then - bundle_status="schema-valid" + # A later variant may only keep the status the earlier one earned; it + # can never upgrade a failure back to valid. + : else case $? in 3) bundle_status="unvalidated-no-jsonschema" ;; @@ -288,6 +303,63 @@ PY else bundle_status="refused" fi + done + + # The zero-work negative control. + # + # A run with no submitter measures nothing, yet the repositories are still + # created — so before this was fixed the bundle reported their fences and + # their signatures as measured work and asserted setup_traffic_excluded, + # unique_blob_tree_commit_ids, and objects_new_equals_three_per_commit over + # an empty set. Every one of those is vacuously true over zero commits, and + # bench/result-schema.json validates such a bundle without complaint: counts + # are nonnegative and the claims are const true. The schema therefore cannot + # be the thing that refuses it, and a gate that only ever ran the happy path + # would not notice the emitter's refusal being removed. + zero_work_root="$bundle_parent/zero-work-root" + zero_work_out="$work/storage-primitive-zero-work.json" + rm -rf "$zero_work_root" "$zero_work_out" + if "$repo_root/target/debug/store-bench" emit-skeleton \ + --root "$zero_work_root" --out "$zero_work_out" --path submit \ + --allow-unsigned --seconds 2 --group-len 16 \ + --submitters-per-shard 0 >&2; then + zero_work_status="emitted" + else + zero_work_status="refused" + fi + # A refusal that still wrote a bundle is not a refusal. + if [ -f "$zero_work_out" ]; then + zero_work_status="emitted" + fi + + # The incomplete-accounting negative control. + # + # A commit whose acknowledgment cannot be journaled is a transaction the + # store performed, fenced, and signed, and that the harness can never + # count. Until this was fixed the run merely stopped: the failure set no + # counter and recorded no error, so the emitter's refused-submit guard + # never saw it, and — because it happens after the first commit — neither + # did the zero-work guard. The bundle that came out omitted a committed + # transaction while still reporting its fence and its signature, and was + # schema-valid, exactly like the zero-work bundle above. The failure is + # induced with a write to /dev/full so the kernel supplies a real ENOSPC + # rather than the harness inventing one; a full tmpfs reaches the same + # code path with EDQUOT. + unaccounted_root="$bundle_parent/unaccounted-root" + unaccounted_out="$work/storage-primitive-unaccounted.json" + rm -rf "$unaccounted_root" "$unaccounted_out" + if "$repo_root/target/debug/store-bench" emit-skeleton \ + --root "$unaccounted_root" --out "$unaccounted_out" --path submit \ + --allow-unsigned --seconds 2 --group-len 4 \ + --shards 1 --submitters-per-shard 1 \ + --fail-ack-append-after 1 >&2; then + unaccounted_status="emitted" + else + unaccounted_status="refused" + fi + if [ -f "$unaccounted_out" ]; then + unaccounted_status="emitted" + fi else bundle_status="build-failed" fi @@ -311,6 +383,8 @@ echo "acknowledged_loss=$total_acknowledged_loss" echo "torn_transactions=$total_torn" echo "repeated_adoptions=$total_repeated" echo "bundle=$bundle_status" +echo "zero_work_run=$zero_work_status" +echo "unaccounted_ack_run=$unaccounted_status" exit_code=0 [ "$matrix_status" = "fail" ] && exit_code=1 @@ -325,6 +399,13 @@ exit_code=0 [ "$bundle_status" = "unvalidated-no-jsonschema" ] && exit_code=1 [ "$bundle_status" = "refused" ] && exit_code=1 [ "$bundle_status" = "build-failed" ] && exit_code=1 +# A zero-work run that produced a bundle is a bundle whose claims are vacuous, +# and it is schema-valid, so this is the only place it can be caught. +[ "$zero_work_status" = "emitted" ] && exit_code=1 +# A run that could not account for a committed transaction and emitted a bundle +# anyway published totals for a workload that did not happen, and that bundle is +# schema-valid too. +[ "$unaccounted_status" = "emitted" ] && exit_code=1 echo "VERIFY_EXIT=$exit_code" exit "$exit_code"