"""Plain `unittest` coverage for `verify.classify` and `verify.validate_expectations_file` — check 5's whole scoring logic and its fail-closed artifact validation, isolated from any live AT-SPI bus. `verify.py`'s `gi.repository.Atspi` import happens inside `main()`, not at module import time, specifically so this file can `import verify` and drive `classify()` directly with synthetic `ObservedNode` trees — no bus, no desktop, no display required. Run with: python3 -m unittest from this directory (`a11y-verifier/`). Every test constructs the *wrong* input for the property it names and asserts the *specific* verdict/outcome that input must produce — never a bare "FAIL" or "not PASS", because a classifier branch that silently degrades to the wrong FAIL reason would pass a weaker test and is exactly the kind of regression this file exists to catch. """ import unittest from verify import ( EXPECTED_FIXTURE_IDS, PLATFORM, VISUAL_ORDER_TRAP, ObservedNode, Verdict, _is_source_bearing_fragment, classify, validate_expectations_file, ) def node(role, name, children=None): """A synthetic `ObservedNode`, the same shape `walk_for_check5` builds from a live AT-SPI tree.""" return ObservedNode(role=role, name=name, children=list(children) if children else []) def expectation( expected_name, accepted_roles=("label", "static", "text", "paragraph"), prohibited_roles=("image", "canvas", "filler", "panel", "unknown"), alternative_forms=None, visual_order_name=None, source_atoms=None, ): """A synthetic fixture entry with the same shape `round2-a11y-oracle/a11y_expectations.json` emits.""" return { "fixture_id": "F-TEST", "expected_name": expected_name, "expected_name_hex": expected_name.encode("utf-8").hex(), "expected_name_byte_len": len(expected_name.encode("utf-8")), "accepted_roles": list(accepted_roles), "prohibited_roles": list(prohibited_roles), "alternative_forms": alternative_forms or {}, "visual_order_name": visual_order_name, # D1: per-segment source atoms (`round2-a11y-oracle`'s # `source_atoms`). Defaults to `None` (classify's own `.get(...) or # []` treats that as no atoms), since most tests don't need one. "source_atoms": source_atoms, } class ByteExactPass(unittest.TestCase): def test_single_node_with_accepted_role_and_exact_name_passes(self): exp = expectation("Allegro affettuoso — al fine") verdict = classify(exp, [node("text", "Allegro affettuoso — al fine")]) self.assertEqual(verdict.verdict, "PASS") self.assertIsNone(verdict.prohibited_outcome) self.assertEqual(verdict.observed_role, "text") self.assertEqual(verdict.observed_name, exp["expected_name"]) def test_a_one_byte_difference_does_not_pass(self): """Mutation guard: if byte comparison were replaced by e.g. a case-insensitive or trimmed comparison, this would wrongly PASS.""" exp = expectation("Allegro") verdict = classify(exp, [node("text", "allegro")]) self.assertNotEqual(verdict.verdict, "PASS") class CompositionConcatenationPass(unittest.TestCase): def test_two_segment_names_concatenated_in_tree_order_pass(self): exp = expectation("Coro אבג") # No single node carries the whole name — only the concatenation of # two text descendants of a common parent, in tree/logical order, # does. run = node("frame", "", children=[node("text", "Coro "), node("text", "אבג")]) verdict = classify(exp, [run]) self.assertEqual(verdict.verdict, "PASS") self.assertIsNone(verdict.prohibited_outcome) self.assertEqual(verdict.observed_name, exp["expected_name"]) def test_concatenation_in_the_wrong_order_does_not_pass(self): """Mutation guard: if concatenation order were unspecified (e.g. a set instead of an ordered list), swapping the two nodes would still wrongly PASS.""" exp = expectation("Coro אבג") run = node("frame", "", children=[node("text", "אבג"), node("text", "Coro ")]) verdict = classify(exp, [run]) self.assertNotEqual(verdict.verdict, "PASS") class ProhibitedOutcomes(unittest.TestCase): """One test per `round2_textkit::a11y::PROHIBITED_OUTCOMES` name.""" def test_absent_from_tree_when_no_candidate_node_is_found(self): exp = expectation("Allegro") verdict = classify(exp, []) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "absent-from-tree") def test_name_empty_is_distinct_from_absent_from_tree(self): """§8.3: "name-empty ... absence wearing a role." A node is present (unlike the absent-from-tree case above) but its name is the empty string — these must classify differently.""" exp = expectation("Allegro") verdict = classify(exp, [node("label", "")]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "name-empty") def test_name_normalized_matches_the_precommitted_nfc_form(self): # F-E's case: source is NFD ("Cafe" + combining acute, "Café"), # a wrong tree exposes the NFC "Café" ("Café", a *different* # string byte-for-byte even though the two render identically) # instead. Written with explicit \N escapes rather than the literal # glyph so the two forms cannot be silently typed as the same string # by accident — that mistake produced a false PASS here once already. nfd = "Cafe\N{COMBINING ACUTE ACCENT}" nfc = "Caf\N{LATIN SMALL LETTER E WITH ACUTE}" self.assertNotEqual(nfd, nfc, "anchor: the two forms must be different strings") exp = expectation( nfd, alternative_forms={"name-normalized": [nfc]}, ) verdict = classify(exp, [node("text", nfc)]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "name-normalized") self.assertEqual(verdict.observed_name, nfc) def test_name_is_shaped_glyphs_matches_the_precommitted_ligature_form(self): # F-A's case: the `ff` ligature collapses, so a wrong tree drops a # letter relative to the source string. exp = expectation( "Allegro affettuoso — al fine", alternative_forms={"name-is-shaped-glyphs": ["Allegro afettuoso — al fne"]}, ) verdict = classify(exp, [node("text", "Allegro afettuoso — al fne")]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "name-is-shaped-glyphs") def test_name_drops_unresolved_codepoints_matches_the_precommitted_form(self): # F-C's case: U+0627 is covered by no declared face and is dropped. exp = expectation( "Coro ا", alternative_forms={"name-drops-unresolved-codepoints": ["Coro "]}, ) verdict = classify(exp, [node("static", "Coro ")]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "name-drops-unresolved-codepoints") def test_a_correct_name_does_not_spuriously_match_an_alternative_form(self): """Mutation guard: if alternative-form matching ran before the exact match, a correct name would risk matching an alternative_forms entry by accident and wrongly FAIL.""" exp = expectation( "Coro ا", alternative_forms={"name-drops-unresolved-codepoints": ["Coro "]}, ) verdict = classify(exp, [node("static", "Coro ا")]) self.assertEqual(verdict.verdict, "PASS") class VisualOrderDiagnosis(unittest.TestCase): def test_visual_order_concatenation_is_named_specifically(self): # F-D's designed trap: a tree walking visual runs left to right # reverses the embedded RTL segment's codepoint order. exp = expectation( "Allegro אבג con brio", visual_order_name="Allegro גבא con brio", ) run = node( "frame", "", children=[node("text", "Allegro "), node("text", "גבא"), node("text", " con brio")], ) verdict = classify(exp, [run]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, VISUAL_ORDER_TRAP) self.assertEqual(verdict.observed_name, exp["visual_order_name"]) def test_visual_order_diagnosis_is_not_reported_as_a_generic_mismatch(self): """Mutation guard: if the visual-order check were deleted, this would still FAIL but with `prohibited_outcome=None` (the generic fallback) instead of the specific diagnosis — asserting the exact name, not just FAIL, is what catches that.""" exp = expectation( "Allegro אבג con brio", visual_order_name="Allegro גבא con brio", ) run = node( "frame", "", children=[node("text", "Allegro "), node("text", "גבא"), node("text", " con brio")], ) verdict = classify(exp, [run]) self.assertIsNotNone(verdict.prohibited_outcome) self.assertNotEqual(verdict.prohibited_outcome, "absent-from-tree") self.assertNotEqual(verdict.prohibited_outcome, "name-empty") class ProhibitedRole(unittest.TestCase): def test_a_prohibited_role_fails_even_with_the_exact_name(self): exp = expectation("Allegro") verdict = classify(exp, [node("canvas", "Allegro")]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.observed_role, "canvas") # This is a role-vocabulary failure (§8.2), not one of the five # name-transformation PROHIBITED_OUTCOMES (§8.3) — it must not be # reported as one. self.assertIsNone(verdict.prohibited_outcome) self.assertEqual( verdict.reason, "a node's name matches expected_name byte-for-byte, but its role 'canvas' is in the " "at-spi2 prohibited set (no accepted-role node also carries it)", ) def test_an_accepted_role_with_the_exact_name_is_not_penalized(self): """Mutation guard: confirms the previous test is actually exercising the role check, not some other reason that name would fail.""" exp = expectation("Allegro") verdict = classify(exp, [node("label", "Allegro")]) self.assertEqual(verdict.verdict, "PASS") def test_an_unlisted_role_carrying_the_exact_name_is_not_absent_from_tree(self): """A role outside `accepted_roles | prohibited_roles` (e.g. `push button`) is never a text-*candidate* for the composition/single-node role checks — `_flatten_candidates`/`_subtree_contributors` exclude it. But a node under that role can still carry the run's exact text, and §8.3's absent-from-tree ("the default outcome for a toolkit that draws to a canvas and stops") does not describe that: the run *is* in the tree. This must FAIL naming the actual observed role, not report absent-from-tree — conflating "wrong role" with "nothing there at all" would hide evidence a real candidate produced.""" exp = expectation("Allegro") verdict = classify(exp, [node("push button", "Allegro")]) self.assertEqual(verdict.verdict, "FAIL") self.assertNotEqual(verdict.prohibited_outcome, "absent-from-tree") self.assertEqual(verdict.observed_role, "push button") self.assertEqual(verdict.observed_name, "Allegro") def test_a_truly_empty_tree_is_still_absent_from_tree(self): """The companion case to the one above, pinned side by side so a regression that merges the two back together (e.g. by making the new unlisted-role scan fire unconditionally) is caught: with nothing in the tree at all, the outcome must still be `absent-from-tree`.""" exp = expectation("Allegro") verdict = classify(exp, []) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "absent-from-tree") def test_an_unlisted_role_carrying_an_alternative_form_is_also_not_absent_from_tree(self): """The whole-tree scan must check precommitted alternative forms too, not only `expected_name` — a candidate that shaped the name wrong *and* exposed it under an unlisted role has still put the (wrong) text in the tree, which is a different, more specific, finding than "nothing is there.""" exp = expectation( "Coro ا", alternative_forms={"name-drops-unresolved-codepoints": ["Coro "]}, ) verdict = classify(exp, [node("push button", "Coro ")]) self.assertEqual(verdict.verdict, "FAIL") self.assertNotEqual(verdict.prohibited_outcome, "absent-from-tree") self.assertEqual(verdict.observed_role, "push button") self.assertEqual(verdict.observed_name, "Coro ") def test_an_unlisted_role_carrying_the_visual_order_form_is_also_not_absent_from_tree(self): exp = expectation( "Allegro אבג con brio", visual_order_name="Allegro גבא con brio", ) verdict = classify(exp, [node("push button", "Allegro גבא con brio")]) self.assertEqual(verdict.verdict, "FAIL") self.assertNotEqual(verdict.prohibited_outcome, "absent-from-tree") self.assertEqual(verdict.observed_role, "push button") def test_an_unlisted_role_carrying_an_unrelated_name_is_still_absent_from_tree(self): """Mutation guard: the whole-tree scan must only match a name against `expected_name`/alternative forms/`visual_order_name` — a node with an unlisted role and completely unrelated text must not rescue the verdict away from absent-from-tree either.""" exp = expectation("Allegro") verdict = classify(exp, [node("push button", "something unrelated")]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "absent-from-tree") class MultipleFormsPerOutcome(unittest.TestCase): """O2: one outcome can carry more than one precommitted rendering — `round2-a11y-oracle` gives `name-is-shaped-glyphs` both a cluster-collapse form and a ligature presentation-form substitution for F-A. Either one observed must classify the same outcome.""" def _f_a_like_expectation(self): return expectation( "Allegro affettuoso — al fine", alternative_forms={ "name-is-shaped-glyphs": [ "Allegro afettuoso — al fne", "Allegro a\N{LATIN SMALL LIGATURE FF}ettuoso — al \N{LATIN SMALL LIGATURE FI}ne", ] }, ) def test_the_cluster_collapse_form_classifies(self): verdict = classify( self._f_a_like_expectation(), [node("text", "Allegro afettuoso — al fne")] ) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "name-is-shaped-glyphs") def test_the_presentation_form_classifies_the_same_outcome(self): presentation = ( "Allegro a\N{LATIN SMALL LIGATURE FF}ettuoso — al \N{LATIN SMALL LIGATURE FI}ne" ) verdict = classify(self._f_a_like_expectation(), [node("text", presentation)]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "name-is-shaped-glyphs") def test_a_third_string_matches_neither_form(self): """Mutation guard: confirms the two tests above are matching a specific listed string, not merely "anything different from expected_name" — if list matching degenerated to that, this would wrongly report name-is-shaped-glyphs too.""" verdict = classify( self._f_a_like_expectation(), [node("text", "something else entirely")], ) self.assertNotEqual(verdict.prohibited_outcome, "name-is-shaped-glyphs") class SubtreeScopedComposition(unittest.TestCase): """B1: composition is scored against one run subtree's own descendants, never the whole application flattened into one list. These are the two cases the coordinator reproduced against the pre-B1 flat classifier: - `[("canvas", "Coro "), ("text", "אבג")]` wrongly PASSed. - `[("text", "Coro "), ("text", "אבג"), ("label", "MyApp Window")]` wrongly FAILed, even though `label` is an accepted at-spi2 role that every real application's window frame carries, elsewhere in the tree. """ def _f_b_like_expectation(self): return expectation("Coro אבג") def test_a_prohibited_role_sibling_in_the_same_run_subtree_fails_with_the_role_named(self): """The false-PASS case (B1's first reproduction), expressed as a real tree: `canvas` and `text` are siblings under one run subtree — together they still spell out expected_name byte-for-byte, but a `canvas` contributed to it, which must FAIL, naming `canvas`, not PASS.""" run = node("frame", "", children=[node("canvas", "Coro "), node("text", "אבג")]) verdict = classify(self._f_b_like_expectation(), [run]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.observed_role, "canvas") self.assertEqual(verdict.observed_name, "Coro אבג") self.assertIn("canvas", verdict.reason) self.assertIn("otherwise-correct composition", verdict.reason) # Not one of the five PROHIBITED_OUTCOMES names — this is a §8.2 # role-vocabulary failure inside a composition, not a §8.3 name # transformation. self.assertIsNone(verdict.prohibited_outcome) def test_an_unrelated_accepted_role_node_elsewhere_does_not_poison_a_correct_composition(self): """The false-FAIL case (B1's second reproduction), expressed as a real tree: the run's own two text nodes are correctly grouped under their own subtree; an unrelated `label` (an *accepted* at-spi2 role — every real window frame carries one) sits elsewhere in the same application. The label must not be able to corrupt the run's own, otherwise-correct, composition into a FAIL.""" application = node( "frame", "", children=[ node("group", "", children=[node("text", "Coro "), node("text", "אבג")]), node("label", "MyApp Window"), ], ) verdict = classify(self._f_b_like_expectation(), [application]) self.assertEqual(verdict.verdict, "PASS") self.assertIsNone(verdict.prohibited_outcome) self.assertEqual(verdict.observed_name, "Coro אבג") def test_the_legitimate_accepted_role_split_still_passes_when_it_is_the_whole_tree(self): """Sanity companion to the two reproductions above: a run correctly split across two accepted-role text nodes, with nothing else in the tree at all, must still PASS — B1's fix must not have become so conservative that it stopped recognizing the ordinary case.""" run = node("paragraph", "", children=[node("text", "Coro "), node("text", "אבג")]) verdict = classify(self._f_b_like_expectation(), [run]) self.assertEqual(verdict.verdict, "PASS") self.assertIsNone(verdict.prohibited_outcome) def test_a_prohibited_contributor_is_named_even_when_a_correct_subtree_exists_elsewhere(self): """The composition scan must not let a PASS found in one subtree erase evidence of a bad contributor found in *another* — but it must still prefer reporting the PASS overall, since a candidate that gets it right anywhere in a legitimate run subtree has satisfied §8.1. This test pins the reverse: when NO subtree passes cleanly, the reported reason must name the actual bad contributor, not a generic mismatch.""" run = node( "frame", "", children=[node("canvas", "Coro "), node("text", "אבג")], ) verdict = classify(self._f_b_like_expectation(), [run]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.observed_role, "canvas") self.assertNotIn("no node or composition matched", verdict.reason) def test_a_composition_that_silently_dropped_a_prohibited_contributor_would_be_wrong(self): """Guards the specific failure mode B1 warns against: a subtree must not PASS by concatenating only its *accepted*-role contributors and ignoring a prohibited one. Here, dropping the `canvas` node would leave just `"אבג"`, which does not equal expected_name either — so this also confirms the concatenation includes every contributor's name, not a filtered subset, before the role check ever runs.""" run = node("frame", "", children=[node("canvas", "Coro "), node("text", "אבג")]) verdict = classify(self._f_b_like_expectation(), [run]) # If contributors had been filtered to accepted-only before # concatenating, the concat would be "אבג" (not expected_name), and # this subtree would be silently skipped rather than FAILed with a # named reason — falling through to a *weaker* diagnosis than the # sharp one B1 requires. self.assertEqual(verdict.verdict, "FAIL") self.assertIn("canvas", verdict.reason) class ContainerNamingDoesNotChangeBlame(unittest.TestCase): """An empty-named structural wrapper (a `frame` around the real contributors, exposing no name of its own) must never be blamed for a bad composition. It contributes zero bytes to the concatenation, so it cannot be what made the composition wrong; blaming it hides the actual offender — here, a *prohibited*-role `canvas` that carried half the run, which is the real §8.2 violation the report exists to name. Same tree content in all three shapes below; only the container's own name (or its absence) differs. All three must name `canvas`.""" def _f_b_like_expectation(self): return expectation("Coro אבג") def test_canvas_and_text_inside_an_unnamed_frame_blames_canvas(self): tree = node( "application", "p", children=[node("frame", "", children=[node("canvas", "Coro "), node("text", "אבג")])], ) verdict = classify(self._f_b_like_expectation(), [tree]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.observed_role, "canvas") def test_canvas_and_text_inside_a_named_frame_blames_canvas(self): tree = node( "application", "p", children=[ node("frame", "MyApp", children=[node("canvas", "Coro "), node("text", "אבג")]) ], ) verdict = classify(self._f_b_like_expectation(), [tree]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.observed_role, "canvas") def test_canvas_and_text_as_direct_siblings_blames_canvas(self): tree = node( "application", "p", children=[node("canvas", "Coro "), node("text", "אבג")], ) verdict = classify(self._f_b_like_expectation(), [tree]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.observed_role, "canvas") def test_an_unlisted_but_not_prohibited_contributor_is_still_named_when_no_prohibited_one_exists( self, ): """The weaker fallback branch stays reachable: with no prohibited-role contributor at all, an unlisted-role one is still named (not silently dropped just because it is the weaker case). Nested under an unnamed wrapper, so this test also isolates the empty-name exclusion on its own: with no prohibited contributor present, the "prefer prohibited" rule cannot be what saves this case from blaming the wrapper — only excluding the empty-named `frame` from the contributor set can. A mutation that deleted the empty-name exclusion (but kept the prohibited-preference) would wrongly blame `frame` here, even though the same mutation happens to survive the two `blames_canvas` tests above (where a prohibited `canvas` is also present and the preference rule alone rescues them).""" tree = node( "application", "p", children=[ node( "frame", "", children=[node("push button", "Coro "), node("text", "אבג")] ) ], ) verdict = classify(self._f_b_like_expectation(), [tree]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.observed_role, "push button") self.assertIn("neither accepted nor prohibited", verdict.reason) def test_a_prohibited_contributor_is_preferred_over_an_unlisted_one_listed_first(self): """Isolates the "prefer prohibited" rule specifically, with no empty-named node anywhere in the tree: an unlisted-role (`push button`) contributor is listed *before* a prohibited-role (`canvas`) one, both non-empty-named. Naming "the first non-accepted contributor" (no preference) would wrongly blame `push button`; only the explicit prohibited-preference blames `canvas`.""" tree = node( "frame", "", children=[node("push button", "Coro "), node("canvas", "אבג")], ) verdict = classify(self._f_b_like_expectation(), [tree]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.observed_role, "canvas") self.assertIn("prohibited", verdict.reason) def test_a_correct_split_run_nested_under_an_unnamed_container_still_passes(self): """Confirms excluding empty-named nodes does not disturb a legitimate PASS: the concatenation is unchanged whether the empty-named wrapper is included or excluded (it contributes zero bytes either way), so a correct split run nested under one must still pass.""" tree = node( "application", "p", children=[node("frame", "", children=[node("text", "Coro "), node("text", "אבג")])], ) verdict = classify(self._f_b_like_expectation(), [tree]) self.assertEqual(verdict.verdict, "PASS") class ExactNamePrecedence(unittest.TestCase): """C1: exact-name matching must evaluate every node before deciding, never return on the first match — an accepted-role node carrying expected_name wins regardless of where in the tree it sits, even when a prohibited-role node carrying the *same* exact name is listed first.""" @staticmethod def _forest(canvas_first): canvas = node("canvas", "Coro אבג") text = node("text", "Coro אבג") return [canvas, text] if canvas_first else [text, canvas] def test_prohibited_role_node_listed_first_still_passes(self): exp = expectation("Coro אבג") verdict = classify(exp, self._forest(canvas_first=True)) self.assertEqual(verdict.verdict, "PASS") self.assertEqual(verdict.observed_role, "text") self.assertIsNone(verdict.prohibited_outcome) def test_accepted_role_node_listed_first_still_passes(self): exp = expectation("Coro אבג") verdict = classify(exp, self._forest(canvas_first=False)) self.assertEqual(verdict.verdict, "PASS") self.assertEqual(verdict.observed_role, "text") self.assertIsNone(verdict.prohibited_outcome) def test_both_orderings_of_the_forest_produce_the_same_verdict(self): """The direct C1 reproduction: the coordinator measured opposite verdicts for the two orderings of this exact forest. Pinned here as one assertion comparing both `classify` calls, not two independently hand-written expectations that could each be individually wrong in the same direction.""" exp = expectation("Coro אבג") v_canvas_first = classify(exp, self._forest(canvas_first=True)) v_text_first = classify(exp, self._forest(canvas_first=False)) self.assertEqual(v_canvas_first.verdict, v_text_first.verdict) self.assertEqual(v_canvas_first.observed_role, v_text_first.observed_role) self.assertEqual(v_canvas_first.prohibited_outcome, v_text_first.prohibited_outcome) self.assertEqual(v_canvas_first.verdict, "PASS") def test_without_any_accepted_role_match_the_prohibited_one_still_fails(self): """Mutation guard: confirms the PASSes above happen *because* an accepted-role match exists, not because exact-name matching became unconditional PASS — with only the prohibited-role node present, this must still FAIL.""" exp = expectation("Coro אבג") verdict = classify(exp, [node("canvas", "Coro אבג")]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.observed_role, "canvas") class SubtreeScopedDiagnoses(unittest.TestCase): """C2: the alternative-form and visual-order diagnoses are scored per subtree, exactly like composition (B1) — an unrelated accepted-role node elsewhere in the application (e.g. a window `label`) must not poison a legitimate subtree's diagnosis into a generic mismatch.""" def test_visual_order_trap_is_found_despite_an_unrelated_window_label(self): """The direct C2 reproduction: F-D's designed visual-order trap, with an unrelated `label` elsewhere in the application.""" exp = expectation( "Allegro אבג con brio", visual_order_name="Allegro גבא con brio", ) application = node( "application", "p", children=[ node("label", "MyApp Window"), node( "frame", "", children=[ node("text", "Allegro "), node("text", "גבא"), node("text", " con brio"), ], ), ], ) verdict = classify(exp, [application]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, VISUAL_ORDER_TRAP) self.assertNotIn("no node or composition matched", verdict.reason) def test_alternative_form_composition_is_found_despite_an_unrelated_window_label(self): """The same poisoning bug, for an alternative-form composition (not visual-order) — the ligature-collapse form split across two text nodes, so this exercises the *subtree-concatenation* alt-form path specifically, not the already-order-independent single-node one.""" exp = expectation( "Allegro affettuoso — al fine", alternative_forms={"name-is-shaped-glyphs": ["Allegro afettuoso — al fne"]}, ) application = node( "application", "p", children=[ node("label", "MyApp Window"), node( "frame", "", children=[ node("text", "Allegro afettuoso — al "), node("text", "fne"), ], ), ], ) verdict = classify(exp, [application]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "name-is-shaped-glyphs") self.assertNotIn("no node or composition matched", verdict.reason) def test_without_the_unrelated_label_the_same_composition_still_matches(self): """Mutation guard: confirms the two tests above are really about the label not poisoning the result, not about some other property of the tree shape — remove the label and the same diagnosis must still fire.""" exp = expectation( "Allegro אבג con brio", visual_order_name="Allegro גבא con brio", ) frame = node( "frame", "", children=[node("text", "Allegro "), node("text", "גבא"), node("text", " con brio")], ) verdict = classify(exp, [frame]) self.assertEqual(verdict.prohibited_outcome, VISUAL_ORDER_TRAP) class UnlistedRoleComposition(unittest.TestCase): """C3: text composed across two or more unlisted-role descendants must not be misreported as absent-from-tree — the same per-subtree composition scoring applied to roles in neither `accepted_roles` nor `prohibited_roles`.""" def test_two_unlisted_role_nodes_composing_the_exact_name_is_not_absent(self): """The direct C3 reproduction: two `push button` nodes whose concatenation is the run's exact text.""" exp = expectation("Coro אבג") application = node( "application", "p", children=[node("push button", "Coro "), node("push button", "אבג")], ) verdict = classify(exp, [application]) self.assertEqual(verdict.verdict, "FAIL") self.assertNotEqual(verdict.prohibited_outcome, "absent-from-tree") self.assertEqual(verdict.observed_name, "Coro אבג") def test_unlisted_role_composition_with_unrelated_text_is_still_absent(self): """The composition-scan analogue of the single-node absent-from-tree guard: two unlisted-role nodes whose concatenation is *not* the run's text, an alternative form, or visual_order_name, must still classify as genuine absence — the scan must not over-fire just because *some* unlisted-role composition exists.""" exp = expectation("Coro אבג") application = node( "application", "p", children=[node("push button", "something"), node("push button", "unrelated")], ) verdict = classify(exp, [application]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "absent-from-tree") class AbsentFromTreeVsMisorderedFragments(unittest.TestCase): """User ruling, following C3: a permutation probe found that reversing the same two contributors gave `absent-from-tree` under unlisted roles but a plain generic mismatch under accepted roles — two different named outcomes for the same underlying defect (contributors in the wrong order). Three cases, side by side, each asserting its own distinct outcome, so a future change cannot silently re-merge them. **Behaviour change, reported explicitly (not adjusted quietly), from the reorder that moved source-bearing detection to run before absence/ name-empty (the very next ruling in this same sequence):** case 1 below used to assert "unchanged, a generic composition FAIL" — that was true only because the fragment scan, at the time, ran solely inside the `flat_candidates`-empty branch and so never even looked at accepted-role contributors. Once source-bearing detection (fragments included) was unified to run over *every* role unconditionally, the identical reversed-`text` case is now *also* caught by the fragment scan, with a more specific message than the old generic mismatch — which is the intended, uniform consequence of "misordered source fragments -> composition/role failure, never absence" applying without a role exception. Nothing about *this* file's tests silently changed; the updated assertion below is that report. 1. reversed **accepted**-role contributors — a role/composition FAIL (fragment-scan diagnosis, naming the fragments), never `absent-from-tree`; 2. reversed **unlisted**-role contributors — the original fix: a role/composition FAIL, never `absent-from-tree`; 3. the true-absence control — nothing resembling the run's text anywhere — still reaches `absent-from-tree`, proving the fix narrowed the bug without making the outcome unreachable. """ def test_reversed_accepted_role_contributors_are_a_composition_failure_not_absence(self): """Behaviour change (see class docstring): this used to assert a generic mismatch (`prohibited_outcome=None`, "no node or composition matched..."). It now asserts the fragment-scan diagnosis — still `prohibited_outcome=None` (a §8.2 role/composition failure, not a §8.3 PROHIBITED_OUTCOMES name), but a more specific reason, because the fragment scan is no longer gated to unlisted roles only.""" exp = expectation("Coro אבג") run = node("frame", "", children=[node("text", "אבג"), node("text", "Coro ")]) verdict = classify(exp, [run]) self.assertEqual(verdict.verdict, "FAIL") self.assertIsNone(verdict.prohibited_outcome) self.assertNotEqual(verdict.prohibited_outcome, "absent-from-tree") self.assertIn("role/composition failure", verdict.reason) def test_reversed_unlisted_role_contributors_are_a_role_failure_not_absence(self): """Item 2, the fix itself — the coordinator's exact reproduction: the identical shape, under unlisted roles, must NOT be `absent-from-tree`. The text is genuinely present; only the order (and the role) is wrong.""" exp = expectation("Coro אבג") application = node( "application", "p", children=[node("push button", "אבג"), node("push button", "Coro ")], ) verdict = classify(exp, [application]) self.assertEqual(verdict.verdict, "FAIL") self.assertNotEqual(verdict.prohibited_outcome, "absent-from-tree") # Not a §8.3 PROHIBITED_OUTCOMES name either — this is the §8.2 # role/composition failure category, the same as C3's other cases. self.assertIsNone(verdict.prohibited_outcome) def test_true_absence_is_still_reachable(self): """Item 1, the control: with nothing resembling the run's text anywhere, `absent-from-tree` must still fire — proving the fix narrowed the bug rather than making the outcome unreachable.""" exp = expectation("Coro אבג") verdict = classify(exp, []) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "absent-from-tree") class SourceBearingFragmentGuards(unittest.TestCase): """Prove `_is_source_bearing_fragment` (C3's fix) is not loose enough to make `absent-from-tree` unreachable in practice — the user's own stated concern: "a rule loose enough that an application's ordinary window label counts as a fragment would make absent-from-tree unreachable in practice, which is a worse failure than the one being fixed." Each guard below is the specific scenario that concern describes.""" def test_a_single_shared_character_is_not_a_fragment(self): """A one-character coincidence — "o" appears in both "Coro" and almost any ordinary English text — must not count; this is exactly the case the length-2 floor exists to exclude.""" self.assertFalse(_is_source_bearing_fragment("o", {"Coro אבג"})) def test_an_ordinary_window_label_is_not_a_fragment(self): """The user's own example, direct: a whole, realistic window title is longer than (and unrelated to) the run's short text, so it can never be a literal substring of it.""" self.assertFalse(_is_source_bearing_fragment("MyApp Window", {"Coro אבג"})) def test_a_whitespace_only_name_is_not_a_fragment(self): self.assertFalse(_is_source_bearing_fragment(" ", {"Coro אבג"})) def test_an_empty_name_is_not_a_fragment(self): self.assertFalse(_is_source_bearing_fragment("", {"Coro אבג"})) def test_a_two_character_real_fragment_does_count(self): """Anchors the floor at exactly two characters, not three or more — confirms the guards above are testing the length-1 boundary specifically, not merely "short strings never match".""" self.assertTrue(_is_source_bearing_fragment("בג", {"Coro אבג"})) def test_an_ordinary_window_label_does_not_rescue_a_tree_from_absence_end_to_end(self): """The end-to-end version of the guard above: a real, unrelated, realistic window label under an unlisted role, with nothing else in the tree, must still classify as absent-from-tree.""" exp = expectation("Coro אבג") application = node( "application", "p", # An unlisted role, not "label" (which is an accepted at-spi2 # role and would exit the C3 branch this test is about via the # ordinary accepted-role path instead). children=[node("push button", "MyApp Window")], ) verdict = classify(exp, [application]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "absent-from-tree") class FCTwoSegmentComposition(unittest.TestCase): """D1/D2 regression group, required together — all four reproduce against F-C's real shape (`"Coro "` + `"ا"`, with `"Coro "` also being F-C's own precommitted `name-drops-unresolved-codepoints` form) and must all hold simultaneously: unlisted "ا" for F-C -> NOT absent-from-tree unrelated single ASCII "o" -> still NOT source-bearing F-C accepted split ["Coro ", "ا"] -> PASS F-C lone accepted node named exactly "Coro " -> still name-drops-unresolved-codepoints The first two prove D1 (an unresolved segment can be a single character, which the length-2 substring rule alone cannot catch, but the coincidence guard must still hold); the last two prove D2 (a legitimate two-node split now PASSes despite the first segment alone matching a precommitted alternative form, and the single-node case — which has no composition to find — still names that outcome exactly as before). """ def _f_c_like_expectation(self): return expectation( "Coro ا", alternative_forms={"name-drops-unresolved-codepoints": ["Coro "]}, source_atoms=["Coro ", "ا"], ) def test_unlisted_role_carrying_f_cs_unresolved_segment_is_not_absent_from_tree(self): """D1, the reported finding: F-C's unresolved segment `ا` is a single character — below `_is_source_bearing_fragment`'s length-2 floor — but it is still a precommitted `source_atoms` entry, so a node carrying it under an unlisted role must be a role/composition failure, never absence. §8.3: "the accessibility tree carries the text, not the ink".""" exp = self._f_c_like_expectation() verdict = classify(exp, [node("push button", "ا")]) self.assertEqual(verdict.verdict, "FAIL") self.assertNotEqual(verdict.prohibited_outcome, "absent-from-tree") self.assertEqual(verdict.observed_role, "push button") def test_an_unrelated_single_ascii_character_is_still_not_source_bearing(self): """Mutation guard, D1: confirms the atom-matching path is additive and precommitted, not a blanket "any single character counts" rule — an unrelated stray `"o"` (present in `"Coro"` only by coincidence, and not a `source_atoms` entry) must still not rescue the tree from absence, exactly as `_is_source_bearing_fragment`'s own coincidence guard already requires on its own.""" exp = self._f_c_like_expectation() verdict = classify(exp, [node("push button", "o")]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "absent-from-tree") def test_the_legitimate_two_node_split_passes(self): """D2, the fix itself — the coordinator's exact reproduction: two ACCEPTED-role text nodes, one per direction run (exactly what §8.1 permits: "a tree that exposes one text node per direction run is not wrong"), must PASS even though the first segment alone happens to equal F-C's own precommitted `name-drops-unresolved-codepoints` form.""" exp = self._f_c_like_expectation() run = node("frame", "", children=[node("text", "Coro "), node("text", "ا")]) verdict = classify(exp, [run]) self.assertEqual(verdict.verdict, "PASS") self.assertIsNone(verdict.prohibited_outcome) def test_a_lone_accepted_node_named_exactly_coro_is_still_the_named_outcome(self): """The required guard proving D2's fix did not simply disable the alternative-form diagnosis: with no second node, there is no composition to find, so a single `text:"Coro "` node must still be classified by name, exactly as before D2.""" exp = self._f_c_like_expectation() verdict = classify(exp, [node("text", "Coro ")]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "name-drops-unresolved-codepoints") class AbsentFromTreeVsNameEmpty(unittest.TestCase): """User ruling: `absent-from-tree` must be gated on *no source-bearing text existing anywhere*, not on *no accepted-or-prohibited-role candidate existing* — the latter gate is nearly always false for a real application (a window `label` alone defeats it), which made `absent-from-tree` — §8.3's own words, "the one this check will most likely actually catch" — effectively unreachable in practice. The pinned precedence: 1. Source-bearing text or a precommitted form present anywhere -> classify its name/role/composition outcome. This runs first, over the whole forest, any role — before any absence/empty-name determination. 2. Otherwise, `name-empty` only when both hold: at least one **accepted**-role candidate node exists, and every accepted-or-prohibited-role candidate's name is empty. 3. Every other no-source-bearing case -> `absent-from-tree`. The intended taxonomy, and the required regression lock: four cases, side by side, so a future change cannot re-merge them. unrelated UI text only -> absent-from-tree empty prohibited canvas only -> absent-from-tree empty accepted text/label node -> name-empty misordered source fragments -> composition/role failure, never absence The distinction being preserved: `name-empty` means an attempted static-text exposure without a name; `absent-from-tree` covers drawing-only or unrelated trees. A prohibited-role empty node is not "wearing a role" in §8.3's sense — it is the draw-and-stop case. """ def test_unrelated_ui_text_only_is_absent_from_tree(self): """The coordinator's exact reproduction: ordinary application chrome — a button, a window label — none of it related to the run. Every real application has role-listed nodes like this, which is exactly why the old "no candidate exists anywhere" gate made `absent-from-tree` nearly unreachable.""" exp = expectation("Coro אבג") application = node( "application", "p", children=[node("push button", "Save"), node("label", "MyApp Window")], ) verdict = classify(exp, [application]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "absent-from-tree") def test_empty_prohibited_canvas_only_is_absent_from_tree(self): """The case that matters most (§8.3's own words): a toolkit that drew to a canvas and stopped. A `canvas` node exposing no name is not "wearing a role" in §8.3's name-empty sense — with no accepted-role candidate anywhere in the tree, this is the draw-and-stop case, `absent-from-tree`, not `name-empty`.""" exp = expectation("Coro אבג") application = node("frame", "MyApp", children=[node("canvas", "")]) verdict = classify(exp, [application]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "absent-from-tree") def test_empty_accepted_text_node_is_name_empty(self): """The companion case: an *accepted*-role node attempting to expose static text, but with no name — this is the genuine name-empty case, "absence wearing a role".""" exp = expectation("Coro אבג") application = node("frame", "MyApp", children=[node("label", "")]) verdict = classify(exp, [application]) self.assertEqual(verdict.verdict, "FAIL") self.assertEqual(verdict.prohibited_outcome, "name-empty") def test_misordered_source_fragments_are_a_composition_failure_not_absence_or_empty(self): """The fourth leg: fragments of the run's actual text, present but in the wrong order, must never be classified as absence or name-empty — a composition/role failure, per item 1's precedence over items 2 and 3.""" exp = expectation("Coro אבג") application = node( "application", "p", children=[node("push button", "אבג"), node("push button", "Coro ")], ) verdict = classify(exp, [application]) self.assertEqual(verdict.verdict, "FAIL") self.assertNotEqual(verdict.prohibited_outcome, "absent-from-tree") self.assertNotEqual(verdict.prohibited_outcome, "name-empty") def test_an_accepted_node_with_real_text_alongside_an_empty_one_is_not_name_empty(self): """Mutation guard: `name-empty` requires *every* candidate to be empty, not merely *an* accepted-role candidate existing — an accepted node with real (if unrelated) text sitting alongside an empty one must not trigger name-empty, since not every text-candidate node is actually empty.""" exp = expectation("Coro אבג") application = node( "frame", "MyApp", children=[node("label", "Random"), node("label", "")], ) verdict = classify(exp, [application]) self.assertEqual(verdict.verdict, "FAIL") self.assertNotEqual(verdict.prohibited_outcome, "name-empty") class ExpectationsFileValidation(unittest.TestCase): """O1/B2: the loader must fail closed on a malformed or stale oracle before any live AT-SPI readback — never a FAIL, always a usage error that the caller (`run_check5`) turns into exit 2.""" VALID_DIGEST = "deadbeef" * 8 # a plausible-looking 64-hex-char sha256 @staticmethod def _fixture(fixture_id, name, alternative_forms=None, source_atoms=None): return { "fixture_id": fixture_id, "expected_name": name, "expected_name_hex": name.encode("utf-8").hex(), "expected_name_byte_len": len(name.encode("utf-8")), "accepted_roles": ["label", "static", "text", "paragraph"], "prohibited_roles": ["image", "canvas", "filler", "panel", "unknown"], # Defaults to one atom equal to the whole name (trivially # satisfies the join-equals-name invariant) — tests of other # fields don't need more than one segment. "source_atoms": source_atoms if source_atoms is not None else [name], "alternative_forms": alternative_forms or {}, } def _valid_file(self): """A fully self-consistent, five-fixture file — every B2/O1/D1 check passes against this by construction. Each test below mutates exactly one thing away from it, so a raised error is attributable to the one defect under test rather than an incidental other one.""" return { "contract": "spec/CONTRACT_EDITOR_T4_SPIKE.md pin 13", "recipe": "spikes/editor-toolkit/ROUND2_TEXT_RECIPE.md §8", "platform": PLATFORM, "source_fixtures_digest": self.VALID_DIGEST, "fixtures": [ self._fixture("F-A", "Allegro affettuoso — al fine"), self._fixture("F-B", "Coro אבג", source_atoms=["Coro ", "אבג"]), self._fixture("F-C", "Coro ا", source_atoms=["Coro ", "ا"]), self._fixture( "F-D", "Allegro אבג con brio", source_atoms=["Allegro ", "אבג", " con brio"] ), self._fixture("F-E", "Café"), ], } def _validate(self, file): validate_expectations_file( file, expected_platform=PLATFORM, expected_source_digest=self.VALID_DIGEST ) # ---- baseline ---- def test_a_fully_valid_file_is_accepted(self): self._validate(self._valid_file()) # must not raise # ---- platform ---- def test_a_wrong_platform_is_refused(self): bad = self._valid_file() bad["platform"] = "aria" with self.assertRaises(ValueError) as ctx: self._validate(bad) msg = str(ctx.exception) self.assertIn("aria", msg) self.assertIn(PLATFORM, msg) # ---- source_fixtures_digest (B2) ---- def test_a_stale_source_digest_is_refused(self): bad = self._valid_file() bad["source_fixtures_digest"] = "stale" + "0" * 60 with self.assertRaises(ValueError) as ctx: self._validate(bad) msg = str(ctx.exception) self.assertIn("stale", msg) self.assertIn(self.VALID_DIGEST, msg) # ---- fixture id completeness/uniqueness ---- def test_a_duplicate_fixture_id_is_refused(self): bad = self._valid_file() bad["fixtures"].append(self._fixture("F-A", "duplicate")) with self.assertRaises(ValueError) as ctx: self._validate(bad) self.assertIn("duplicate", str(ctx.exception).lower()) self.assertIn("F-A", str(ctx.exception)) def test_a_missing_fixture_is_refused(self): bad = self._valid_file() bad["fixtures"] = [f for f in bad["fixtures"] if f["fixture_id"] != "F-E"] with self.assertRaises(ValueError) as ctx: self._validate(bad) self.assertIn("F-E", str(ctx.exception)) def test_an_extra_fixture_id_is_refused(self): bad = self._valid_file() bad["fixtures"].append(self._fixture("F-Z", "unexpected")) with self.assertRaises(ValueError) as ctx: self._validate(bad) self.assertIn("F-Z", str(ctx.exception)) def test_the_expected_fixture_id_set_is_exactly_f_a_through_f_e(self): """Anchors `EXPECTED_FIXTURE_IDS` itself, independent of `validate_expectations_file` — if this constant silently gained or lost an id, the two tests above could pass against the wrong set.""" self.assertEqual(EXPECTED_FIXTURE_IDS, frozenset({"F-A", "F-B", "F-C", "F-D", "F-E"})) # ---- expected_name / expected_name_hex / expected_name_byte_len self-consistency ---- def test_a_wrong_hex_is_refused(self): bad = self._valid_file() bad["fixtures"][0]["expected_name_hex"] = "ff" * 10 with self.assertRaises(ValueError) as ctx: self._validate(bad) msg = str(ctx.exception) self.assertIn("F-A", msg) self.assertIn("hex", msg) def test_an_uppercase_hex_is_refused(self): """§8.1 specifically requires *lowercase* hex — an otherwise-correct but uppercase rendering must still be refused, not accepted as "close enough".""" bad = self._valid_file() bad["fixtures"][0]["expected_name_hex"] = bad["fixtures"][0]["expected_name_hex"].upper() with self.assertRaises(ValueError) as ctx: self._validate(bad) self.assertIn("hex", str(ctx.exception)) def test_a_wrong_byte_length_is_refused(self): bad = self._valid_file() bad["fixtures"][0]["expected_name_byte_len"] += 1 with self.assertRaises(ValueError) as ctx: self._validate(bad) msg = str(ctx.exception) self.assertIn("F-A", msg) self.assertIn("byte_len", msg) def test_a_correct_hex_and_length_pair_is_not_refused(self): """Mutation guard: confirms the two tests above are checking the actual computed hex/length, not merely "is a string of digits" or some other weaker property.""" self._validate(self._valid_file()) # must not raise # ---- O1: cross-outcome collision (unchanged, folded into this entry point) ---- def test_a_colliding_file_is_refused_naming_the_fixture_and_both_outcomes(self): bad = self._valid_file() bad["fixtures"][2]["alternative_forms"] = { # F-C "name-drops-unresolved-codepoints": ["Coro "], "name-is-shaped-glyphs": ["Coro "], } with self.assertRaises(ValueError) as ctx: self._validate(bad) msg = str(ctx.exception) self.assertIn("F-C", msg) self.assertIn("name-drops-unresolved-codepoints", msg) self.assertIn("name-is-shaped-glyphs", msg) def test_a_collision_between_list_entries_is_also_refused(self): """The collision can be between any entry in one outcome's list and any entry in another's, not just single-value outcomes.""" bad = self._valid_file() bad["fixtures"][0]["alternative_forms"] = { # F-A "name-normalized": ["alpha", "shared"], "name-is-shaped-glyphs": ["beta", "shared"], } with self.assertRaises(ValueError) as ctx: self._validate(bad) self.assertIn("shared", str(ctx.exception)) def test_a_repeated_value_within_the_same_outcome_is_not_a_collision(self): """Mutation guard: if the collision check fired on *any* repeated value rather than specifically a *cross-outcome* one, this would wrongly raise — two entries in one outcome's own list happening to repeat is not the ambiguity O1 refuses.""" fine = self._valid_file() fine["fixtures"][0]["alternative_forms"] = {"name-normalized": ["same", "same"]} self._validate(fine) # must not raise def test_the_real_committed_file_is_valid(self): """Grounds the synthetic tests above in the actual generated artifact — the file `run_check5` will really load. Uses the file's own `platform`/`source_fixtures_digest` as the "expected" values (this is the one place a real digest isn't known statically), so this test is really only exercising the fixture-id/name-consistency/ collision checks against real data, not the digest-mismatch check.""" import json import os path = os.path.join( os.path.dirname(__file__), "..", "round2-a11y-oracle", "a11y_expectations.json" ) if not os.path.exists(path): self.skipTest(f"{path} absent — run the round2-a11y-oracle generator first") with open(path, "r", encoding="utf-8") as f: real_file = json.load(f) validate_expectations_file( real_file, expected_platform=real_file.get("platform"), expected_source_digest=real_file.get("source_fixtures_digest"), ) # must not raise if __name__ == "__main__": unittest.main()