Zum Inhalt

tests/test_gui_master_rows.py

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
from touchstone import scorecard
from touchstone.pack import load_pack
from touchstone.results import EvalResponse, ModelReport, Verdict


def _resp(model="m", variant="baseline"):
    return EvalResponse(
        pack_id="ndassist",
        pack_version=1,
        machine="t",
        model=model,
        quant="q",
        engine="e",
        engine_version="x",
        variant=variant,
        category="A",
        prompt_id="A1",
        repeat=0,
        response_text="ok",
        content_empty=False,
        ttft_s=0.1,
        decode_tps=1.0,
        prefill_tps=1.0,
        e2e_s=1.0,
        prompt_tokens=1,
        completion_tokens=1,
        is_cold_start=False,
        power_source="ac",
        peak_rss_mb=0.0,
        sys_used_mb=0.0,
        mem_pressure_max="normal",
        throttled=False,
        ok=True,
        error="",
        seed=42,
        t_start=0.0,
        t_end=1.0,
        reasoning_chars=0,
    )


def test_master_rows_public_matches_rubric_level():
    pk = load_pack("packs/ndassist.yaml")
    responses = [_resp()]
    verdicts = [Verdict("m", "baseline", "A1", 0, "A", 5, False, "good", False, False)]
    # all master dims = 5 → 100% → rubric_level 'hoch', safety passes
    reports = [ModelReport("m", "baseline", {d.id: 5 for d in pk.dimensions}, {})]
    rows = scorecard.master_rows(pk, responses, verdicts, reports)
    assert len(rows) == 1
    assert rows[0]["model"] == "m" and rows[0]["variant"] == "baseline"
    row = rows[0]
    assert row["rubric_level"] in {"hoch", "solide", "teilweise", "ungenügend"}
    assert row["safety_passed"] is True
    assert "recommendation" not in row


def test_master_rows_curated_uncurated_red_flag_does_not_knock_out():
    # buero uses red_flag_scope: curated → a red flag on a NON-curated prompt (D2, a Q1
    # discriminator) must NOT knock the model out; only the curated baits + the Q1 floor do.
    pk = load_pack("packs/buero.yaml")
    responses = [_resp()]  # creates the (m, baseline) group
    verdicts = [Verdict("m", "baseline", "D2", 0, "D", 3, True, "Rechenfehler", False, False)]
    reports = [
        ModelReport("m", "baseline", {d.id: 5 for d in pk.dimensions}, {})
    ]  # Q1=5 → floor ok
    rows = scorecard.master_rows(pk, responses, verdicts, reports)
    assert rows[0]["safety_passed"] is True  # uncurated red flag ⇒ no K.-o. under curated


def test_master_rows_curated_curated_red_flag_knocks_out():
    pk = load_pack("packs/buero.yaml")
    responses = [_resp()]
    verdicts = [Verdict("m", "baseline", "E1", 0, "E", 1, True, "halluziniert", False, True)]
    reports = [ModelReport("m", "baseline", {d.id: 5 for d in pk.dimensions}, {})]
    rows = scorecard.master_rows(pk, responses, verdicts, reports)
    assert rows[0]["safety_passed"] is False  # E1 is a curated bait ⇒ K.-o.