Zum Inhalt

tests/test_eval_web.py

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
import json
from types import SimpleNamespace

from touchstone import events as ev
from touchstone.cli import _eval_event_writers, _run_event_writers


def _cell():
    return SimpleNamespace(
        model=SimpleNamespace(id="m"),
        variant=SimpleNamespace(id="v"),
        category=SimpleNamespace(id="A"),
        prompt=SimpleNamespace(id="p"),
        repeat=0,
    )


def _resp():
    return SimpleNamespace(
        model="m",
        variant="v",
        prompt_id="p",
        repeat=0,
        ok=True,
        ttft_s=0.3,
        e2e_s=1.2,
        decode_tps=5.0,
        completion_tokens=7,
        content_empty=False,
        error="",
        reasoning_chars=12,
    )


def test_eval_event_writers_emit_well_formed_events(tmp_path):
    path = tmp_path / "events.jsonl"
    on_run_start, on_cell_start, on_cell_done, _on_preflight, run_done = _eval_event_writers(path)
    on_run_start(3)
    on_cell_start(0, _cell())
    on_cell_done(0, _resp())
    run_done([_resp(), _resp()])

    parsed = [ev.parse_line(ln) for ln in path.read_text(encoding="utf-8").splitlines()]
    parsed = [p for p in parsed if p is not None]
    assert [p["type"] for p in parsed] == ["run_start", "cell_start", "cell_done", "run_done"]
    assert parsed[0]["total"] == 3
    assert parsed[1]["model"] == "m" and parsed[1]["prompt_id"] == "p"
    assert parsed[2]["ok"] is True and parsed[2]["e2e_s"] == 1.2
    assert parsed[2]["reasoning_chars"] == 12
    assert parsed[3]["ok"] == 2  # both responses ok


def _run_cell():
    return SimpleNamespace(model=SimpleNamespace(id="m"), scenario="chat", target_ctx=0)


def _run_rec(**kw):
    base = dict(
        model="m",
        scenario="chat",
        target_ctx=0,
        ok=True,
        ttft_s=0.2,
        e2e_s=1.0,
        decode_tps=50.0,
        completion_tokens=10,
        error="",
    )
    base.update(kw)
    return SimpleNamespace(**base)


def test_eval_event_writers_emits_preflight(tmp_path):
    from touchstone.preflight import PreflightResult

    ep = tmp_path / "events.jsonl"
    _, _, _, on_preflight, run_done = _eval_event_writers(ep, append=False)
    on_preflight([PreflightResult("gemma", "reasoning_only", 0, 1400, "nur Reasoning")])
    run_done([])
    types = [json.loads(line)["type"] for line in ep.read_text().splitlines()]
    assert "preflight" in types


def test_run_event_writers_emit_well_formed_events(tmp_path):
    path = tmp_path / "events.jsonl"
    on_run_start, on_cell_start, on_cell_done, run_done = _run_event_writers(path)
    on_run_start(2)
    on_cell_start(0, _run_cell())
    on_cell_done(0, _run_rec())
    run_done([_run_rec(), _run_rec(ok=False)])

    parsed = [ev.parse_line(ln) for ln in path.read_text(encoding="utf-8").splitlines()]
    parsed = [p for p in parsed if p is not None]
    assert [p["type"] for p in parsed] == ["run_start", "cell_start", "cell_done", "run_done"]
    assert parsed[1]["variant"] == "native" and parsed[1]["prompt_id"] == "chat"
    assert parsed[2]["ok"] is True and parsed[2]["decode_tps"] == 50.0
    assert parsed[3]["ok"] == 1  # one ok, one failed