Zum Inhalt

tests/test_scorecard_e2e.py

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
import math

from touchstone.results import EvalResponse
from touchstone.scorecard import _perf_summary


def _resp(*, e2e_s, ok=True, is_cold_start=False):
    return EvalResponse(
        pack_id="demo",
        pack_version=1,
        machine="M",
        model="m1",
        quant="n/a",
        engine="ollama",
        engine_version="0",
        variant="none",
        category="A",
        prompt_id="A1",
        repeat=0,
        response_text="x",
        content_empty=False,
        ttft_s=0.2,
        decode_tps=12.0,
        prefill_tps=100.0,
        e2e_s=e2e_s,
        prompt_tokens=10,
        completion_tokens=20,
        is_cold_start=is_cold_start,
        power_source="ac",
        peak_rss_mb=None,
        sys_used_mb=18000.0,
        mem_pressure_max="normal",
        throttled=False,
        ok=ok,
        error="",
        seed=42,
        t_start=0.0,
        t_end=1.0,
    )


def test_perf_summary_includes_e2e_med():
    resps = [_resp(e2e_s=1.0, ok=True), _resp(e2e_s=3.0, ok=True)]
    p = _perf_summary(resps)
    assert p["e2e_med"] == 2.0


def test_perf_summary_e2e_med_ignores_nan_e2e():
    resps = [_resp(e2e_s=2.0, ok=True), _resp(e2e_s=math.nan, ok=True)]
    p = _perf_summary(resps)
    assert p["e2e_med"] == 2.0