byeongwook.choiandClaude Fable 5.1
11ae3629b2
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
204 lines
7.7 KiB
Python
204 lines
7.7 KiB
Python
"""LLM 백엔드 검증 — file 큐(키 불필요 입구)와 재시도/백오프."""
|
|
import json
|
|
import urllib.error
|
|
|
|
import pytest
|
|
|
|
from summarize import jobs
|
|
from summarize.llm_client import (
|
|
FileQueueLLM,
|
|
PendingResponse,
|
|
_extract_json,
|
|
_meta_from_prompt,
|
|
create_llm,
|
|
job_id,
|
|
)
|
|
|
|
UNIT_PROMPT_SAMPLE = """[작업] extract_chunks
|
|
[프로그램] ZFIR10070 — 총계정원장 조회
|
|
[단위] unit_id: ZFIR10070#ZFIR10070_F01#FORM#SELECT_DATA
|
|
unit_type: FORM, name: SELECT_DATA
|
|
[창] 이 단위는 길어서 3개 창으로 나눠 보여준다. 지금은 2번째 창 (L10-L20).
|
|
"""
|
|
|
|
|
|
def test_job_id_is_stable_and_content_addressed():
|
|
a = job_id("sys", "user")
|
|
assert a == job_id("sys", "user")
|
|
assert a != job_id("sys", "user2")
|
|
|
|
|
|
def test_meta_parsed_from_prompt():
|
|
m = _meta_from_prompt(UNIT_PROMPT_SAMPLE)
|
|
assert m["task"] == "extract_chunks"
|
|
assert m["program"] == "ZFIR10070"
|
|
assert m["unit_id"] == "ZFIR10070#ZFIR10070_F01#FORM#SELECT_DATA"
|
|
assert m["unit_type"] == "FORM"
|
|
assert m["window"] == "2"
|
|
|
|
|
|
def test_file_backend_emits_prompt_then_reads_answer(tmp_path):
|
|
llm = FileQueueLLM(tmp_path)
|
|
|
|
# 1) 응답이 없으면 프롬프트를 내놓고 PendingResponse
|
|
with pytest.raises(PendingResponse) as ei:
|
|
llm.complete_json("sys", UNIT_PROMPT_SAMPLE)
|
|
jid = ei.value.job_id
|
|
assert llm.prompt_path(jid).exists()
|
|
prompt_text = llm.prompt_path(jid).read_text(encoding="utf-8")
|
|
assert "## SYSTEM" in prompt_text and "extract_chunks" in prompt_text
|
|
# 인덱스에 메타가 기록된다
|
|
rows = [json.loads(l) for l in (tmp_path / "index.jsonl").read_text(encoding="utf-8").splitlines()]
|
|
assert rows[0]["job_id"] == jid and rows[0]["program"] == "ZFIR10070"
|
|
|
|
# 2) 응답을 채우면 그걸 돌려준다
|
|
llm.response_path(jid).write_text('{"unit_purpose_ko": "x", "chunks": []}', encoding="utf-8")
|
|
out = llm.complete_json("sys", UNIT_PROMPT_SAMPLE)
|
|
assert out == {"unit_purpose_ko": "x", "chunks": []}
|
|
assert llm.usage["calls"] == 1
|
|
|
|
|
|
def test_file_backend_does_not_duplicate_index_rows(tmp_path):
|
|
llm = FileQueueLLM(tmp_path)
|
|
for _ in range(3):
|
|
with pytest.raises(PendingResponse):
|
|
llm.complete_json("sys", UNIT_PROMPT_SAMPLE)
|
|
lines = (tmp_path / "index.jsonl").read_text(encoding="utf-8").strip().splitlines()
|
|
assert len(lines) == 1
|
|
|
|
|
|
def test_create_llm_backends(tmp_path):
|
|
from summarize.llm_client import FakeLLM
|
|
|
|
assert isinstance(create_llm(backend="fake"), FakeLLM)
|
|
assert isinstance(create_llm(fake=True), FakeLLM) # 하위호환
|
|
assert isinstance(create_llm(backend="file", jobs_dir=tmp_path), FileQueueLLM)
|
|
with pytest.raises(ValueError):
|
|
create_llm(backend="없는백엔드")
|
|
|
|
|
|
def test_extract_json_survives_code_fence():
|
|
assert _extract_json('```json\n{"a": 1}\n```') == {"a": 1}
|
|
assert _extract_json('설명입니다\n{"a": 2}\n끝') == {"a": 2}
|
|
|
|
|
|
def test_api_backend_retries_on_429(monkeypatch):
|
|
from config.settings import settings
|
|
from summarize import llm_client
|
|
|
|
monkeypatch.setattr(settings, "llm_base_url", "http://x")
|
|
monkeypatch.setattr(settings, "llm_api_key", "k")
|
|
monkeypatch.setattr(settings, "llm_backoff_base", 0.0)
|
|
monkeypatch.setattr(settings, "llm_backoff_max", 0.0)
|
|
monkeypatch.setattr(llm_client.time, "sleep", lambda *_: None)
|
|
|
|
client = llm_client.OpenAICompatClient()
|
|
calls = {"n": 0}
|
|
|
|
def flaky(system, user):
|
|
calls["n"] += 1
|
|
if calls["n"] < 3:
|
|
raise urllib.error.HTTPError("u", 429, "Too Many Requests", {}, None)
|
|
return {"choices": [{"message": {"content": '{"ok": true}'}}], "usage": {"prompt_tokens": 5}}
|
|
|
|
monkeypatch.setattr(client, "_post_once", flaky)
|
|
assert client.complete_json("s", "u") == {"ok": True}
|
|
assert calls["n"] == 3
|
|
assert client.usage["retries"] == 2
|
|
assert client.usage["calls"] == 1
|
|
|
|
|
|
def test_api_backend_does_not_retry_on_400(monkeypatch):
|
|
from config.settings import settings
|
|
from summarize import llm_client
|
|
|
|
monkeypatch.setattr(settings, "llm_base_url", "http://x")
|
|
monkeypatch.setattr(settings, "llm_api_key", "k")
|
|
client = llm_client.OpenAICompatClient()
|
|
calls = {"n": 0}
|
|
|
|
def bad(system, user):
|
|
calls["n"] += 1
|
|
raise urllib.error.HTTPError("u", 400, "Bad Request", {}, None)
|
|
|
|
monkeypatch.setattr(client, "_post_once", bad)
|
|
with pytest.raises(urllib.error.HTTPError):
|
|
client.complete_json("s", "u")
|
|
assert calls["n"] == 1, "4xx(429 제외)는 재시도하지 않아야 한다"
|
|
|
|
|
|
# ------------------------------------------------------------------ jobs CLI
|
|
|
|
def test_jobs_answer_validates_schema(tmp_path, capsys):
|
|
llm = FileQueueLLM(tmp_path)
|
|
with pytest.raises(PendingResponse) as ei:
|
|
llm.complete_json("sys", UNIT_PROMPT_SAMPLE)
|
|
jid = ei.value.job_id
|
|
|
|
bad = tmp_path / "bad.json"
|
|
bad.write_text('{"unit_purpose_ko": 1, "chunks": [{"line_start": "x"}]}', encoding="utf-8")
|
|
args = type("A", (), {"dir": str(tmp_path), "job_id": jid, "file": str(bad),
|
|
"stdin": False, "no_validate": False})()
|
|
assert jobs.cmd_answer(args) == 1
|
|
assert not llm.response_path(jid).exists()
|
|
|
|
good = tmp_path / "good.json"
|
|
good.write_text('{"unit_purpose_ko": "정상", "chunks": []}', encoding="utf-8")
|
|
args.file = str(good)
|
|
assert jobs.cmd_answer(args) == 0
|
|
assert json.loads(llm.response_path(jid).read_text(encoding="utf-8"))["unit_purpose_ko"] == "정상"
|
|
|
|
|
|
def test_jobs_answer_reports_missing_file(tmp_path):
|
|
args = type("A", (), {"dir": str(tmp_path), "job_id": "deadbeef", "file": "nope.json",
|
|
"stdin": False, "no_validate": False})()
|
|
assert jobs.cmd_answer(args) == 1 # 프롬프트도 없으므로 1
|
|
|
|
|
|
# ------------------------------------------------- /ingest 의 백엔드 선택
|
|
|
|
INGEST_PAYLOAD = {
|
|
"MAIN_PROGRAM": "ZBK_T1",
|
|
"INCLUDE_PROGRAM": [{"INCLUDE": "ZBK_T1", "SOURCE_CODE":
|
|
"REPORT zbk_t1.\nSTART-OF-SELECTION.\n PERFORM go.\n"
|
|
"FORM go.\n SELECT * FROM t001 INTO TABLE gt.\nENDFORM."}],
|
|
}
|
|
|
|
|
|
@pytest.fixture()
|
|
def api_client(tmp_path, monkeypatch):
|
|
from fastapi.testclient import TestClient
|
|
|
|
import query.api as api
|
|
from config.settings import settings
|
|
|
|
monkeypatch.setattr(settings, "database_url", f"sqlite:///{tmp_path / 'i.db'}")
|
|
monkeypatch.setattr(settings, "data_normalized", tmp_path / "n")
|
|
monkeypatch.setattr(settings, "data_parsed", tmp_path / "p")
|
|
monkeypatch.setattr(settings, "data_llm_jobs", tmp_path / "jobs")
|
|
# /ingest 의 백그라운드 작업이 요약 후 위키를 쓴다 — 실제 wiki/ 로 새지 않게 격리
|
|
monkeypatch.setattr(settings, "wiki_dir", tmp_path / "wiki")
|
|
return TestClient(api.app), settings, tmp_path
|
|
|
|
|
|
def test_ingest_backend_off_skips_summary(api_client, monkeypatch):
|
|
c, settings, _ = api_client
|
|
monkeypatch.setattr(settings, "summarize_backend", "off")
|
|
monkeypatch.setattr(settings, "llm_base_url", "http://x")
|
|
monkeypatch.setattr(settings, "llm_api_key", "k")
|
|
r = c.post("/ingest", json=INGEST_PAYLOAD).json()
|
|
assert r["summarize"] == "disabled" and r["summarize_backend"] == "off"
|
|
|
|
|
|
def test_ingest_backend_file_queues_without_key(api_client, monkeypatch):
|
|
"""키가 없어도 file 백엔드면 프롬프트가 큐에 쌓인다."""
|
|
c, settings, tmp_path = api_client
|
|
monkeypatch.setattr(settings, "summarize_backend", "file")
|
|
monkeypatch.setattr(settings, "llm_base_url", "") # 키 없음
|
|
monkeypatch.setattr(settings, "llm_api_key", "")
|
|
r = c.post("/ingest", json=INGEST_PAYLOAD).json()
|
|
assert r["summarize"] == "scheduled", r
|
|
q = c.get("/summaries/jobs").json()
|
|
assert q["pending"] >= 1, q
|
|
assert any(n["task"] == "program_summary" for n in q["next"]), q
|