byeongwook.choiandClaude Fable 5.1
edb0b111a8
고객사 실측: 패키지 목록의 CHANGED_ON 에서 TypeError 가 나며 정규화가 중단돼 206본 중 49본만 처리됐다. clean_text_field 가 값을 문자열로 바꿔 처리하고, 파일별 처리 전체를 try 로 감싸 [FAIL] 기록 후 계속한다. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
175 lines
8.3 KiB
Python
175 lines
8.3 KiB
Python
"""Stage 0 (ingest/from_sap.py) — SAP 응답을 가짜로 넣어 저장 형식과 normalize 연결을 확인한다.
|
||
|
||
실제 SAP 은 부르지 않는다. 응답 모양은 ../abap-api-tester/tools/spec/samples 의 실측 JSON 과 같다.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
|
||
import pytest
|
||
|
||
from ingest import from_sap
|
||
from ingest.normalize import classify, parse_collected_file, run as normalize_run
|
||
|
||
PROGRAM_LIST = {
|
||
"RETURN": {"TYPE": "S", "MESSAGE": "", "TOTAL_ROWS": "2"},
|
||
"RESULT": [
|
||
{"OBJ_NAME": "ZFIR0010", "TEXT": "거래처 I/F 이력", "SUBC": "1", "UDAT": "2024-01-02", "UNAM": "U1"},
|
||
{"OBJ_NAME": "SAPMZSFT0", "TEXT": "달력 유지보수", "SUBC": "M", "UDAT": "2020-10-16", "UNAM": "U2"},
|
||
],
|
||
}
|
||
|
||
SOURCES = {
|
||
"ZFIR0010": {
|
||
"RETURN": {"TYPE": "S", "MESSAGE": "", "TOTAL_ROWS": "0"},
|
||
"PROGRAM": "ZFIR0010",
|
||
"SOURCE_CODE": "REPORT zfir0010.\nINCLUDE zfir0010_top.\nSTART-OF-SELECTION.\n PERFORM main.\n",
|
||
"INCLUDE_LIST": [
|
||
{"INCL_NAME": "ZFIR0010_TOP", "SOURCE": "DATA gv_bukrs TYPE bukrs.\n"},
|
||
{"INCL_NAME": "DB__SSEL", "SOURCE": ""}, # 빈 소스는 버린다
|
||
{"INCL_NAME": "ZFIR0010_TOP", "SOURCE": "DATA dup.\n"}, # 중복은 버린다
|
||
],
|
||
"SCREEN_LIST": "",
|
||
"TCODE_LIST": [{"TCODE": "ZFIR0010", "TTEXT": "거래처 I/F 이력조회", "PGMNA": "ZFIR0010"}],
|
||
},
|
||
"SAPMZSFT0": {
|
||
"RETURN": {"TYPE": "E", "MESSAGE": "프로그램 소스를 읽을 수 없습니다", "TOTAL_ROWS": "0"},
|
||
"PROGRAM": "SAPMZSFT0", "SOURCE_CODE": "", "INCLUDE_LIST": "", "SCREEN_LIST": "", "TCODE_LIST": "",
|
||
},
|
||
}
|
||
|
||
|
||
def fake_sap_call(method: str, params: dict, timeout: int = 180) -> dict:
|
||
if method == "GET_PROGRAM_LIST":
|
||
assert params["IV_PACKAGE"] == "ZFI01"
|
||
data = PROGRAM_LIST
|
||
elif method == "GET_PROGRAM_SOURCE":
|
||
assert params["IV_WITH_INCLUDE"] == "X"
|
||
data = SOURCES[params["IV_PROGRAM"]]
|
||
else:
|
||
raise AssertionError(method)
|
||
ret = data["RETURN"]
|
||
return {"method": method, "parsed": data, "error": None,
|
||
"return": {"type": ret["TYPE"], "message": ret["MESSAGE"]}}
|
||
|
||
|
||
@pytest.fixture
|
||
def sap(monkeypatch):
|
||
import sap.sap_client as client
|
||
monkeypatch.setattr(client, "sap_call", fake_sap_call)
|
||
monkeypatch.setattr(from_sap.time, "sleep", lambda s: None)
|
||
|
||
|
||
def test_run_writes_raw_files_and_records_errors(tmp_path, sap):
|
||
stats = from_sap.run(["zfi01"], tmp_path)
|
||
assert stats == {"packages": 1, "listed": 2, "fetched": 1, "skipped": 0, "errors": 1, "includes": 2}
|
||
|
||
# 패키지 목록: normalize 가 package_list 로 분류하는 모양
|
||
pkg = parse_collected_file((tmp_path / "ZFI01.txt").read_text(encoding="utf-8"))
|
||
assert classify(pkg) == "package_list"
|
||
assert pkg[0] == {"DEVCLASS": "ZFI01", "OBJ_NAME": "ZFIR0010", "TEXT": "거래처 I/F 이력",
|
||
"CREATED_ON": "", "CHANGED_ON": "2024-01-02"}
|
||
|
||
# 프로그램 소스: 메인 + 인클루드(빈 것·중복 제외), 줄바꿈은 JSON 이스케이프로 보존
|
||
prog = parse_collected_file((tmp_path / "ZFIR0010.txt").read_text(encoding="utf-8"))
|
||
assert classify(prog) == "program_source"
|
||
assert prog["DESCRIPTION"] == "거래처 I/F 이력"
|
||
assert [i["INCLUDE"] for i in prog["INCLUDE_PROGRAM"]] == ["ZFIR0010", "ZFIR0010_TOP"]
|
||
assert prog["INCLUDE_PROGRAM"][0]["SOURCE_CODE"].splitlines()[0] == "REPORT zfir0010."
|
||
assert prog["TEXT_SYMBOL"] == []
|
||
assert prog["TCODE_LIST"][0]["TCODE"] == "ZFIR0010"
|
||
|
||
# 실패한 프로그램은 파일 없이 _errors.log 에
|
||
assert not (tmp_path / "SAPMZSFT0.txt").exists()
|
||
assert "SAPMZSFT0\tSOURCE\tSAP 오류" in (tmp_path / "_errors.log").read_text(encoding="utf-8")
|
||
|
||
|
||
def test_rerun_skips_existing_unless_force(tmp_path, sap):
|
||
from_sap.run(["ZFI01"], tmp_path)
|
||
again = from_sap.run(["ZFI01"], tmp_path)
|
||
assert again["skipped"] == 1 and again["fetched"] == 0
|
||
forced = from_sap.run(["ZFI01"], tmp_path, force=True)
|
||
assert forced["fetched"] == 1
|
||
|
||
|
||
def test_normalize_consumes_output_and_builds_tcodes(tmp_path, sap):
|
||
raw, out = tmp_path / "raw", tmp_path / "normalized"
|
||
from_sap.run(["ZFI01"], raw)
|
||
stats = normalize_run(raw, out)
|
||
assert stats["programs"] == 1 and stats["package_rows"] == 2 and stats["errors"] == 0
|
||
|
||
assert (out / "ZFIR0010" / "ZFIR0010.abap").read_text(encoding="utf-8").startswith("REPORT zfir0010.\n")
|
||
assert (out / "ZFIR0010" / "ZFIR0010_TOP.abap").exists()
|
||
meta = json.loads((out / "ZFIR0010" / "ZFIR0010.meta.json").read_text(encoding="utf-8"))
|
||
assert meta["description"] == "거래처 I/F 이력" and len(meta["includes"]) == 2
|
||
|
||
tcodes = [json.loads(l) for l in (out / "tcodes.jsonl").read_text(encoding="utf-8").splitlines()]
|
||
assert tcodes == [{"tcode": "ZFIR0010", "program": "ZFIR0010", "text_ko": "거래처 I/F 이력조회"}]
|
||
|
||
|
||
def test_dry_run_writes_nothing(tmp_path, sap):
|
||
stats = from_sap.run(["ZFI01"], tmp_path, dry_run=True)
|
||
assert stats["listed"] == 2 and stats["fetched"] == 0
|
||
assert list(tmp_path.iterdir()) == []
|
||
|
||
|
||
def test_sap_client_parses_json_and_xml_responses():
|
||
from sap.sap_client import parse_response
|
||
|
||
js = parse_response('{"RETURN":{"TYPE":"W","MESSAGE":"잘림","TOTAL_ROWS":177},"RESULT":[{"OBJ_NAME":"ZFI0000"}],"INCLUDE_LIST":[]}'.encode("utf-8"))
|
||
assert js["error"] is None and js["return"] == {"type": "W", "message": "잘림"}
|
||
assert js["parsed"]["RESULT"][0]["OBJ_NAME"] == "ZFI0000" and js["parsed"]["INCLUDE_LIST"] == []
|
||
|
||
xml = parse_response(b'\xef\xbb\xbf<?xml version="1.0" encoding="utf-16"?><asx:abap xmlns:asx="x"><asx:values><DATA>'
|
||
b'<RETURN><TYPE>S</TYPE><MESSAGE/></RETURN><RESULT/></DATA></asx:values></asx:abap>')
|
||
assert xml["error"] is None and xml["return"]["type"] == "S" and xml["parsed"]["RESULT"] == []
|
||
|
||
bad = parse_response(b"<html>login</html>")
|
||
assert bad["return"]["type"] == "X"
|
||
|
||
|
||
def test_sap_call_sends_fields_without_wrapper(monkeypatch):
|
||
import sap.sap_client as client
|
||
|
||
captured = {}
|
||
|
||
class _Resp:
|
||
status = 200
|
||
def __enter__(self): return self
|
||
def __exit__(self, *a): return False
|
||
def read(self): return b'{"RETURN":{"TYPE":"S","MESSAGE":"","TOTAL_ROWS":0},"RESULT":[]}'
|
||
|
||
def fake_urlopen(req, context=None, timeout=None):
|
||
captured["body"] = json.loads(req.data.decode("utf-8"))
|
||
captured["url"] = req.full_url
|
||
return _Resp()
|
||
|
||
monkeypatch.setattr(client.urllib.request, "urlopen", fake_urlopen)
|
||
res = client.sap_call("GET_PROGRAM_LIST", {"IV_PACKAGE": "ZFI01", "IV_PATTERN": "", "IV_MAX_ROWS": 5})
|
||
assert captured["body"] == {"IV_PACKAGE": "ZFI01", "IV_MAX_ROWS": 5} # 래퍼 없음, 빈 값 제외, 숫자 유지
|
||
assert captured["url"].endswith("/GET_PROGRAM_LIST")
|
||
assert res["return"]["type"] == "S" and res["parsed"]["RESULT"] == []
|
||
|
||
|
||
def test_normalize_survives_non_string_fields_and_bad_file(tmp_path):
|
||
"""JSON 규약 응답엔 숫자·null 이 섞인다. 파일 하나가 깨져도 나머지는 계속 처리해야 한다."""
|
||
from ingest.normalize import clean_text_field
|
||
assert clean_text_field(None) == "" and clean_text_field(20240102) == "20240102" and clean_text_field(0) == "0"
|
||
|
||
raw, out = tmp_path / "raw", tmp_path / "normalized"
|
||
raw.mkdir()
|
||
(raw / "ZPKG.txt").write_text(json.dumps([
|
||
{"DEVCLASS": "ZPKG", "OBJ_NAME": "ZA", "TEXT": None, "CREATED_ON": "", "CHANGED_ON": 20240102},
|
||
{"DEVCLASS": "ZPKG", "OBJ_NAME": "ZB", "TEXT": "b", "CREATED_ON": None, "CHANGED_ON": None},
|
||
]), encoding="utf-8")
|
||
(raw / "ZBAD.txt").write_text('{"MAIN_PROGRAM": "ZBAD", "INCLUDE_PROGRAM": "not-a-list"}', encoding="utf-8")
|
||
(raw / "ZOK.txt").write_text(json.dumps({"MAIN_PROGRAM": "ZOK", "DESCRIPTION": "ok",
|
||
"INCLUDE_PROGRAM": [{"INCLUDE": "ZOK", "SOURCE_CODE": "REPORT zok.\n"}]}),
|
||
encoding="utf-8")
|
||
stats = normalize_run(raw, out)
|
||
assert stats["programs"] == 1 and stats["package_rows"] == 2 and stats["errors"] == 1
|
||
assert (out / "ZOK" / "ZOK.abap").exists()
|
||
rows = [json.loads(l) for l in (out / "packages.jsonl").read_text(encoding="utf-8").splitlines()]
|
||
assert rows[0]["changed_on"] == "20240102" and rows[0]["text"] == "" and rows[1]["changed_on"] == ""
|
||
assert "ZBAD.txt" in (out / "_errors.log").read_text(encoding="utf-8")
|