Stage 0: SAP(ZAA_ICF)에서 패키지 단위로 프로그램 소스 수집 (ingest/from_sap.py)

- sap/: abap-mcp 의 catalog/sap_client/mcp_server 복사. import 와 .env 탐색만 이 저장소에 맞춤 - ingest/from_sap.py: 패키지명 → GET_PROGRAM_LIST → GET_PROGRAM_SOURCE(+Include) → data/raw/*.txt (normalize 가 읽는 수집 JSON 형식 그대로). 받은 파일은 건너뛰고 --force 로 재수집 - normalize: 응답의 TCODE_LIST 를 tcodes.jsonl 로 (from_dir 와 같은 모양, 로더가 적재) - .env.example 에 SAP_URL/SAP_USER/SAP_PASS, tests/test_from_sap.py Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-21 14:13:14 +09:00
parent 11ae3629b2
commit 921b8ce229
13 changed files with 1595 additions and 1 deletions
@@ -0,0 +1,113 @@
+"""Stage 0 (ingest/from_sap.py) — SAP 응답을 가짜로 넣어 저장 형식과 normalize 연결을 확인한다.
+
+실제 SAP 은 부르지 않는다. 응답 모양은 ../abap-api-tester/tools/spec/samples 의 실측 JSON 과 같다.
+"""
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from ingest import from_sap
+from ingest.normalize import classify, parse_collected_file, run as normalize_run
+
+PROGRAM_LIST = {
+    "RETURN": {"TYPE": "S", "MESSAGE": "", "TOTAL_ROWS": "2"},
+    "RESULT": [
+        {"OBJ_NAME": "ZFIR0010", "TEXT": "거래처 I/F 이력", "SUBC": "1", "UDAT": "2024-01-02", "UNAM": "U1"},
+        {"OBJ_NAME": "SAPMZSFT0", "TEXT": "달력 유지보수", "SUBC": "M", "UDAT": "2020-10-16", "UNAM": "U2"},
+    ],
+}
+
+SOURCES = {
+    "ZFIR0010": {
+        "RETURN": {"TYPE": "S", "MESSAGE": "", "TOTAL_ROWS": "0"},
+        "PROGRAM": "ZFIR0010",
+        "SOURCE_CODE": "REPORT zfir0010.\nINCLUDE zfir0010_top.\nSTART-OF-SELECTION.\n  PERFORM main.\n",
+        "INCLUDE_LIST": [
+            {"INCL_NAME": "ZFIR0010_TOP", "SOURCE": "DATA gv_bukrs TYPE bukrs.\n"},
+            {"INCL_NAME": "DB__SSEL", "SOURCE": ""},                 # 빈 소스는 버린다
+            {"INCL_NAME": "ZFIR0010_TOP", "SOURCE": "DATA dup.\n"},  # 중복은 버린다
+        ],
+        "SCREEN_LIST": "",
+        "TCODE_LIST": [{"TCODE": "ZFIR0010", "TTEXT": "거래처 I/F 이력조회", "PGMNA": "ZFIR0010"}],
+    },
+    "SAPMZSFT0": {
+        "RETURN": {"TYPE": "E", "MESSAGE": "프로그램 소스를 읽을 수 없습니다", "TOTAL_ROWS": "0"},
+        "PROGRAM": "SAPMZSFT0", "SOURCE_CODE": "", "INCLUDE_LIST": "", "SCREEN_LIST": "", "TCODE_LIST": "",
+    },
+}
+
+
+def fake_sap_call(method: str, params: dict, timeout: int = 180) -> dict:
+    if method == "GET_PROGRAM_LIST":
+        assert params["IV_PACKAGE"] == "ZFI01"
+        data = PROGRAM_LIST
+    elif method == "GET_PROGRAM_SOURCE":
+        assert params["IV_WITH_INCLUDE"] == "X"
+        data = SOURCES[params["IV_PROGRAM"]]
+    else:
+        raise AssertionError(method)
+    ret = data["RETURN"]
+    return {"method": method, "parsed": data, "error": None,
+            "return": {"type": ret["TYPE"], "message": ret["MESSAGE"]}}
+
+
+@pytest.fixture
+def sap(monkeypatch):
+    import sap.sap_client as client
+    monkeypatch.setattr(client, "sap_call", fake_sap_call)
+    monkeypatch.setattr(from_sap.time, "sleep", lambda s: None)
+
+
+def test_run_writes_raw_files_and_records_errors(tmp_path, sap):
+    stats = from_sap.run(["zfi01"], tmp_path)
+    assert stats == {"packages": 1, "listed": 2, "fetched": 1, "skipped": 0, "errors": 1, "includes": 2}
+
+    # 패키지 목록: normalize 가 package_list 로 분류하는 모양
+    pkg = parse_collected_file((tmp_path / "ZFI01.txt").read_text(encoding="utf-8"))
+    assert classify(pkg) == "package_list"
+    assert pkg[0] == {"DEVCLASS": "ZFI01", "OBJ_NAME": "ZFIR0010", "TEXT": "거래처 I/F 이력",
+                      "CREATED_ON": "", "CHANGED_ON": "2024-01-02"}
+
+    # 프로그램 소스: 메인 + 인클루드(빈 것·중복 제외), 줄바꿈은 JSON 이스케이프로 보존
+    prog = parse_collected_file((tmp_path / "ZFIR0010.txt").read_text(encoding="utf-8"))
+    assert classify(prog) == "program_source"
+    assert prog["DESCRIPTION"] == "거래처 I/F 이력"
+    assert [i["INCLUDE"] for i in prog["INCLUDE_PROGRAM"]] == ["ZFIR0010", "ZFIR0010_TOP"]
+    assert prog["INCLUDE_PROGRAM"][0]["SOURCE_CODE"].splitlines()[0] == "REPORT zfir0010."
+    assert prog["TEXT_SYMBOL"] == []
+    assert prog["TCODE_LIST"][0]["TCODE"] == "ZFIR0010"
+
+    # 실패한 프로그램은 파일 없이 _errors.log 에
+    assert not (tmp_path / "SAPMZSFT0.txt").exists()
+    assert "SAPMZSFT0\tSOURCE\tSAP 오류" in (tmp_path / "_errors.log").read_text(encoding="utf-8")
+
+
+def test_rerun_skips_existing_unless_force(tmp_path, sap):
+    from_sap.run(["ZFI01"], tmp_path)
+    again = from_sap.run(["ZFI01"], tmp_path)
+    assert again["skipped"] == 1 and again["fetched"] == 0
+    forced = from_sap.run(["ZFI01"], tmp_path, force=True)
+    assert forced["fetched"] == 1
+
+
+def test_normalize_consumes_output_and_builds_tcodes(tmp_path, sap):
+    raw, out = tmp_path / "raw", tmp_path / "normalized"
+    from_sap.run(["ZFI01"], raw)
+    stats = normalize_run(raw, out)
+    assert stats["programs"] == 1 and stats["package_rows"] == 2 and stats["errors"] == 0
+
+    assert (out / "ZFIR0010" / "ZFIR0010.abap").read_text(encoding="utf-8").startswith("REPORT zfir0010.\n")
+    assert (out / "ZFIR0010" / "ZFIR0010_TOP.abap").exists()
+    meta = json.loads((out / "ZFIR0010" / "ZFIR0010.meta.json").read_text(encoding="utf-8"))
+    assert meta["description"] == "거래처 I/F 이력" and len(meta["includes"]) == 2
+
+    tcodes = [json.loads(l) for l in (out / "tcodes.jsonl").read_text(encoding="utf-8").splitlines()]
+    assert tcodes == [{"tcode": "ZFIR0010", "program": "ZFIR0010", "text_ko": "거래처 I/F 이력조회"}]
+
+
+def test_dry_run_writes_nothing(tmp_path, sap):
+    stats = from_sap.run(["ZFI01"], tmp_path, dry_run=True)
+    assert stats["listed"] == 2 and stats["fetched"] == 0
+    assert list(tmp_path.iterdir()) == []