Initial commit: ABAP indexing pipeline (ingest, parser, summarize, index, query, wiki)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

tests/test_parser_fieldsymbol.py

@@ -0,0 +1,96 @@
+"""필드심볼·테이블본문 대입 파싱 검증.
+
+이 문장 형태들이 토큰 3개로 쪼개지면 (a) 쓰기 지점이 누락되고 (b) 미인식 문장으로 잡힌다.
+ALV 필드카탈로그를 채우는 관용구라 프로그램에 따라 미인식률이 12% 까지 올라갔다.
+"""
+from pathlib import Path
+
+import pytest
+
+from parser.dataflow import base_symbol, extract_writes
+from parser.run import parse_program
+from parser.statements import assignment_eq_index, split_statements
+from parser.tokenizer import tokenize_code
+
+ROOT = Path(__file__).resolve().parent.parent
+NORM = ROOT / "data" / "normalized"
+
+
+def test_field_symbol_component_is_one_token():
+    assert tokenize_code("<ls_fcat>-fieldname = 'X'.")[:2] == ["<ls_fcat>-fieldname", "="]
+    assert tokenize_code("<go_grid>->check_changed_data( ).")[0] == "<go_grid>->check_changed_data"
+    assert tokenize_code("<fs>-a-b = 1.")[0] == "<fs>-a-b"
+
+
+def test_plain_field_symbol_unaffected():
+    assert tokenize_code("READ TABLE t ASSIGNING <fs>.")[-2] == "<fs>"
+    assert tokenize_code("<fs> = ls_y.")[:2] == ["<fs>", "="]
+
+
+def test_macro_param_tokenized():
+    assert tokenize_code("&1 = |{ &2 }|.")[0] == "&1"
+
+
+def test_base_symbol_normalizes_field_symbol():
+    assert base_symbol("<LS_FCAT>-FIELDNAME") == "<LS_FCAT>"
+    assert base_symbol("<GO_GRID>->METH") == "<GO_GRID>"
+    assert base_symbol("GS_HEAD-BELNR") == "GS_HEAD"
+
+
+@pytest.mark.parametrize(
+    "code, expect_eq",
+    [
+        ("lv_a = 1.", 1),
+        ("<ls_fcat>-fieldname = 'X'.", 1),
+        ("gt_tab[] = gt_src[].", 3),
+        ("ls_r-opt[] = VALUE #( ).", 3),
+        ("IF lv_a = 1.", None),          # 조건문은 대입이 아니다
+        ("CLEAR lv_a.", None),
+        ("PERFORM f USING a.", None),
+    ],
+)
+def test_assignment_eq_index(code, expect_eq):
+    sts, _ = split_statements(code, "T")
+    assert assignment_eq_index(sts[0].upper) == expect_eq
+
+
+def _writes_for(code: str):
+    sts, _ = split_statements(code, "T")
+    known = {"GT_TAB", "GT_SRC", "<LS_FCAT>", "LS_R"}
+    writes, _ = extract_writes(sts, list(range(len(sts))), "U", known)
+    return {(w.symbol, w.kind) for w in writes}
+
+
+def test_field_symbol_component_write_recorded():
+    got = _writes_for("<ls_fcat>-fieldname = 'X'.")
+    assert ("<LS_FCAT>", "assign:field=FIELDNAME") in got
+
+
+def test_object_attribute_write_has_clean_field_name():
+    got = _writes_for("<go_dd>->html_control = x.")
+    # '->' 가 field 이름에 '>' 로 새어 들어가지 않아야 한다
+    assert ("<GO_DD>", "assign:field=HTML_CONTROL") in got
+
+
+def test_table_body_assignment_write_recorded():
+    got = _writes_for("gt_tab[] = gt_src[].")
+    assert ("GT_TAB", "assign") in got
+
+
+def _normalized_dirs() -> list[Path]:
+    return sorted(p for p in NORM.iterdir() if p.is_dir()) if NORM.exists() else []
+
+
+@pytest.mark.skipif(not _normalized_dirs(), reason="정규화 산출물 없음")
+def test_unknown_ratio_low_across_all_programs():
+    """§4.6 — 한 프로그램만 보면 놓친다. 전체를 재고 최악값도 함께 본다.
+
+    특정 샘플 이름에 묶지 않는다 — 어떤 덤프가 들어와도 전체를 재야 의미가 있다.
+    """
+    worst = []
+    for d in _normalized_dirs():
+        stats = parse_program(d)["stats"]
+        worst.append((stats["unknown_ratio"], d.name))
+    worst.sort(reverse=True)
+    top_ratio, top_name = worst[0]
+    assert top_ratio < 0.03, f"미인식률 최악: {top_name} {top_ratio:.2%} (전체: {worst[:5]})"