Initial commit: ABAP indexing pipeline (ingest, parser, summarize, index, query, wiki)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-21 13:23:37 +09:00
commit 11ae3629b2
453 changed files with 259183 additions and 0 deletions
@@ -0,0 +1,194 @@
+"""위키 뷰어 빌더 — wiki/ 를 스캔해 단일 HTML(뷰어)로 만든다.
+
+- 템플릿: wiki_out/viewer.html (플레이스홀더 __DATA_JSON__, __GENERATED_AT__)
+- 빌드:   python -m wiki_out.viewer [--out PATH]   (기본 data/wiki-viewer.html)
+- 라이브: query/api.py 의 GET /wiki-viewer 가 요청마다 build_html() 호출
+- 공유용 스냅샷은 이 출력을 Claude 아티팩트로 게시 (index.db 관측소와 같은 패턴)
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from datetime import datetime
+from pathlib import Path
+
+from config.settings import settings
+
+from .merge import is_human_verified, split_frontmatter
+
+TEMPLATE = Path(__file__).parent / "viewer.html"
+
+
+def _fm_get(fm: str, key: str) -> str | None:
+    m = re.search(rf"^{re.escape(key)}: (.*)$", fm, re.M)
+    return m.group(1).strip() if m else None
+
+
+def _unquote(v: str | None) -> str:
+    if not v:
+        return ""
+    if v.startswith('"'):
+        try:
+            return json.loads(v)
+        except ValueError:
+            return v.strip('"')
+    return v
+
+
+def _list(v: str | None) -> list:
+    if not v or not v.startswith("["):
+        return []
+    try:
+        return json.loads(v)
+    except ValueError:
+        return []
+
+
+# 프로그램 문서의 조각 섹션을 읽어 사이드바 내비게이션을 만든다.
+# okf_writer 가 생성하는 모양에 의존한다:
+#   ### FORM SELECT_DATA — `ZFIR0030F01` L64-L80 (17줄)
+#   #### [sql_select] 미처리 건을 읽는다
+_UNIT_H = re.compile(r"^### (\w+) (.+?) — `([^`]+)` L(\d+)-L(\d+)", re.M)
+_CHUNK_H = re.compile(r"^#### \[(\w+)\] (.+)$", re.M)
+
+
+def _chunk_nav(body: str) -> list[dict]:
+    """프로그램 문서 본문 → [{unit, unit_type, include, kind, purpose}] (문서 순서 유지).
+
+    구조 변경(2026-09-16)으로 units/ 문서가 없어졌으므로, 내비게이션은 프로그램 문서 안의
+    조각 섹션에서 뽑는다. 예전 x-unit-* frontmatter 기반 코드는 항상 빈 목록을 돌려줬다.
+    """
+    out: list[dict] = []
+    cur = {"unit": "", "unit_type": "", "include": ""}
+    for m in re.finditer(r"^(###|####) (.*)$", body, re.M):
+        level, text = m.group(1), m.group(2)
+        if level == "###":
+            um = _UNIT_H.match(m.group(0))
+            if um:
+                cur = {"unit": um.group(2), "unit_type": um.group(1), "include": um.group(3)}
+            else:
+                cur = {"unit": text, "unit_type": "", "include": ""}
+        else:
+            cm = _CHUNK_H.match(m.group(0))
+            if cm:
+                out.append({**cur, "kind": cm.group(1), "purpose": cm.group(2)})
+    return out
+
+
+def build_data(wiki_dir: Path | None = None) -> dict:
+    wiki_dir = wiki_dir or settings.wiki_dir
+    docs: dict[str, dict] = {}
+    chunk_nav: dict[str, list[dict]] = {}   # program → 조각 목록
+
+    for path in sorted(wiki_dir.rglob("*.md")):
+        rel = path.relative_to(wiki_dir).as_posix()
+        if path.name in ("index.md", "log.md", "_review.md"):
+            continue
+        fm, body = split_frontmatter(path.read_text(encoding="utf-8"))
+        if fm is None:
+            continue
+        gen = ""
+        m = re.search(r'generated:.*?by:\s*"([^"]+)".*?at:\s*"([^"]+)"', fm)
+        if m:
+            gen = f"{m.group(1)} · {m.group(2)}"
+        doc = {
+            "t": _fm_get(fm, "type") or "?",
+            "title": _unquote(_fm_get(fm, "title")) or rel,
+            "d": _unquote(_fm_get(fm, "description")),
+            "tags": _list(_fm_get(fm, "tags")),
+            "st": _fm_get(fm, "status") or "stable",
+            "gen": gen,
+            "res": _unquote(_fm_get(fm, "resource")),
+            "ver": is_human_verified(fm),
+            "body": body.strip(),
+        }
+        docs[rel] = doc
+        if doc["t"] == "abap-program":
+            name = rel.rsplit("/", 1)[-1][:-3]
+            chunk_nav[name] = _chunk_nav(doc["body"])
+            doc["chunk_count"] = len(chunk_nav[name])
+
+    programs = []
+    for rel, d in docs.items():
+        if d["t"] != "abap-program":
+            continue
+        name = rel.rsplit("/", 1)[-1][:-3]
+        chunks = chunk_nav.get(name, [])
+        # unit 단위로 묶어 사이드바에 접히는 목록으로 준다
+        by_unit: list[dict] = []
+        for c in chunks:
+            if not by_unit or by_unit[-1]["name"] != c["unit"]:
+                by_unit.append({"name": c["unit"], "type": c["unit_type"],
+                                "include": c["include"], "chunks": []})
+            by_unit[-1]["chunks"].append({"kind": c["kind"], "purpose": c["purpose"]})
+        # 개요 표에는 짧은 타이틀(frontmatter title 의 "NAME — " 뒤)만 싣고, 긴 LLM 설명은 desc 로 따로 준다
+        short = re.sub(rf"^{re.escape(name)}\s*[—-]\s*", "", d["title"]).strip() or name
+        programs.append({
+            "name": name, "path": rel, "title": short, "desc": d["d"] or "",
+            "status": d["st"], "verified": d["ver"],
+            "units": by_unit,
+            "chunks": len(chunks),
+            "done": len(by_unit), "total": len(by_unit),
+        })
+    programs.sort(key=lambda p: p["name"])
+
+    packages = sorted(
+        ({"name": rel.rsplit("/", 1)[-1][:-3], "path": rel,
+          "count": len([1 for ln in d["body"].splitlines() if ln.startswith("| [")])}
+         for rel, d in docs.items() if d["t"] == "abap-package"),
+        key=lambda p: p["name"])
+    concepts = sorted(
+        ({"name": d["title"], "path": rel} for rel, d in docs.items() if d["t"] == "concept"),
+        key=lambda c: c["name"])
+
+    review_path = wiki_dir / "_review.md"
+    review = (sum(1 for ln in review_path.read_text(encoding="utf-8").splitlines()
+                  if ln.startswith("- [ ]")) if review_path.exists() else 0)
+
+    tables = sorted(
+        ({"name": rel.rsplit("/", 1)[-1][:-3], "path": rel}
+         for rel, d in docs.items() if d["t"] == "abap-table"),
+        key=lambda c: c["name"])
+    functions = sorted(
+        ({"name": rel.rsplit("/", 1)[-1][:-3], "path": rel}
+         for rel, d in docs.items() if d["t"] == "abap-function"),
+        key=lambda c: c["name"])
+
+    n_chunks = sum(p["chunks"] for p in programs)
+    units_total = sum(p["total"] for p in programs)
+    return {
+        "stats": {
+            "programs": len(programs),
+            "chunks": n_chunks,
+            "units_total": units_total, "units_done": units_total,
+            "tables": len(tables), "functions": len(functions),
+            "human_verified": sum(1 for d in docs.values() if d["ver"]),
+            "review": review,
+        },
+        "docs": docs, "programs": programs, "packages": packages, "concepts": concepts,
+        "tables": tables, "functions": functions,
+    }
+
+
+def build_html(wiki_dir: Path | None = None) -> str:
+    data = build_data(wiki_dir)
+    payload = json.dumps(data, ensure_ascii=False, separators=(",", ":")).replace("</", "<\\/")
+    return (TEMPLATE.read_text(encoding="utf-8")
+            .replace("__DATA_JSON__", payload)
+            .replace("__GENERATED_AT__", datetime.now().strftime("%Y-%m-%d %H:%M")))
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description="위키 뷰어 HTML 빌드")
+    ap.add_argument("--out", default=None)
+    args = ap.parse_args()
+    out = Path(args.out) if args.out else settings.wiki_dir.parent / "data" / "wiki-viewer.html"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    html = build_html()
+    out.write_text(html, encoding="utf-8")
+    print(f"{out} ({len(html) / 1024:.0f} KB)")
+
+
+if __name__ == "__main__":
+    main()