"""Stage 4 로더 — packages.jsonl + parse.json → SQLite upsert. python -m index.loader [--program X] [--limit N] 증분: program.source_hash(전체 인클루드 해시 결합)가 같으면 스킵. 재적재 시 code_hash 가 같은 unit 의 요약·로직 조각(logic_chunk)은 보존한다. """ from __future__ import annotations import argparse import hashlib import json from pathlib import Path from config.settings import settings from .db import bigrams, clean_comment, connect, loads def load_packages(con) -> int: path = settings.data_normalized / "packages.jsonl" if not path.exists(): return 0 n = 0 with path.open(encoding="utf-8") as f: for line in f: row = json.loads(line) con.execute( "INSERT INTO package(devclass, text_ko) VALUES(?,?) " "ON CONFLICT(devclass) DO NOTHING", (row["devclass"], ""), ) con.execute( "INSERT INTO program(name, devclass, title_ko, created_on, changed_on) " "VALUES(?,?,?,?,?) " "ON CONFLICT(name) DO UPDATE SET devclass=excluded.devclass, " "title_ko=excluded.title_ko, created_on=excluded.created_on, changed_on=excluded.changed_on", (row["obj_name"], row["devclass"], row["text"], row["created_on"], row["changed_on"]), ) n += 1 return n def load_tcodes(con) -> int: """data/normalized/tcodes.jsonl → tcode 테이블. TSTC 를 못 받은 상태(ASSUMPTIONS.md §5)에서, 덤프 헤더 주석의 `* T_CODE :` 값을 ingest/from_dir.py 가 모아둔 것을 적재한다. tcodes/ 위키 페이지의 입력이 된다. """ path = settings.data_normalized / "tcodes.jsonl" if not path.exists(): return 0 n = 0 with path.open(encoding="utf-8") as f: for line in f: if not line.strip(): continue row = json.loads(line) con.execute( "INSERT INTO tcode(tcode, program, text_ko) VALUES(?,?,?) " "ON CONFLICT(tcode) DO UPDATE SET program=excluded.program, text_ko=excluded.text_ko", (row["tcode"].upper(), (row.get("program") or "").upper(), row.get("text_ko") or ""), ) n += 1 return n def load_parsed(con, parse_path: Path, force: bool = False) -> str: """parse.json 한 건 적재. force=True 면 source_hash 가 같아도 구조를 다시 적재한다. 파서나 색인 로직이 바뀐 경우(소스는 그대로인데 산출물이 달라지는 경우) 강제 재적재가 필요하다. """ d = json.loads(parse_path.read_text(encoding="utf-8")) program = d["program"] source_hash = hashlib.sha256( "".join(i["sha256"] for i in d["includes"]).encode() ).hexdigest() cur = con.execute("SELECT source_hash FROM program WHERE name=?", (program,)) row = cur.fetchone() if row and row["source_hash"] == source_hash and not force: # 구조는 그대로 — 나중에 추가된 컬럼(텍스트 심볼)만 비어 있으면 채운다 con.execute("UPDATE program SET text_symbols_json=? WHERE name=? AND text_symbols_json IS NULL", (json.dumps(d.get("text_symbols", []), ensure_ascii=False), program)) return "skip" title = d.get("description", "") text_symbols = json.dumps(d.get("text_symbols", []), ensure_ascii=False) con.execute( "INSERT INTO program(name, title_ko, source_hash, has_source, text_symbols_json) VALUES(?,?,?,1,?) " "ON CONFLICT(name) DO UPDATE SET " "title_ko=CASE WHEN excluded.title_ko!='' THEN excluded.title_ko ELSE program.title_ko END, " "source_hash=excluded.source_hash, has_source=1, summary_status='stale', " "text_symbols_json=excluded.text_symbols_json", (program, title, source_hash, text_symbols), ) # 재적재 전, code_hash 가 같은 unit 의 기존 요약(얇은 색인)·로직 조각을 보존해 둔다 kept_summaries = { r["unit_id"]: (r["code_hash"], r["summary_json"], r["prompt_version"], r["chunk_count"]) for r in con.execute( "SELECT unit_id, code_hash, summary_json, prompt_version, chunk_count FROM unit " "WHERE program=? AND summary_status='done'", (program,) ).fetchall() } # 기존 구조 데이터 삭제 후 재적재 (unit 단위 증분은 v2) for table in ("include", "unit", "symbol", "declaration", "symbol_write", "symbol_read", "table_ref", "call_edge", "topo"): con.execute(f"DELETE FROM {table} WHERE program=?", (program,)) con.execute("DELETE FROM unit_fts WHERE program=?", (program,)) norm_dir = settings.data_normalized / program for inc in d["includes"]: code_file = norm_dir / f"{inc['include']}.abap" code = code_file.read_text(encoding="utf-8") if code_file.exists() else "" con.execute( "INSERT INTO include(program, include, code_hash, line_count, code) VALUES(?,?,?,?,?)", (program, inc["include"], inc["sha256"], inc["line_count"], code), ) kept_unit_ids: set[str] = set() for u in d["units"]: con.execute( "INSERT INTO unit(unit_id, program, include, unit_type, name, line_start, line_end, " "code_hash, signature, header_comment, loc, refs_json, sub_chunks_json) " "VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)", (u["unit_id"], program, u["include"], u["unit_type"], u["name"], u["line_start"], u["line_end"], u["code_hash"], u.get("signature", ""), u.get("header_comment", ""), u.get("loc", 0), json.dumps(u.get("refs"), ensure_ascii=False) if u.get("refs") else None, json.dumps(u.get("sub_chunks", []), ensure_ascii=False)), ) kept = kept_summaries.get(u["unit_id"]) if kept and kept[0] == u["code_hash"]: con.execute( "UPDATE unit SET summary_json=?, summary_status='done', prompt_version=?, chunk_count=? " "WHERE unit_id=?", (kept[1], kept[2], kept[3] or 0, u["unit_id"]), ) kept_unit_ids.add(u["unit_id"]) refs = u.get("refs") or {} for t in refs.get("tables_read", []): con.execute("INSERT INTO table_ref VALUES(?,?,?,?,?)", (program, u["unit_id"], t, "read", 0)) for t in refs.get("tables_write", []): con.execute("INSERT INTO table_ref VALUES(?,?,?,?,?)", (program, u["unit_id"], t, "write", 0)) # unit FTS — 얇은 색인 (이름/시그니처/주석; 요약이 있으면 한 줄 요약도) # 장식 문자('----* INITIALIZATION ----*')를 그대로 넣으면 색인이 쓰레기가 된다 (수정사항 1번) purpose = clean_comment(u.get("header_comment", "")) if kept and kept[0] == u["code_hash"] and kept[1]: purpose = (json.loads(kept[1]).get("purpose_ko") or purpose) text = " ".join(filter(None, [u["name"], u.get("signature", ""), purpose])) con.execute( "INSERT INTO unit_fts(unit_id, program, name, purpose, keywords, bigrams) VALUES(?,?,?,?,?,?)", (u["unit_id"], program, u["name"], purpose, "", bigrams(text)), ) # 로직 조각: code_hash 가 같은 unit 의 조각만 남기고 나머지는 삭제 (재추출 대상) stale_chunks = [ r["chunk_id"] for r in con.execute( "SELECT chunk_id, unit_id FROM logic_chunk WHERE program=?", (program,) ).fetchall() if r["unit_id"] not in kept_unit_ids ] for cid in stale_chunks: con.execute("DELETE FROM logic_chunk WHERE chunk_id=?", (cid,)) con.execute("DELETE FROM chunk_fts WHERE chunk_id=?", (cid,)) for s in d["symbols"]: con.execute( "INSERT INTO symbol VALUES(?,?,?,?,?,?,?,?,?)", (program, s["name"], s["scope"], s.get("unit_id"), s.get("decl_include", ""), s.get("decl_line", 0), s.get("type_text", ""), s.get("ddic_ref", ""), s.get("kind", "")), ) # 정의부 — 붙여넣기용 선언 원문 (parser/declarations.py) for dc in d.get("declarations", []): con.execute( "INSERT INTO declaration(program, name, kind, scope, unit_id, include, line_start, " "line_end, code, type_text, depends_json) VALUES(?,?,?,?,?,?,?,?,?,?,?)", (program, dc["name"], dc["kind"], dc["scope"], dc.get("unit_id"), dc["include"], dc["line_start"], dc["line_end"], dc["code"], dc.get("type_text", ""), json.dumps(dc.get("depends", []), ensure_ascii=False)), ) for w in d["writes"]: con.execute( "INSERT INTO symbol_write(program, symbol, unit_id, include, line, kind, " "source_symbols, source_tables, stmt_text, callee) VALUES(?,?,?,?,?,?,?,?,?,?)", (program, w["symbol"], w["unit_id"], w["include"], w["line"], w["kind"], json.dumps(w.get("source_symbols", [])), json.dumps(w.get("source_tables", [])), w.get("stmt_text", ""), w.get("callee", "")), ) for r in d["reads"]: con.execute( "INSERT INTO symbol_read VALUES(?,?,?,?,?)", (program, r["symbol"], r["unit_id"], r["include"], r["line"]), ) for e in d["call_edges"]: con.execute( "INSERT INTO call_edge VALUES(?,?,?,?,?,?)", (program, e["from_unit"], e.get("to_unit"), e.get("external_name"), e["call_type"], e.get("line", 0)), ) for i, uid in enumerate(d.get("topo_order", [])): con.execute("INSERT OR REPLACE INTO topo VALUES(?,?,?)", (program, uid, i)) return "loaded" def _identity_row(row) -> tuple: """program_fts — 이름·타이틀·패키지만. **짧게 유지한다.** 요약을 여기에 섞으면 bm25 의 길이 정규화 때문에 요약이 있는 프로그램이 오히려 밀린다 (index/db.py 의 주석 참고). 서술 텍스트는 program_desc_fts 로 분리한다. """ title = row["title_ko"] or "" text = " ".join(filter(None, [title, row["pkg_text"]])) return (row["name"], title, " ".join(filter(None, [row["devclass"] or "", row["pkg_text"] or ""])), bigrams(text)) def _desc_row(con, row) -> tuple | None: """program_desc_fts — LLM 요약·조각 키워드·테이블·텍스트 심볼 (수정사항 1·3번). "입고 처리하는 프로그램" 처럼 이름·타이틀에 없는 말로 물어도 걸리게 하는 색인이다. 넣을 게 없으면 None (빈 행을 만들면 길이 정규화만 왜곡된다). """ name = row["name"] summary = loads(row["summary_json"]) or {} purpose = " ".join(filter(None, [ summary.get("business_purpose_ko") or "", summary.get("business_purpose_en") or "", " ".join(summary.get("main_flow") or []), ])) kw: list[str] = list(summary.get("keywords_ko") or []) + list(summary.get("keywords_en") or []) kw += list(summary.get("business_tags") or []) if summary.get("sap_module"): kw.append(summary["sap_module"]) objs: list[str] = list(summary.get("related_tcodes") or []) for r in con.execute( "SELECT keywords_ko, keywords_en, sap_objects FROM logic_chunk WHERE program=?", (name,) ): kw += (loads(r["keywords_ko"]) or []) + (loads(r["keywords_en"]) or []) objs += loads(r["sap_objects"]) or [] objs += [t["table_name"] for t in con.execute( "SELECT DISTINCT table_name FROM table_ref WHERE program=? LIMIT 200", (name,))] # 텍스트 심볼의 한국어 원문 — LLM 없이도 자연어 앵커가 생긴다 (수정사항 3번) ts = [t.get("text", "") for t in (loads(row["text_symbols_json"]) or []) if t.get("text")] kw = list(dict.fromkeys(k for k in kw if k))[:200] objs = list(dict.fromkeys(o.upper() for o in objs if o))[:200] purpose_text = " ".join(filter(None, [purpose, " ".join(ts)])) if not (purpose_text or kw or objs): return None return (name, purpose_text, " ".join(kw), " ".join(objs), bigrams(" ".join(filter(None, [purpose, " ".join(kw), " ".join(ts)])))) _PROGRAM_FTS_SQL = ( "SELECT p.name, p.title_ko, p.devclass, p.summary_json, p.text_symbols_json, " "COALESCE(k.text_ko,'') AS pkg_text FROM program p " "LEFT JOIN package k ON k.devclass = p.devclass" ) def refresh_program_fts(con, program: str | None = None) -> int: """program_fts + program_desc_fts 재구축. program 을 주면 그 한 건만 갱신한다. 전량 재구축을 /ingest 마다 하면 1만 본에서 O(N²) 가 된다 — 단건 경로가 필요하다. """ if program: name = program.upper() con.execute("DELETE FROM program_fts WHERE name=?", (name,)) con.execute("DELETE FROM program_desc_fts WHERE name=?", (name,)) rows = con.execute(_PROGRAM_FTS_SQL + " WHERE p.name=?", (name,)).fetchall() else: con.execute("DELETE FROM program_fts") con.execute("DELETE FROM program_desc_fts") rows = con.execute(_PROGRAM_FTS_SQL).fetchall() for row in rows: con.execute( "INSERT INTO program_fts(name, title, keywords, bigrams) VALUES(?,?,?,?)", _identity_row(row), ) desc = _desc_row(con, row) if desc: con.execute( "INSERT INTO program_desc_fts(name, purpose, keywords, objects, bigrams) " "VALUES(?,?,?,?,?)", desc, ) return len(rows) def main() -> None: ap = argparse.ArgumentParser(description="Stage 4 — 인덱스 적재") ap.add_argument("--program", default=None) ap.add_argument("--limit", type=int, default=None) ap.add_argument("--force", action="store_true", help="source_hash 가 같아도 재적재 (파서·색인 로직이 바뀐 경우)") args = ap.parse_args() con = connect() n_pkg = load_packages(con) n_tcode = load_tcodes(con) parsed = sorted(settings.data_parsed.glob("*.parse.json")) if settings.data_parsed.exists() else [] if args.program: parsed = [p for p in parsed if p.stem.replace(".parse", "") == args.program.upper()] if args.limit: parsed = parsed[: args.limit] stats = {"package_rows": n_pkg, "tcodes": n_tcode, "loaded": 0, "skipped": 0, "errors": 0} con.commit() for p in parsed: try: r = load_parsed(con, p, force=args.force) con.commit() # 프로그램 단위 커밋 — 실패 시 해당 프로그램만 롤백 stats["loaded" if r == "loaded" else "skipped"] += 1 except Exception as e: # noqa: BLE001 con.rollback() stats["errors"] += 1 print(f"[FAIL] {p.name}: {type(e).__name__}: {e}") refresh_program_fts(con) con.commit() con.close() print(json.dumps(stats, ensure_ascii=False)) if __name__ == "__main__": main()