20260807_OPENCODE_TEMP/ABAP-Indexing
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
ce459ba8f00db60f4149ba1684f68349e1355845
ABAP-Indexing/index/loader.py
T

332 lines
15 KiB
Python
Raw Blame History

"""Stage 4 로더 — packages.jsonl + parse.json → SQLite upsert.
python -m index.loader [--program X] [--limit N]
증분: program.source_hash(전체 인클루드 해시 결합)가 같으면 스킵.
재적재 시 code_hash 가 같은 unit 의 요약·로직 조각(logic_chunk)은 보존한다.
"""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
from config.settings import settings
from .db import bigrams, clean_comment, connect, loads
def load_packages(con) -> int:
path = settings.data_normalized / "packages.jsonl"
if not path.exists():
return 0
n = 0
with path.open(encoding="utf-8") as f:
for line in f:
row = json.loads(line)
con.execute(
"INSERT INTO package(devclass, text_ko) VALUES(?,?) "
"ON CONFLICT(devclass) DO NOTHING",
(row["devclass"], ""),
)
con.execute(
"INSERT INTO program(name, devclass, title_ko, created_on, changed_on) "
"VALUES(?,?,?,?,?) "
"ON CONFLICT(name) DO UPDATE SET devclass=excluded.devclass, "
"title_ko=excluded.title_ko, created_on=excluded.created_on, changed_on=excluded.changed_on",
(row["obj_name"], row["devclass"], row["text"], row["created_on"], row["changed_on"]),
)
n += 1
return n
def load_tcodes(con) -> int:
"""data/normalized/tcodes.jsonl → tcode 테이블.
TSTC 를 못 받은 상태(ASSUMPTIONS.md §5)에서, 덤프 헤더 주석의 `* T_CODE :` 값을
ingest/from_dir.py 가 모아둔 것을 적재한다. tcodes/ 위키 페이지의 입력이 된다.
"""
path = settings.data_normalized / "tcodes.jsonl"
if not path.exists():
return 0
n = 0
with path.open(encoding="utf-8") as f:
for line in f:
if not line.strip():
continue
row = json.loads(line)
con.execute(
"INSERT INTO tcode(tcode, program, text_ko) VALUES(?,?,?) "
"ON CONFLICT(tcode) DO UPDATE SET program=excluded.program, text_ko=excluded.text_ko",
(row["tcode"].upper(), (row.get("program") or "").upper(), row.get("text_ko") or ""),
)
n += 1
return n
def load_parsed(con, parse_path: Path, force: bool = False) -> str:
"""parse.json 한 건 적재. force=True 면 source_hash 가 같아도 구조를 다시 적재한다.
파서나 색인 로직이 바뀐 경우(소스는 그대로인데 산출물이 달라지는 경우) 강제 재적재가 필요하다.
"""
d = json.loads(parse_path.read_text(encoding="utf-8"))
program = d["program"]
source_hash = hashlib.sha256(
"".join(i["sha256"] for i in d["includes"]).encode()
).hexdigest()
cur = con.execute("SELECT source_hash FROM program WHERE name=?", (program,))
row = cur.fetchone()
if row and row["source_hash"] == source_hash and not force:
# 구조는 그대로 — 나중에 추가된 컬럼(텍스트 심볼)만 비어 있으면 채운다
con.execute("UPDATE program SET text_symbols_json=? WHERE name=? AND text_symbols_json IS NULL",
(json.dumps(d.get("text_symbols", []), ensure_ascii=False), program))
return "skip"
title = d.get("description", "")
text_symbols = json.dumps(d.get("text_symbols", []), ensure_ascii=False)
con.execute(
"INSERT INTO program(name, title_ko, source_hash, has_source, text_symbols_json) VALUES(?,?,?,1,?) "
"ON CONFLICT(name) DO UPDATE SET "
"title_ko=CASE WHEN excluded.title_ko!='' THEN excluded.title_ko ELSE program.title_ko END, "
"source_hash=excluded.source_hash, has_source=1, summary_status='stale', "
"text_symbols_json=excluded.text_symbols_json",
(program, title, source_hash, text_symbols),
)
# 재적재 전, code_hash 가 같은 unit 의 기존 요약(얇은 색인)·로직 조각을 보존해 둔다
kept_summaries = {
r["unit_id"]: (r["code_hash"], r["summary_json"], r["prompt_version"], r["chunk_count"])
for r in con.execute(
"SELECT unit_id, code_hash, summary_json, prompt_version, chunk_count FROM unit "
"WHERE program=? AND summary_status='done'", (program,)
).fetchall()
}
# 기존 구조 데이터 삭제 후 재적재 (unit 단위 증분은 v2)
for table in ("include", "unit", "symbol", "declaration", "symbol_write", "symbol_read",
"table_ref", "call_edge", "topo"):
con.execute(f"DELETE FROM {table} WHERE program=?", (program,))
con.execute("DELETE FROM unit_fts WHERE program=?", (program,))
norm_dir = settings.data_normalized / program
for inc in d["includes"]:
code_file = norm_dir / f"{inc['include']}.abap"
code = code_file.read_text(encoding="utf-8") if code_file.exists() else ""
con.execute(
"INSERT INTO include(program, include, code_hash, line_count, code) VALUES(?,?,?,?,?)",
(program, inc["include"], inc["sha256"], inc["line_count"], code),
)
kept_unit_ids: set[str] = set()
for u in d["units"]:
con.execute(
"INSERT INTO unit(unit_id, program, include, unit_type, name, line_start, line_end, "
"code_hash, signature, header_comment, loc, refs_json, sub_chunks_json) "
"VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)",
(u["unit_id"], program, u["include"], u["unit_type"], u["name"],
u["line_start"], u["line_end"], u["code_hash"], u.get("signature", ""),
u.get("header_comment", ""), u.get("loc", 0),
json.dumps(u.get("refs"), ensure_ascii=False) if u.get("refs") else None,
json.dumps(u.get("sub_chunks", []), ensure_ascii=False)),
)
kept = kept_summaries.get(u["unit_id"])
if kept and kept[0] == u["code_hash"]:
con.execute(
"UPDATE unit SET summary_json=?, summary_status='done', prompt_version=?, chunk_count=? "
"WHERE unit_id=?",
(kept[1], kept[2], kept[3] or 0, u["unit_id"]),
)
kept_unit_ids.add(u["unit_id"])
refs = u.get("refs") or {}
for t in refs.get("tables_read", []):
con.execute("INSERT INTO table_ref VALUES(?,?,?,?,?)", (program, u["unit_id"], t, "read", 0))
for t in refs.get("tables_write", []):
con.execute("INSERT INTO table_ref VALUES(?,?,?,?,?)", (program, u["unit_id"], t, "write", 0))
# unit FTS — 얇은 색인 (이름/시그니처/주석; 요약이 있으면 한 줄 요약도)
# 장식 문자('----* INITIALIZATION ----*')를 그대로 넣으면 색인이 쓰레기가 된다 (수정사항 1번)
purpose = clean_comment(u.get("header_comment", ""))
if kept and kept[0] == u["code_hash"] and kept[1]:
purpose = (json.loads(kept[1]).get("purpose_ko") or purpose)
text = " ".join(filter(None, [u["name"], u.get("signature", ""), purpose]))
con.execute(
"INSERT INTO unit_fts(unit_id, program, name, purpose, keywords, bigrams) VALUES(?,?,?,?,?,?)",
(u["unit_id"], program, u["name"], purpose, "", bigrams(text)),
)
# 로직 조각: code_hash 가 같은 unit 의 조각만 남기고 나머지는 삭제 (재추출 대상)
stale_chunks = [
r["chunk_id"] for r in con.execute(
"SELECT chunk_id, unit_id FROM logic_chunk WHERE program=?", (program,)
).fetchall() if r["unit_id"] not in kept_unit_ids
]
for cid in stale_chunks:
con.execute("DELETE FROM logic_chunk WHERE chunk_id=?", (cid,))
con.execute("DELETE FROM chunk_fts WHERE chunk_id=?", (cid,))
for s in d["symbols"]:
con.execute(
"INSERT INTO symbol VALUES(?,?,?,?,?,?,?,?,?)",
(program, s["name"], s["scope"], s.get("unit_id"), s.get("decl_include", ""),
s.get("decl_line", 0), s.get("type_text", ""), s.get("ddic_ref", ""), s.get("kind", "")),
)
# 정의부 — 붙여넣기용 선언 원문 (parser/declarations.py)
for dc in d.get("declarations", []):
con.execute(
"INSERT INTO declaration(program, name, kind, scope, unit_id, include, line_start, "
"line_end, code, type_text, depends_json) VALUES(?,?,?,?,?,?,?,?,?,?,?)",
(program, dc["name"], dc["kind"], dc["scope"], dc.get("unit_id"), dc["include"],
dc["line_start"], dc["line_end"], dc["code"], dc.get("type_text", ""),
json.dumps(dc.get("depends", []), ensure_ascii=False)),
)
for w in d["writes"]:
con.execute(
"INSERT INTO symbol_write(program, symbol, unit_id, include, line, kind, "
"source_symbols, source_tables, stmt_text, callee) VALUES(?,?,?,?,?,?,?,?,?,?)",
(program, w["symbol"], w["unit_id"], w["include"], w["line"], w["kind"],
json.dumps(w.get("source_symbols", [])), json.dumps(w.get("source_tables", [])),
w.get("stmt_text", ""), w.get("callee", "")),
)
for r in d["reads"]:
con.execute(
"INSERT INTO symbol_read VALUES(?,?,?,?,?)",
(program, r["symbol"], r["unit_id"], r["include"], r["line"]),
)
for e in d["call_edges"]:
con.execute(
"INSERT INTO call_edge VALUES(?,?,?,?,?,?)",
(program, e["from_unit"], e.get("to_unit"), e.get("external_name"),
e["call_type"], e.get("line", 0)),
)
for i, uid in enumerate(d.get("topo_order", [])):
con.execute("INSERT OR REPLACE INTO topo VALUES(?,?,?)", (program, uid, i))
return "loaded"
def _identity_row(row) -> tuple:
"""program_fts — 이름·타이틀·패키지만. **짧게 유지한다.**
요약을 여기에 섞으면 bm25 의 길이 정규화 때문에 요약이 있는 프로그램이 오히려 밀린다
(index/db.py 의 주석 참고). 서술 텍스트는 program_desc_fts 로 분리한다.
"""
title = row["title_ko"] or ""
text = " ".join(filter(None, [title, row["pkg_text"]]))
return (row["name"], title,
" ".join(filter(None, [row["devclass"] or "", row["pkg_text"] or ""])),
bigrams(text))
def _desc_row(con, row) -> tuple | None:
"""program_desc_fts — LLM 요약·조각 키워드·테이블·텍스트 심볼 (수정사항 1·3번).
"입고 처리하는 프로그램" 처럼 이름·타이틀에 없는 말로 물어도 걸리게 하는 색인이다.
넣을 게 없으면 None (빈 행을 만들면 길이 정규화만 왜곡된다).
"""
name = row["name"]
summary = loads(row["summary_json"]) or {}
purpose = " ".join(filter(None, [
summary.get("business_purpose_ko") or "",
summary.get("business_purpose_en") or "",
" ".join(summary.get("main_flow") or []),
]))
kw: list[str] = list(summary.get("keywords_ko") or []) + list(summary.get("keywords_en") or [])
kw += list(summary.get("business_tags") or [])
if summary.get("sap_module"):
kw.append(summary["sap_module"])
objs: list[str] = list(summary.get("related_tcodes") or [])
for r in con.execute(
"SELECT keywords_ko, keywords_en, sap_objects FROM logic_chunk WHERE program=?", (name,)
):
kw += (loads(r["keywords_ko"]) or []) + (loads(r["keywords_en"]) or [])
objs += loads(r["sap_objects"]) or []
objs += [t["table_name"] for t in con.execute(
"SELECT DISTINCT table_name FROM table_ref WHERE program=? LIMIT 200", (name,))]
# 텍스트 심볼의 한국어 원문 — LLM 없이도 자연어 앵커가 생긴다 (수정사항 3번)
ts = [t.get("text", "") for t in (loads(row["text_symbols_json"]) or []) if t.get("text")]
kw = list(dict.fromkeys(k for k in kw if k))[:200]
objs = list(dict.fromkeys(o.upper() for o in objs if o))[:200]
purpose_text = " ".join(filter(None, [purpose, " ".join(ts)]))
if not (purpose_text or kw or objs):
return None
return (name, purpose_text, " ".join(kw), " ".join(objs),
bigrams(" ".join(filter(None, [purpose, " ".join(kw), " ".join(ts)]))))
_PROGRAM_FTS_SQL = (
"SELECT p.name, p.title_ko, p.devclass, p.summary_json, p.text_symbols_json, "
"COALESCE(k.text_ko,'') AS pkg_text FROM program p "
"LEFT JOIN package k ON k.devclass = p.devclass"
)
def refresh_program_fts(con, program: str | None = None) -> int:
"""program_fts + program_desc_fts 재구축. program 을 주면 그 한 건만 갱신한다.
전량 재구축을 /ingest 마다 하면 1만 본에서 O(N²) 가 된다 — 단건 경로가 필요하다.
"""
if program:
name = program.upper()
con.execute("DELETE FROM program_fts WHERE name=?", (name,))
con.execute("DELETE FROM program_desc_fts WHERE name=?", (name,))
rows = con.execute(_PROGRAM_FTS_SQL + " WHERE p.name=?", (name,)).fetchall()
else:
con.execute("DELETE FROM program_fts")
con.execute("DELETE FROM program_desc_fts")
rows = con.execute(_PROGRAM_FTS_SQL).fetchall()
for row in rows:
con.execute(
"INSERT INTO program_fts(name, title, keywords, bigrams) VALUES(?,?,?,?)",
_identity_row(row),
)
desc = _desc_row(con, row)
if desc:
con.execute(
"INSERT INTO program_desc_fts(name, purpose, keywords, objects, bigrams) "
"VALUES(?,?,?,?,?)",
desc,
)
return len(rows)
def main() -> None:
ap = argparse.ArgumentParser(description="Stage 4 — 인덱스 적재")
ap.add_argument("--program", default=None)
ap.add_argument("--limit", type=int, default=None)
ap.add_argument("--force", action="store_true",
help="source_hash 가 같아도 재적재 (파서·색인 로직이 바뀐 경우)")
args = ap.parse_args()
con = connect()
n_pkg = load_packages(con)
n_tcode = load_tcodes(con)
parsed = sorted(settings.data_parsed.glob("*.parse.json")) if settings.data_parsed.exists() else []
if args.program:
parsed = [p for p in parsed if p.stem.replace(".parse", "") == args.program.upper()]
if args.limit:
parsed = parsed[: args.limit]
stats = {"package_rows": n_pkg, "tcodes": n_tcode, "loaded": 0, "skipped": 0, "errors": 0}
con.commit()
for p in parsed:
try:
r = load_parsed(con, p, force=args.force)
con.commit() # 프로그램 단위 커밋 — 실패 시 해당 프로그램만 롤백
stats["loaded" if r == "loaded" else "skipped"] += 1
except Exception as e: # noqa: BLE001
con.rollback()
stats["errors"] += 1
print(f"[FAIL] {p.name}: {type(e).__name__}: {e}")
refresh_program_fts(con)
con.commit()
con.close()
print(json.dumps(stats, ensure_ascii=False))
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink