diff --git a/ingest/normalize.py b/ingest/normalize.py index 10c5cff..02be242 100644 --- a/ingest/normalize.py +++ b/ingest/normalize.py @@ -33,10 +33,16 @@ def parse_collected_file(raw: str) -> dict | list: return json.loads(normalize_raw_text(raw)) -def clean_text_field(value: str | None) -> str: - """DESCRIPTION/TEXT 등 텍스트 필드의 잔여 아티팩트(연속 공백) 정리.""" - if not value: +def clean_text_field(value: object) -> str: + """DESCRIPTION/TEXT 등 텍스트 필드의 잔여 아티팩트(연속 공백) 정리. + + JSON 규약(2026-09-14)의 SAP 응답은 숫자·null 이 섞여 온다(예: TOTAL_ROWS, 빈 날짜). + 문자열이 아니어도 죽지 않고 문자열로 바꿔 정리한다 (고객사 실측: CHANGED_ON 에서 TypeError). + """ + if value is None or value is False: return "" + if not isinstance(value, str): + value = str(value) return re.sub(r"\s+", " ", value).strip() @@ -106,41 +112,11 @@ def run(raw_dir: Path, out_dir: Path, limit: int | None = None, dry_run: bool = stats["files"] += 1 try: data = parse_collected_file(path.read_text(encoding="utf-8")) + _handle_file(path, data, out_dir, dry_run, stats, package_rows, tcode_rows, error_lines) except Exception as e: # noqa: BLE001 — 실패 기록 후 계속 (계획서 §3) stats["errors"] += 1 error_lines.append(f"{path}\t{type(e).__name__}: {e}") - continue - - kind = classify(data) - if kind == "package_list": - for row in data: - package_rows.append( - { - "devclass": clean_text_field(row.get("DEVCLASS")).upper(), - "obj_name": clean_text_field(row.get("OBJ_NAME")).upper(), - "text": clean_text_field(row.get("TEXT")), - "created_on": clean_text_field(row.get("CREATED_ON")), - "changed_on": clean_text_field(row.get("CHANGED_ON")), - } - ) - stats["package_rows"] += len(data) - elif kind == "program_source": - if dry_run: - stats["programs"] += 1 - stats["includes"] += len(data.get("INCLUDE_PROGRAM", [])) - else: - meta = write_program(data, out_dir) - stats["programs"] += 1 - stats["includes"] += len(meta["includes"]) - stats["total_lines"] += meta["total_lines"] - for t in data.get("TCODE_LIST") or []: - tcode = clean_text_field(t.get("TCODE")).upper() - if tcode: - tcode_rows.append({"tcode": tcode, "program": meta["program"], - "text_ko": clean_text_field(t.get("TTEXT")) or meta["description"]}) - else: - stats["errors"] += 1 - error_lines.append(f"{path}\tUNKNOWN_FORMAT") + print(f"[FAIL] {path.name}: {type(e).__name__}: {e}", file=sys.stderr) if not dry_run: if package_rows: @@ -157,6 +133,41 @@ def run(raw_dir: Path, out_dir: Path, limit: int | None = None, dry_run: bool = return stats +def _handle_file(path: Path, data, out_dir: Path, dry_run: bool, stats: dict, + package_rows: list, tcode_rows: list, error_lines: list) -> None: + """파일 한 건 분류·처리. 예외는 호출자가 기록하고 다음 파일로 넘어간다.""" + kind = classify(data) + if kind == "package_list": + for row in data: + package_rows.append( + { + "devclass": clean_text_field(row.get("DEVCLASS")).upper(), + "obj_name": clean_text_field(row.get("OBJ_NAME")).upper(), + "text": clean_text_field(row.get("TEXT")), + "created_on": clean_text_field(row.get("CREATED_ON")), + "changed_on": clean_text_field(row.get("CHANGED_ON")), + } + ) + stats["package_rows"] += len(data) + elif kind == "program_source": + if dry_run: + stats["programs"] += 1 + stats["includes"] += len(data.get("INCLUDE_PROGRAM", [])) + else: + meta = write_program(data, out_dir) + stats["programs"] += 1 + stats["includes"] += len(meta["includes"]) + stats["total_lines"] += meta["total_lines"] + for t in data.get("TCODE_LIST") or []: + tcode = clean_text_field(t.get("TCODE")).upper() + if tcode: + tcode_rows.append({"tcode": tcode, "program": meta["program"], + "text_ko": clean_text_field(t.get("TTEXT")) or meta["description"]}) + else: + stats["errors"] += 1 + error_lines.append(f"{path}\tUNKNOWN_FORMAT") + + def main() -> None: ap = argparse.ArgumentParser(description="Stage 1 — 수집 JSON 정규화") ap.add_argument("raw_dir", nargs="?", default=None, help="수집 원본 디렉토리 (기본 data/raw)") diff --git a/tests/test_from_sap.py b/tests/test_from_sap.py index 9dfaf18..c1358e1 100644 --- a/tests/test_from_sap.py +++ b/tests/test_from_sap.py @@ -149,3 +149,26 @@ def test_sap_call_sends_fields_without_wrapper(monkeypatch): assert captured["body"] == {"IV_PACKAGE": "ZFI01", "IV_MAX_ROWS": 5} # 래퍼 없음, 빈 값 제외, 숫자 유지 assert captured["url"].endswith("/GET_PROGRAM_LIST") assert res["return"]["type"] == "S" and res["parsed"]["RESULT"] == [] + + +def test_normalize_survives_non_string_fields_and_bad_file(tmp_path): + """JSON 규약 응답엔 숫자·null 이 섞인다. 파일 하나가 깨져도 나머지는 계속 처리해야 한다.""" + from ingest.normalize import clean_text_field + assert clean_text_field(None) == "" and clean_text_field(20240102) == "20240102" and clean_text_field(0) == "0" + + raw, out = tmp_path / "raw", tmp_path / "normalized" + raw.mkdir() + (raw / "ZPKG.txt").write_text(json.dumps([ + {"DEVCLASS": "ZPKG", "OBJ_NAME": "ZA", "TEXT": None, "CREATED_ON": "", "CHANGED_ON": 20240102}, + {"DEVCLASS": "ZPKG", "OBJ_NAME": "ZB", "TEXT": "b", "CREATED_ON": None, "CHANGED_ON": None}, + ]), encoding="utf-8") + (raw / "ZBAD.txt").write_text('{"MAIN_PROGRAM": "ZBAD", "INCLUDE_PROGRAM": "not-a-list"}', encoding="utf-8") + (raw / "ZOK.txt").write_text(json.dumps({"MAIN_PROGRAM": "ZOK", "DESCRIPTION": "ok", + "INCLUDE_PROGRAM": [{"INCLUDE": "ZOK", "SOURCE_CODE": "REPORT zok.\n"}]}), + encoding="utf-8") + stats = normalize_run(raw, out) + assert stats["programs"] == 1 and stats["package_rows"] == 2 and stats["errors"] == 1 + assert (out / "ZOK" / "ZOK.abap").exists() + rows = [json.loads(l) for l in (out / "packages.jsonl").read_text(encoding="utf-8").splitlines()] + assert rows[0]["changed_on"] == "20240102" and rows[0]["text"] == "" and rows[1]["changed_on"] == "" + assert "ZBAD.txt" in (out / "_errors.log").read_text(encoding="utf-8")