feat(observability): 트레이스에 model(metadata) + 사용 태그 — skill:<이름> / wiki / mcp:<함수>
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
lee-hyeon-cheol
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
823292cbb6
commit
727486e7cb
@@ -228,6 +228,7 @@ async def _finalize(session: ChatSession, state: TurnState, started: float, *, f
|
||||
usage: dict | None = None
|
||||
title = state.title
|
||||
tool_parts: list[dict] = []
|
||||
model = ""
|
||||
try:
|
||||
msgs = await opencode_service.list_messages_a(session.id) or []
|
||||
assistant = [m for m in msgs if (m.get("info") or {}).get("role") == "assistant"]
|
||||
@@ -236,6 +237,7 @@ async def _finalize(session: ChatSession, state: TurnState, started: float, *, f
|
||||
parts = assistant[-1].get("parts") or []
|
||||
# 이 턴의 assistant 메시지가 여러 개(도구 왕복마다 하나)라 도구 파트는 전부 모음
|
||||
tool_parts = [p for m in assistant for p in (m.get("parts") or []) if p.get("type") == "tool"]
|
||||
model = info.get("modelID") or ""
|
||||
full = "".join(p.get("text", "") for p in parts if p.get("type") == "text" and not p.get("synthetic"))
|
||||
if full:
|
||||
content = full
|
||||
@@ -276,9 +278,9 @@ async def _finalize(session: ChatSession, state: TurnState, started: float, *, f
|
||||
t0 = datetime.fromtimestamp(started, timezone.utc).isoformat()
|
||||
root = langfuse.trace(
|
||||
tid, "chat", userId=user_email, sessionId=session.id, input=state.user_text, output=content,
|
||||
tags=["codeassist"], start=t0, end=langfuse.now_iso(),
|
||||
usage=langfuse.usage_of(usage["input"], usage["output"]) if usage else None,
|
||||
metadata={"elapsed_ms": usage["elapsed_ms"]} if usage else {}, error=failed,
|
||||
tags=["codeassist", *langfuse.usage_tags(tool_parts)], start=t0, end=langfuse.now_iso(),
|
||||
usage=langfuse.usage_of(usage["input"], usage["output"]) if usage else None, model=model,
|
||||
metadata={"model": model, **({"elapsed_ms": usage["elapsed_ms"]} if usage else {})}, error=failed,
|
||||
)
|
||||
langfuse.send_later([root, *_tool_spans(tid, root, tool_parts)])
|
||||
return usage, title
|
||||
|
||||
@@ -150,6 +150,35 @@ def tool_span(trace_id: str, name: str, *, parent: dict, input=None, output=None
|
||||
return _span(trace_id, name, attrs, start=start, end=end, parent=parent["spanId"], error=error)
|
||||
|
||||
|
||||
def usage_tags(tool_parts: list[dict]) -> list[str]:
|
||||
"""이 턴에 뭘 써서 답했는지 태그로 — Phoenix 에서 metadata.tags 로 필터. 순서 유지·중복 제거.
|
||||
skill:<이름> = OpenCode skill 도구, 또는 read 로 skills/<이름>/SKILL.md 를 연 것(ABAP_OPENCODE 방식)
|
||||
wiki = wiki/ 아래 파일을 읽음(CodeAssist 위키)
|
||||
mcp:<함수> = MCP 도구 호출. OpenCode 는 "<서버>_<함수>" 로 이름 붙임 → 서버 접두어 떼고 함수만"""
|
||||
import re
|
||||
|
||||
out: list[str] = []
|
||||
for p in tool_parts:
|
||||
tool = p.get("tool") or ""
|
||||
inp = (p.get("state") or {}).get("input") or {}
|
||||
path = str(inp.get("filePath") or inp.get("path") or "").replace("\\", "/")
|
||||
name = inp.get("name") if tool == "skill" else None
|
||||
if not name and tool == "read":
|
||||
m = re.search(r"skills/([^/]+)/SKILL\.md$", path)
|
||||
name = m.group(1) if m else None
|
||||
if name:
|
||||
out.append(f"skill:{name}")
|
||||
if "wiki/" in path:
|
||||
out.append("wiki")
|
||||
for server in MCP_SERVERS:
|
||||
if tool.startswith(server + "_"):
|
||||
out.append("mcp:" + tool[len(server) + 1:])
|
||||
return list(dict.fromkeys(out))
|
||||
|
||||
|
||||
MCP_SERVERS = ("sap-icf",) # opencode.json 의 mcp 이름들. 새 MCP 붙이면 여기 추가
|
||||
|
||||
|
||||
def ms_iso(ms: int | float | None) -> str | None:
|
||||
"""OpenCode 의 epoch ms → ISO. 없으면 None(=지금)."""
|
||||
return datetime.fromtimestamp(ms / 1000, timezone.utc).isoformat() if ms else None
|
||||
|
||||
@@ -88,8 +88,9 @@ async def test_finalize_sends_user_trace(lf_server, django_user_model, monkeypat
|
||||
async def _msgs(_sid):
|
||||
tool = {"type": "tool", "tool": "sap-icf_get_program_source", "state": {"status": "completed", "input": {"program_name": "ZTEST"}, "output": "REPORT ztest.", "title": "ZTEST", "time": {"start": 1100, "end": 1300}}}
|
||||
bad = {"type": "tool", "tool": "read", "state": {"status": "error", "input": {"filePath": "x"}, "error": "no such file", "time": {"start": 1300, "end": 1310}}}
|
||||
return [{"info": {"role": "assistant"}, "parts": [tool, bad]},
|
||||
{"info": {"role": "assistant", "tokens": {"input": 3, "output": 4, "reasoning": 1}, "time": {"created": 1000, "completed": 1500}}, "parts": [{"type": "text", "text": "답"}]}]
|
||||
wiki = {"type": "tool", "tool": "read", "state": {"status": "completed", "input": {"filePath": "/workspace/wiki/programs/ZTEST.md"}, "output": "x", "time": {"start": 1310, "end": 1320}}}
|
||||
return [{"info": {"role": "assistant"}, "parts": [tool, bad, wiki]},
|
||||
{"info": {"role": "assistant", "modelID": "z-ai/glm-5.2", "tokens": {"input": 3, "output": 4, "reasoning": 1}, "time": {"created": 1000, "completed": 1500}}, "parts": [{"type": "text", "text": "답"}]}]
|
||||
|
||||
async def _sess(_sid):
|
||||
return {"title": "MARA 조회"}
|
||||
@@ -103,11 +104,14 @@ async def test_finalize_sends_user_trace(lf_server, django_user_model, monkeypat
|
||||
await asyncio.gather(*list(langfuse._pending))
|
||||
assert len(lf_server) == 1
|
||||
spans = lf_server[0]["body"]["resourceSpans"][0]["scopeSpans"][0]["spans"]
|
||||
assert [sp["name"] for sp in spans] == ["chat", "sap-icf_get_program_source", "read"]
|
||||
assert [sp["name"] for sp in spans] == ["chat", "sap-icf_get_program_source", "read", "read"]
|
||||
t = _attrs(spans[0])
|
||||
assert t["langfuse.user.id"] == "u@x.com" and t["langfuse.session.id"] == "ses_1"
|
||||
assert t["langfuse.trace.input"] == "질문" and t["langfuse.trace.output"] == "답"
|
||||
assert t["langfuse.observation.type"] == "generation" # 루트가 곧 답변 — 토큰은 여기
|
||||
assert t["llm.model_name"] == "z-ai/glm-5.2"
|
||||
meta = json.loads(t["metadata"])
|
||||
assert meta["model"] == "z-ai/glm-5.2" and meta["tags"] == ["codeassist", "mcp:get_program_source", "wiki"]
|
||||
assert json.loads(t["langfuse.observation.usage_details"]) == {"input": 3, "output": 5, "total": 8}
|
||||
a1, a2 = _attrs(spans[1]), _attrs(spans[2])
|
||||
assert spans[1]["parentSpanId"] == spans[0]["spanId"] and a1["langfuse.observation.type"] == "tool"
|
||||
@@ -131,3 +135,19 @@ async def test_phoenix_target_no_auth_and_openinference_attrs(lf_server):
|
||||
assert r["llm.model_name"] == "581" and r["llm.token_count.total"] == "10" and r["session.id"] == "s9" and r["user.id"] == "u@x.com"
|
||||
assert tl["openinference.span.kind"] == "TOOL" and tl["tool.name"] == "read" and json.loads(tl["input.value"]) == {"filePath": "a.md"}
|
||||
|
||||
|
||||
def test_usage_tags_skill_wiki_mcp():
|
||||
parts = [
|
||||
{"tool": "skill", "state": {"input": {"name": "classic-abap-fs"}}},
|
||||
{"tool": "read", "state": {"input": {"filePath": "/workspace/skills/abap-program-analyze/SKILL.md"}}},
|
||||
{"tool": "read", "state": {"input": {"filePath": "C:\\ws\\wiki\\programs\\Z.md"}}},
|
||||
{"tool": "sap-icf_get_table_fields", "state": {"input": {"table": "MARA"}}},
|
||||
{"tool": "sap-icf_get_program_source", "state": {"input": {"program_name": "Z"}}},
|
||||
{"tool": "read", "state": {"input": {"filePath": "/workspace/skills/classic-abap-fs/glossary.md"}}}, # SKILL.md 아님
|
||||
{"tool": "grep", "state": {"input": {"pattern": "x"}}}, # 태그 없음
|
||||
{"tool": "skill", "state": {"input": {"name": "classic-abap-fs"}}}, # 중복
|
||||
]
|
||||
assert langfuse.usage_tags(parts) == [
|
||||
"skill:classic-abap-fs", "skill:abap-program-analyze", "wiki", "mcp:get_table_fields", "mcp:get_program_source",
|
||||
]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user