fix(gateway): FabriX 가 맞는 인증 조합에도 간헐 401 — 4개 조합 뒤 유력 조합 1초 쉬고 한 번 더

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
lee-hyeon-cheol
2026-09-21 19:41:41 +09:00
co-authored by Claude Fable 5.1
parent 9bd0849f27
commit b08afa389a
2 changed files with 29 additions and 0 deletions
+9
View File
@@ -9,6 +9,7 @@
from __future__ import annotations from __future__ import annotations
import asyncio
import json import json
import logging import logging
from typing import AsyncIterator from typing import AsyncIterator
@@ -26,6 +27,7 @@ log = logging.getLogger(__name__)
TRANSPORT: httpx.AsyncBaseTransport | None = None TRANSPORT: httpx.AsyncBaseTransport | None = None
# 마지막에 통과한 인증 형식 — 프로세스당 하나. 다음 요청은 이것부터 시도. # 마지막에 통과한 인증 형식 — 프로세스당 하나. 다음 요청은 이것부터 시도.
_last_ok: dict[str, Variant | None] = {"variant": None} _last_ok: dict[str, Variant | None] = {"variant": None}
RETRY_PAUSE_S = 1.0 # 마지막 재시도 전 쉬는 시간
def _cfg() -> FabrixConfig: def _cfg() -> FabrixConfig:
@@ -90,6 +92,9 @@ async def chat_completions(request: HttpRequest) -> HttpResponse:
kinds = sorted({p.get("type", "?") for m in payload.get("messages") or [] for p in (m.get("content") if isinstance(m.get("content"), list) else [])}) kinds = sorted({p.get("type", "?") for m in payload.get("messages") or [] for p in (m.get("content") if isinstance(m.get("content"), list) else [])})
log.info("ito → model=%s parts=%s stream=%s", headers.get("x-llm-model-id"), kinds or ["text"], bool(body.get("stream"))) log.info("ito → model=%s parts=%s stream=%s", headers.get("x-llm-model-id"), kinds or ["text"], bool(body.get("stream")))
variants = auth_variants(headers, _last_ok["variant"]) variants = auth_variants(headers, _last_ok["variant"])
# FabriX 가 맞는 조합에도 가끔 401 을 뱉음(2026-09-21 고객사 실측: curl 10번 중 1~2번).
# 4개 조합 다 돌고 나서 제일 유력한 조합(맨 앞)을 잠깐 쉬고 한 번 더 — 그래도 401 이면 진짜 인증 문제
variants = variants + variants[:1]
if body.get("stream"): if body.get("stream"):
@@ -103,6 +108,8 @@ async def chat_completions(request: HttpRequest) -> HttpResponse:
log.info("ito ← %s (auth %s/%s: %s)", resp.status_code, i + 1, len(variants), vkey) log.info("ito ← %s (auth %s/%s: %s)", resp.status_code, i + 1, len(variants), vkey)
if resp.status_code == 401 and i + 1 < len(variants): if resp.status_code == 401 and i + 1 < len(variants):
await resp.aclose() # 인증 형식이 안 맞은 것 — 다른 조합으로 한 번 더 await resp.aclose() # 인증 형식이 안 맞은 것 — 다른 조합으로 한 번 더
if i + 2 == len(variants):
await asyncio.sleep(RETRY_PAUSE_S)
continue continue
break break
assert resp is not None assert resp is not None
@@ -126,6 +133,8 @@ async def chat_completions(request: HttpRequest) -> HttpResponse:
for i, (vkey, vh) in enumerate(variants): for i, (vkey, vh) in enumerate(variants):
resp = await client.post(cfg.url, headers=vh, json=body) resp = await client.post(cfg.url, headers=vh, json=body)
if resp.status_code == 401 and i + 1 < len(variants): if resp.status_code == 401 and i + 1 < len(variants):
if i + 2 == len(variants):
await asyncio.sleep(RETRY_PAUSE_S)
continue continue
if resp.status_code < 400: if resp.status_code < 400:
_last_ok["variant"] = vkey _last_ok["variant"] = vkey
+20
View File
@@ -126,6 +126,26 @@ async def test_401_retries_other_auth_shape_and_remembers(upstream):
assert len(calls) == n + 1 and calls[-1].headers["x-openapi-token"] == "Bearer tok" assert len(calls) == n + 1 and calls[-1].headers["x-openapi-token"] == "Bearer tok"
@override_settings(FABRIX_ENV=FULL_ENV)
async def test_flaky_401_on_right_auth_is_retried_once(upstream, monkeypatch):
"""FabriX 가 맞는 조합에도 가끔 401 — 4개 다 돈 뒤 맨 앞 조합 한 번 더."""
from apps.gateway import views
monkeypatch.setattr(views, "RETRY_PAUSE_S", 0)
hits = {"n": 0}
def handler(req: httpx.Request) -> httpx.Response:
hits["n"] += 1
if hits["n"] <= 4:
return httpx.Response(401, content=b"flaky")
return httpx.Response(200, json={"choices": []})
calls = upstream(handler)
resp = await _post({"messages": []})
assert resp.status_code == 200 and len(calls) == 5
assert calls[0].headers.get("x-openapi-token") == calls[4].headers.get("x-openapi-token")
@override_settings(FABRIX_ENV=FULL_ENV) @override_settings(FABRIX_ENV=FULL_ENV)
async def test_stream_upstream_error_becomes_sse_error(upstream): async def test_stream_upstream_error_becomes_sse_error(upstream):
upstream(lambda r: _sse(b"boom", 500)) upstream(lambda r: _sse(b"boom", 500))