fix(agent-ops): 토큰 발급 재연결 검증을 보강한다

agent-ops/skills/project/openai-usage-token-issue/SKILL.md

@@ -104,6 +104,7 @@ python3 agent-ops/skills/project/openai-usage-token-issue/scripts/issue_token.py
 - raw token은 remote process argument에 넣지 않고 SSH stdin payload로만 전달한다.
 - Edge config에는 `token_ref`, SHA-256 hash, `principal_ref`, `principal_alias`만 기록한다.
 - `openai.principal_tokens[]` 변경은 restart-required로 처리한다. candidate check, cutover, exact listener identity 확인, restart, rollback을 생략하지 않는다.
+- Edge 재시작 직후 Node 재연결 유예를 위해 chat smoke의 HTTP `502`/`503`/`504`만 총 32초 이내의 제한된 backoff로 재시도한다. 다른 HTTP 오류는 재시도하지 않고 기존 rollback 경계를 유지한다.
 - dev-corp Confluence 표에는 사용자, alias, token ref, 상태, 동기화 시각만 기록한다. raw token, token hash, Authorization, provider credential을 넣지 않는다.
 - Confluence write는 최신 version에 한 번만 수행하고 409를 포함한 실패를 자동 재시도하지 않는다.
 - Confluence 실패는 활성화된 Edge/store를 되돌리지 않고 clipboard 전달을 막아 동일 command로 재개한다.

agent-ops/skills/project/openai-usage-token-issue/scripts/issue_token.py

@@ -29,7 +29,7 @@ import urllib.request
 from contextlib import contextmanager
 from html.parser import HTMLParser
 from pathlib import Path
-from typing import Any, NoReturn, cast
+from typing import Any, Callable, NoReturn, cast
 
 ATLASSIAN_BASE_URL = "https://lgucorp.atlassian.net"
 CONFLUENCE_FOLDER_ID = "650886407"
@@ -37,6 +37,8 @@ CONFLUENCE_TITLE = "IOP 계정 발급 현황"
 MANAGED_HEADING = "IOP 사용자 토큰 발급 현황"
 TABLE_HEADERS = ("사용자", "principal alias", "token ref", "상태", "동기화 시각")
 SUPPORTED_ENVIRONMENTS = ("dev", "dev-corp")
+OPENAI_RECONNECT_RETRY_STATUSES = frozenset({502, 503, 504})
+OPENAI_RECONNECT_RETRY_DELAYS_SECONDS = (1, 2, 3, 5, 8, 13)
 
 
 class WorkflowFailure(RuntimeError):
@@ -51,6 +53,12 @@ class ConfluenceHTTPFailure(WorkflowFailure):
         super().__init__(f"confluence_http_{status_code}")
 
 
+class OpenAIHTTPFailure(WorkflowFailure):
+    def __init__(self, status_code: int):
+        self.status_code = status_code
+        super().__init__("openai_smoke_http_failed")
+
+
 class CurlTransportFailure(WorkflowFailure):
     def __init__(self):
         super().__init__("curl_transport_failed")
@@ -838,7 +846,7 @@ def api_json(
     except CurlTransportFailure:
         fail("openai_smoke_network_failed")
     if status_code < 200 or status_code >= 300:
-        fail("openai_smoke_http_failed")
+        raise OpenAIHTTPFailure(status_code)
     try:
         value = json.loads(data)
     except json.JSONDecodeError:
@@ -848,6 +856,25 @@ def api_json(
     return value
 
 
+def with_openai_reconnect_retry(
+    request: Callable[[], dict[str, Any]],
+    *,
+    retry_delays: tuple[int, ...] = OPENAI_RECONNECT_RETRY_DELAYS_SECONDS,
+    sleep: Callable[[float], None] = time.sleep,
+) -> dict[str, Any]:
+    for attempt in range(len(retry_delays) + 1):
+        try:
+            return request()
+        except OpenAIHTTPFailure as error:
+            if (
+                error.status_code not in OPENAI_RECONNECT_RETRY_STATUSES
+                or attempt == len(retry_delays)
+            ):
+                raise
+            sleep(retry_delays[attempt])
+    fail("openai_smoke_retry_invalid")
+
+
 def api_smoke(root: Path, profile: dict[str, Any], raw_token: str) -> None:
     if profile["openai_smoke_transport"] == "ssh-loopback":
         remote_call(
@@ -862,18 +889,22 @@ def api_smoke(root: Path, profile: dict[str, Any], raw_token: str) -> None:
     models = api_json(root, profile, "/models", raw_token, timeout=15)
     if not isinstance(models.get("data"), list) or not models["data"]:
         fail("openai_models_invalid")
-    response = api_json(
-        root,
-        profile,
-        "/chat/completions",
-        raw_token,
-        {
-            "model": profile["smoke_model"],
-            "messages": [{"role": "user", "content": "Reply with the single word OK."}],
-            "max_tokens": 2048,
-            "temperature": 0,
-        },
-        timeout=120,
+    response = with_openai_reconnect_retry(
+        lambda: api_json(
+            root,
+            profile,
+            "/chat/completions",
+            raw_token,
+            {
+                "model": profile["smoke_model"],
+                "messages": [
+                    {"role": "user", "content": "Reply with the single word OK."}
+                ],
+                "max_tokens": 2048,
+                "temperature": 0,
+            },
+            timeout=120,
+        )
     )
     choices = response.get("choices")
     if not isinstance(choices, list) or not choices or not isinstance(choices[0], dict):
@@ -1216,6 +1247,45 @@ def selftest() -> dict[str, Any]:
         fail("selftest_alias_failed")
     if normalize_alias("a@example.invalid", None) != "a":
         fail("selftest_short_alias_failed")
+    retry_attempts = 0
+    retry_sleeps: list[float] = []
+
+    def transient_request() -> dict[str, Any]:
+        nonlocal retry_attempts
+        retry_attempts += 1
+        if retry_attempts < 3:
+            raise OpenAIHTTPFailure(502)
+        return {"status": "ok"}
+
+    retry_result = with_openai_reconnect_retry(
+        transient_request,
+        retry_delays=(1, 2),
+        sleep=retry_sleeps.append,
+    )
+    if (
+        retry_result.get("status") != "ok"
+        or retry_attempts != 3
+        or retry_sleeps != [1, 2]
+    ):
+        fail("selftest_openai_retry_failed")
+    non_retryable_attempts = 0
+
+    def non_retryable_request() -> dict[str, Any]:
+        nonlocal non_retryable_attempts
+        non_retryable_attempts += 1
+        raise OpenAIHTTPFailure(401)
+
+    try:
+        with_openai_reconnect_retry(
+            non_retryable_request,
+            retry_delays=(1,),
+            sleep=lambda _delay: fail("selftest_openai_non_retryable_slept"),
+        )
+    except OpenAIHTTPFailure as error:
+        if error.status_code != 401 or non_retryable_attempts != 1:
+            raise
+    else:
+        fail("selftest_openai_non_retryable_failed")
     try:
         parse_request('{"env":"dev","principal_ref":null}')
     except WorkflowFailure as error: