From 438b2f69f6598dba716f5842bcccf1d177c306e3 Mon Sep 17 00:00:00 2001 From: toki Date: Wed, 5 Aug 2026 14:13:58 +0900 Subject: [PATCH] =?UTF-8?q?fix(agent-ops):=20=ED=86=A0=ED=81=B0=20?= =?UTF-8?q?=EB=B0=9C=EA=B8=89=20=EC=9E=AC=EC=97=B0=EA=B2=B0=20=EA=B2=80?= =?UTF-8?q?=EC=A6=9D=EC=9D=84=20=EB=B3=B4=EA=B0=95=ED=95=9C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../project/openai-usage-token-issue/SKILL.md | 1 + .../scripts/issue_token.py | 98 ++++++++++++++++--- 2 files changed, 85 insertions(+), 14 deletions(-) diff --git a/agent-ops/skills/project/openai-usage-token-issue/SKILL.md b/agent-ops/skills/project/openai-usage-token-issue/SKILL.md index 24d92095..0e4e43a0 100644 --- a/agent-ops/skills/project/openai-usage-token-issue/SKILL.md +++ b/agent-ops/skills/project/openai-usage-token-issue/SKILL.md @@ -104,6 +104,7 @@ python3 agent-ops/skills/project/openai-usage-token-issue/scripts/issue_token.py - raw token은 remote process argument에 넣지 않고 SSH stdin payload로만 전달한다. - Edge config에는 `token_ref`, SHA-256 hash, `principal_ref`, `principal_alias`만 기록한다. - `openai.principal_tokens[]` 변경은 restart-required로 처리한다. candidate check, cutover, exact listener identity 확인, restart, rollback을 생략하지 않는다. +- Edge 재시작 직후 Node 재연결 유예를 위해 chat smoke의 HTTP `502`/`503`/`504`만 총 32초 이내의 제한된 backoff로 재시도한다. 다른 HTTP 오류는 재시도하지 않고 기존 rollback 경계를 유지한다. - dev-corp Confluence 표에는 사용자, alias, token ref, 상태, 동기화 시각만 기록한다. raw token, token hash, Authorization, provider credential을 넣지 않는다. - Confluence write는 최신 version에 한 번만 수행하고 409를 포함한 실패를 자동 재시도하지 않는다. - Confluence 실패는 활성화된 Edge/store를 되돌리지 않고 clipboard 전달을 막아 동일 command로 재개한다. diff --git a/agent-ops/skills/project/openai-usage-token-issue/scripts/issue_token.py b/agent-ops/skills/project/openai-usage-token-issue/scripts/issue_token.py index 0fc89e77..e2d9add7 100644 --- a/agent-ops/skills/project/openai-usage-token-issue/scripts/issue_token.py +++ b/agent-ops/skills/project/openai-usage-token-issue/scripts/issue_token.py @@ -29,7 +29,7 @@ import urllib.request from contextlib import contextmanager from html.parser import HTMLParser from pathlib import Path -from typing import Any, NoReturn, cast +from typing import Any, Callable, NoReturn, cast ATLASSIAN_BASE_URL = "https://lgucorp.atlassian.net" CONFLUENCE_FOLDER_ID = "650886407" @@ -37,6 +37,8 @@ CONFLUENCE_TITLE = "IOP 계정 발급 현황" MANAGED_HEADING = "IOP 사용자 토큰 발급 현황" TABLE_HEADERS = ("사용자", "principal alias", "token ref", "상태", "동기화 시각") SUPPORTED_ENVIRONMENTS = ("dev", "dev-corp") +OPENAI_RECONNECT_RETRY_STATUSES = frozenset({502, 503, 504}) +OPENAI_RECONNECT_RETRY_DELAYS_SECONDS = (1, 2, 3, 5, 8, 13) class WorkflowFailure(RuntimeError): @@ -51,6 +53,12 @@ class ConfluenceHTTPFailure(WorkflowFailure): super().__init__(f"confluence_http_{status_code}") +class OpenAIHTTPFailure(WorkflowFailure): + def __init__(self, status_code: int): + self.status_code = status_code + super().__init__("openai_smoke_http_failed") + + class CurlTransportFailure(WorkflowFailure): def __init__(self): super().__init__("curl_transport_failed") @@ -838,7 +846,7 @@ def api_json( except CurlTransportFailure: fail("openai_smoke_network_failed") if status_code < 200 or status_code >= 300: - fail("openai_smoke_http_failed") + raise OpenAIHTTPFailure(status_code) try: value = json.loads(data) except json.JSONDecodeError: @@ -848,6 +856,25 @@ def api_json( return value +def with_openai_reconnect_retry( + request: Callable[[], dict[str, Any]], + *, + retry_delays: tuple[int, ...] = OPENAI_RECONNECT_RETRY_DELAYS_SECONDS, + sleep: Callable[[float], None] = time.sleep, +) -> dict[str, Any]: + for attempt in range(len(retry_delays) + 1): + try: + return request() + except OpenAIHTTPFailure as error: + if ( + error.status_code not in OPENAI_RECONNECT_RETRY_STATUSES + or attempt == len(retry_delays) + ): + raise + sleep(retry_delays[attempt]) + fail("openai_smoke_retry_invalid") + + def api_smoke(root: Path, profile: dict[str, Any], raw_token: str) -> None: if profile["openai_smoke_transport"] == "ssh-loopback": remote_call( @@ -862,18 +889,22 @@ def api_smoke(root: Path, profile: dict[str, Any], raw_token: str) -> None: models = api_json(root, profile, "/models", raw_token, timeout=15) if not isinstance(models.get("data"), list) or not models["data"]: fail("openai_models_invalid") - response = api_json( - root, - profile, - "/chat/completions", - raw_token, - { - "model": profile["smoke_model"], - "messages": [{"role": "user", "content": "Reply with the single word OK."}], - "max_tokens": 2048, - "temperature": 0, - }, - timeout=120, + response = with_openai_reconnect_retry( + lambda: api_json( + root, + profile, + "/chat/completions", + raw_token, + { + "model": profile["smoke_model"], + "messages": [ + {"role": "user", "content": "Reply with the single word OK."} + ], + "max_tokens": 2048, + "temperature": 0, + }, + timeout=120, + ) ) choices = response.get("choices") if not isinstance(choices, list) or not choices or not isinstance(choices[0], dict): @@ -1216,6 +1247,45 @@ def selftest() -> dict[str, Any]: fail("selftest_alias_failed") if normalize_alias("a@example.invalid", None) != "a": fail("selftest_short_alias_failed") + retry_attempts = 0 + retry_sleeps: list[float] = [] + + def transient_request() -> dict[str, Any]: + nonlocal retry_attempts + retry_attempts += 1 + if retry_attempts < 3: + raise OpenAIHTTPFailure(502) + return {"status": "ok"} + + retry_result = with_openai_reconnect_retry( + transient_request, + retry_delays=(1, 2), + sleep=retry_sleeps.append, + ) + if ( + retry_result.get("status") != "ok" + or retry_attempts != 3 + or retry_sleeps != [1, 2] + ): + fail("selftest_openai_retry_failed") + non_retryable_attempts = 0 + + def non_retryable_request() -> dict[str, Any]: + nonlocal non_retryable_attempts + non_retryable_attempts += 1 + raise OpenAIHTTPFailure(401) + + try: + with_openai_reconnect_retry( + non_retryable_request, + retry_delays=(1,), + sleep=lambda _delay: fail("selftest_openai_non_retryable_slept"), + ) + except OpenAIHTTPFailure as error: + if error.status_code != 401 or non_retryable_attempts != 1: + raise + else: + fail("selftest_openai_non_retryable_failed") try: parse_request('{"env":"dev","principal_ref":null}') except WorkflowFailure as error: