diff --git a/agent-client/pi/install.sh b/agent-client/pi/install.sh index e801de1a..dbb2f941 100755 --- a/agent-client/pi/install.sh +++ b/agent-client/pi/install.sh @@ -182,6 +182,8 @@ settings.setdefault("defaultThinkingLevel", "high") settings.setdefault("hideThinkingBlock", False) settings["httpIdleTimeoutMs"] = 0 retry = settings.setdefault("retry", {}) +retry["enabled"] = False +retry["maxRetries"] = 0 provider_retry = retry.setdefault("provider", {}) provider_retry.pop("timeoutMs", None) provider_retry["maxRetries"] = 0 diff --git a/agent-client/pi/install_test.py b/agent-client/pi/install_test.py index f63522a1..ae34db6c 100644 --- a/agent-client/pi/install_test.py +++ b/agent-client/pi/install_test.py @@ -86,6 +86,8 @@ class TestPreservesOpenaiResponsesConsumptionAndUserSettings(_TempHomeMixin, uni "hideThinkingBlock": True, # user preference "httpIdleTimeoutMs": 300000, # user value — must be overridden to 0 "retry": { + "enabled": True, # must be disabled + "maxRetries": 3, # must be overridden to 0 "provider": { "timeoutMs": 120000, # must be removed "maxRetries": 3, # must be overridden to 0 @@ -152,6 +154,8 @@ class TestPreservesOpenaiResponsesConsumptionAndUserSettings(_TempHomeMixin, uni self.assertNotIn("timeoutMs", provider_retry) # 8) Retry policy is set + self.assertEqual(settings["retry"]["enabled"], False) + self.assertEqual(settings["retry"]["maxRetries"], 0) self.assertEqual(provider_retry["maxRetries"], 0) self.assertEqual(provider_retry["maxRetryDelayMs"], 60000) diff --git a/agent-contract/inner/edge-config-runtime-refresh.md b/agent-contract/inner/edge-config-runtime-refresh.md index 8f68071e..15863759 100644 --- a/agent-contract/inner/edge-config-runtime-refresh.md +++ b/agent-contract/inner/edge-config-runtime-refresh.md @@ -68,7 +68,7 @@ tracked config에는 public 예시와 기본 구조만 두고, 실제 endpoint/c - `execution_presets[].single_request`는 operator-owned fixed single-request policy다. 설정 시 preset은 `allowed_modes=["light"]`, `stages=[plan, work, review]`의 승인된 plan→work→review 경로를 고수한다. 절대 상한은 `wall_clock_ms ≤ 1800000`, `timeout_ms ≤ 600000`, `max_tool_iterations ≤ 64`, `max_output_bytes ≤ 16777216`이며 `timeout_ms`는 `wall_clock_ms`를 초과할 수 없다. selector와 plan/review stage는 `reasoning_effort=high`를 강제하고 work stage는 `reasoning_effort`를 선언할 수 없다. `workspace_ref`는 비어있을 수 없으며 raw path, credential, Node id, endpoint를 포함하지 않는다. `templates` 섹션을 통해 optional `plan_file` 및 `review_file` (edge.yaml 상대 경로) 커스텀 Markdown 템플릿을 지정할 수 있으며, load 시점에 8192바이트 상한 및 문법 검증이 수행되고 생략 시 built-in default 템플릿이 적용된다. config refresh diff reporting 시 템플릿 파일 경로나 본문은 노출되지 않고 SHA-256 digest만 보고된다. single_request preset은 `workspace_tools`를 선언할 수 없다. catalog 변경과 mapping 변경은 live-apply로 분류되며 refresh 이후 새로 시작되는 logical request에만 적용된다. admitted single-request binding은 refresh 이후에도 frozen public model, stage binding, workspace reference, limits, effective templates를 유지한다. - `nodes[].providers[]`는 Node 아래 resource/provider catalog다. `category`는 `api`, `cli`, `local_inference` resource kind를 나타낸다. - `nodes[].providers[].type`의 `seulgivibe_claude`와 `seulgivibe_openai`는 runtime type을 `openai_compat`로 정규화한다. Edge가 Node adapter payload를 만들 때 명시 provider label이 없으면 원래 Seulgivibe type alias를 `OpenAICompatAdapterConfig.provider`로 보존한다. -- `nodes[].providers[].response_stall_timeout_ms`는 provider-originated response-stall timeout을 밀리초 단위로 선언한다. 양수 값은 그대로 사용되고, 0 또는 생략은 문서화된 기본값 `300000`을 적용한다. 음수 값과 safe duration bound를 초과하는 양수 값은 `NodeProviderConf.Validate()`에서 거부한다. effective 값은 `NodeProviderConf.EffectiveResponseStallTimeoutMS()`에서 계산한다. 이 필드는 config refresh에서 `restart_required`로 분류되며, effective-zero 등가성(생략 vs 명시적 0)은 변경으로 보고되지 않는다. request hard timeout, queue timeout, heartbeat/disconnect, CLI `response_idle_timeout_ms`는 기존 소유권을 유지한다. +- `nodes[].providers[].response_stall_timeout_ms`는 provider-originated response-stall timeout을 밀리초 단위로 선언한다. 양수 값은 그대로 사용되고, 0 또는 생략은 문서화된 기본값 `60000`을 적용한다. 음수 값과 safe duration bound를 초과하는 양수 값은 `NodeProviderConf.Validate()`에서 거부한다. effective 값은 `NodeProviderConf.EffectiveResponseStallTimeoutMS()`에서 계산한다. 이 필드는 config refresh에서 `restart_required`로 분류되며, effective-zero 등가성(생략 vs 명시적 0)은 변경으로 보고되지 않는다. request hard timeout, queue timeout, heartbeat/disconnect, CLI `response_idle_timeout_ms`는 기존 소유권을 유지한다. - `nodes[].providers[].id`는 전체 Edge config 안에서 중복되면 안 된다. - `nodes[].providers[].adapter`는 같은 Node 안의 enabled adapter instance key를 참조해야 한다. Exact instance key를 우선하고, legacy type-name route는 같은 type의 enabled instance가 정확히 하나일 때만 허용한다. - `nodes[].providers[].enabled`: 생략 또는 `true` → provider pool dispatch 후보에 포함. `false` → dispatch pool에서 제외. 비활성화된 provider는 status snapshot에 `status=disabled`, `health=disabled`, `capacity=0`으로 표시된다. adapter process lifecycle 변경 없음. config refresh 시 `enabled` 토글은 live-apply(restart 불필요)로 분류된다. disabled provider의 adapter reference check는 skip되지만 structural validation(type, category, models, numeric bounds)은 수행된다. diff --git a/agent-contract/inner/edge-node-runtime-wire.md b/agent-contract/inner/edge-node-runtime-wire.md index afec901e..a1a75e80 100644 --- a/agent-contract/inner/edge-node-runtime-wire.md +++ b/agent-contract/inner/edge-node-runtime-wire.md @@ -58,7 +58,7 @@ Edge는 Node 연결을 수락하고, Node는 연결 직후 등록 요청을 보 - disconnect/reconnect: current dispatch-ready owner의 close/heartbeat timeout만 해당 connection generation을 fence한다. Edge는 같은 authoritative lifecycle에서 provider lease를 정확히 한 번 반환하고 resource를 offline/excluded로 만든 뒤 queue를 live candidate 기준으로 재평가한다. accepted Node의 ready transition은 새 generation resource를 활성화하고 기존 waiter를 즉시 pump한다. stale/rejected connection callback은 live state나 lifecycle event를 바꾸지 않는다. - execution: Edge가 `RunRequest`를 보내고 Node가 `RunEvent` stream으로 실행 상태를 보낸다. - provider raw tunnel: Edge가 기존 Edge-Node socket으로 `ProviderTunnelRequest`를 보내고 Node가 provider HTTP/SSE 요청을 연 뒤 `ProviderTunnelFrame` stream으로 provider status/header/body/end/error/usage 후보를 sequence와 함께 돌려준다. 이 경로는 OpenAI-compatible provider passthrough용이며 `RunEvent` 실행 stream과 분리된다. -- response_stall_timeout_ms: `RunRequest.response_stall_timeout_ms`와 `ProviderTunnelRequest.response_stall_timeout_ms`는 int64 필드로, 선택된 provider의 response-stall timeout을 밀리초 단위로 운반한다. Zero는 Node가 문서화된 기본값(300000ms)을 적용함을 의미한다. Negative 또는 overflow 값은 Node 경계에서 router/provider 호출 전에 reject된다. Edge provider-pool dispatch는 winning candidate의 effective timeout을 각 요청에 복사한다. Direct/non-pool 호출은 wire에서 zero를 사용하고 Node 기본값을 적용한다. +- response_stall_timeout_ms: `RunRequest.response_stall_timeout_ms`와 `ProviderTunnelRequest.response_stall_timeout_ms`는 int64 필드로, 선택된 provider의 response-stall timeout을 밀리초 단위로 운반한다. Zero는 Node가 문서화된 기본값(60000ms)을 적용함을 의미한다. Negative 또는 overflow 값은 Node 경계에서 router/provider 호출 전에 reject된다. Edge provider-pool dispatch는 winning candidate의 effective timeout을 각 요청에 복사한다. Direct/non-pool 호출은 wire에서 zero를 사용하고 Node 기본값을 적용한다. - response stall terminal: Node observes only the execution activity contract. On expiry it cancels and fences the local provider attempt, joins the bounded close-grace fence and an independent exact-target health probe without extending either serially, then emits exactly one normalized `RunEvent{type=error}` or tunnel `ProviderTunnelFrame{kind=ERROR}` with `failure_code=response_stalled` and populates the optional wire `ExecutionFailure` field (field 13 on `RunEvent`, field 15 on `ProviderTunnelFrame`). Terminal metadata is allowlisted (three-way health evidence as the `provider_health` status paired with the `liveness_classification` normalization — `available`/`request_stalled`, `unavailable`/`provider_unhealthy`, or `unknown`/`health_unknown`; idle duration; Node-owned run/attempt identity; fence; adapter; target; and an optional connection-scoped `health_observation_seq`); it contains no caller-controlled identity, raw payload, credential, or `recovery_eligible`. Nil and non-stalled failures leave wire `ExecutionFailure` absent while preserving legacy error string fields (`RunEvent.Error` / `ProviderTunnelFrame.Error`). `health_observation_seq` starts at one per connection and increases uniquely across the connection's normalized and tunnel observations; an unbound session omits it. Probe availability is evidence only and never resets progress, changes the fence, or authorizes retry. A confirmed fence is a capability hint only, not Node retry authorization. - Edge terminal handoff: transport reception identity, not payload identity, supplies `(node_id, connection_generation)`. Before a normalized or tunnel terminal can affect provider health, Edge compares that identity and the typed adapter/target evidence with the tracked immutable provider lease. A current terminal releases that lease exactly once even when optional health evidence is rejected. Edge adds `provider_id`, validated `provider_health`, and `recovery_handoff=confirmed` to every validated current bound stall before downstream routing, including sequence-stale request-local handoff; only a fresh `unavailable` observation lowers the separate runtime overlay. The handoff token is not replay approval, and Edge never adds `recovery_eligible` here. - CAPABILITIES recovery probe: Node resolves the requested adapter instance, runs the bounded fail-closed exact-target `ProbeHealth`, and returns stable `adapter_key`, `target`, normalized `provider_status`, and the next Session-owned `health_observation_seq`. Edge retains the command's dispatch node/generation and may clear one unavailable overlay only when a higher-sequence `available` response identifies exactly one same-generation provider binding. Empty, malformed, ambiguous, mismatched, stale, `unknown`, and `unavailable` results do not change the overlay. diff --git a/agent-contract/inner/execution-runtime.md b/agent-contract/inner/execution-runtime.md index fb792442..b762d4a0 100644 --- a/agent-contract/inner/execution-runtime.md +++ b/agent-contract/inner/execution-runtime.md @@ -34,22 +34,22 @@ The execution package defines host-neutral provider primitives. It owns provider - Registry lookup uses provider identity and returns typed failures for missing or unavailable providers. - Callers must reject commands outside the closed provider-command allowlist before provider lookup. - Token usage remains observation data attached to execution or tunnel results. -- `DefaultResponseStallTimeoutMS = 300000` is the documented default. `ResolveStallTimeoutMS(ms)` validates then maps zero to the default; safe positive values pass through, while negative or overflow values return an error. +- `DefaultResponseStallTimeoutMS = 60000` is the documented default. `ResolveStallTimeoutMS(ms)` validates then maps zero to the default; safe positive values pass through, while negative or overflow values return an error. - `ClassifyRuntimeEvent` returns `start` for `EventTypeStart`, `progress` for non-empty `delta`/`message` or non-terminal usage, `terminal` for `complete`/`error`/`cancelled` (before usage check), and `none` for empty/unknown events. - `ClassifyProviderTunnelFrame` returns `progress` for `response_start` (with or without headers) and non-empty `body`, `terminal` for `end`/`error` (before payload check), `progress` for `usage`, and `none` for empty/unknown frames. - `ValidateStallTimeoutMS(ms)` rejects negative values and values exceeding `maxSafeStallTimeoutMS`; zero is allowed (use default). - `NodeProviderConf.EffectiveResponseStallTimeoutMS()` returns the effective timeout for a provider candidate. - `RunRequest.ResponseStallTimeoutMS` and `ProviderTunnelRequest.ResponseStallTimeoutMS` carry the selected provider's effective timeout; zero on the wire means the Node applies the documented default. -- The Node wire boundary normalizes zero to `300000` and rejects negative or overflow values before router/provider invocation. +- The Node wire boundary normalizes zero to `60000` and rejects negative or overflow values before router/provider invocation. - `response_stalled` is a stable typed failure. Node transport mappers (`runEventToProto` and `tunnelFrameToProto`) populate the optional wire `ExecutionFailure` message only for `FailureCodeResponseStalled`, attaching a defensive clone of allowlisted metadata keys (`failure_code`, `provider_health`, `liveness_classification`, `idle_duration_ms`, `run_id`, `attempt_id`, `attempt_fence`, `adapter`, `target`, and `health_observation_seq`); nil and non-stalled failures leave wire `ExecutionFailure` absent while preserving legacy error string fields (`RunEvent.Error` / `ProviderTunnelFrame.Error`). Caller metadata cannot override these values, and no raw payload, credential, or `recovery_eligible` signal is admitted. - The Node watchdog starts from attempt admission, resets only on the documented progress dispositions, stops on provider terminal, and emits one typed stall terminal. It does not retry providers or infer recovery eligibility. `Retryable=true` means only that the local provider ownership fence was confirmed within the bounded close grace. - After the watchdog claims a stall it joins two independent bounded outcomes without extending either serially — the fixed close-grace fence and the exact-target health probe — then assembles exactly one allowlisted terminal. The joined `liveness_classification`/`provider_health` pair is exactly `request_stalled`/`available`, `provider_unhealthy`/`unavailable`, or `health_unknown`/`unknown` (fail-closed default). Provider availability observed here is evidence only: it never resets progress, changes the fence, revives output, or authorizes retry, and late provider output stays fenced. - `health_observation_seq` is a connection-scoped monotonic sequence sourced from the transport Session. A new connection starts at zero, so the first finalized observation is one; normalized and tunnel observations on the same connection share the source and receive unique, increasing values under concurrency. Internal or unbound execution paths omit the key entirely and never encode a process-global generation. -- `ProviderPoolDispatchRequest` carries two request-local recovery-hint fields: `AvoidProviderID` (non-empty to prefer a runtime-eligible alternate over the avoided provider) and `AllowAvoidedProviderFallback` (explicit permission to retain the avoided provider when no alternate exists and it remains runtime eligible). The queue applies identical avoidance filtering to both initial and queued re-resolution. Zero values preserve current selection behavior. This is selection policy only: it does not create a retry loop, reserve a slot, change provider priority, persist the hints, or count retries. The fallback permission is always derived from exact probe-backed `available` evidence by the caller (never from current overlay state). +- `ProviderPoolDispatchRequest` carries two request-local recovery-hint fields: `AvoidProviderID` (non-empty to require a runtime-eligible alternate over the avoided provider) and `AllowAvoidedProviderFallback` (a compatibility field that liveness recovery leaves false). The queue applies identical avoidance filtering to both initial and queued re-resolution. Zero values preserve current selection behavior. This is selection policy only: it does not create a retry loop, reserve a slot, change provider priority, persist the hints, or count retries. A provider whose request produced `response_stalled` is never selected again for that request, even when its exact-target health probe reports `available`. - A Node `capabilities` command performs the same bounded exact-target `ProbeHealth` operation. Its stable result evidence is the requested adapter instance key (`adapter_key`), exact `target`, fail-closed normalized `provider_status`, and the next `health_observation_seq` from that same transport Session. Probe errors, unsupported probing, and adapter/instance/target mismatches report `unknown`; raw capability status is not recovery evidence. - Edge accepts a typed stall observation for provider-wide projection only after authoritative reception `(node_id, connection_generation)` matches the tracked immutable dispatch lease `(node_id, connection_generation, provider_id, adapter, target)`, the local attempt fence is confirmed, and the observation sequence is strictly newer. A current terminal still releases its lease exactly once when health evidence is absent, malformed, mismatched, or stale; a reception-owner mismatch changes neither overlay nor lease state. - Every validated current bound stall is annotated with Edge-owned `provider_id`, the validated `provider_health`, and `recovery_handoff=confirmed`, including an out-of-order terminal whose health projection is sequence-stale. Only a fresh `unavailable` observation lowers the generation-scoped runtime overlay. The token proves reception, lease binding, and local-fence handoff only; it is never `recovery_eligible` and never authorizes retry. -- Every supported OpenAI Chat/Responses normalized or tunnel request enters one request-local StreamGate runtime, which is the sole liveness owner even when configured semantic filtering is disabled. That runtime may consume the confirmed handoff as a raw-free `response_stalled` provider error while its endpoint adapters preserve the disabled-semantic native status, headers, JSON/SSE/tunnel order, validation, usage, cancellation, and terminal behavior. It retains only the stable failure code, confirmed-handoff token, and `available|unavailable|unknown` health classification; Node/provider messages and arbitrary metadata are not copied. Exact replay additionally requires the existing uncommitted, uncancelled, side-effect-safe, snapshot-backed, shared-budget gate. A confirmed old terminal closes its Edge transport without another `CancelRun`; pool re-admission consumes the provider once as `AvoidProviderID`, with same-provider fallback only for exact `available` evidence. +- Every supported OpenAI Chat/Responses normalized or tunnel request enters one request-local StreamGate runtime, which is the sole liveness owner even when configured semantic filtering is disabled. That runtime may consume the confirmed handoff as a raw-free `response_stalled` provider error while its endpoint adapters preserve the disabled-semantic native status, headers, JSON/SSE/tunnel order, validation, usage, cancellation, and terminal behavior. It retains only the stable failure code, confirmed-handoff token, and `available|unavailable|unknown` health classification; Node/provider messages and arbitrary metadata are not copied. Exact replay additionally requires the existing uncommitted, uncancelled, side-effect-safe, snapshot-backed, shared-budget gate and is bounded to one liveness replay per request. A confirmed old terminal closes its Edge transport without another `CancelRun`; pool re-admission consumes the provider once as `AvoidProviderID` and never falls back to that stalled provider. A stall on the replacement attempt terminates without another replay. - The runtime overlay is keyed by `(node_id, connection_generation, provider_id)` and remains separate from configuration health. It excludes the provider from effective admission and projects it unavailable in status snapshots. Recovery requires a later CAPABILITIES result for the same current adapter/target mapping with strictly higher sequence and exact normalized `available`; malformed, ambiguous, stale-generation, unknown, and unavailable results are no-ops. ## Health probe contract diff --git a/agent-spec/input/openai-compatible-surface.md b/agent-spec/input/openai-compatible-surface.md index 8a31da40..84d9c2de 100644 --- a/agent-spec/input/openai-compatible-surface.md +++ b/agent-spec/input/openai-compatible-surface.md @@ -252,7 +252,7 @@ sequenceDiagram - `credential_plane.enabled` is the startup-only managed/legacy switch. Managed mode requires TLS on OpenAI ingress, CP-Edge, and Edge-Node hops; config validation rejects legacy principal/provider-auth and static provider credential sources. - Managed authentication and model resolution use one immutable projection view per request. Trusted principal/route/slot/revision metadata overwrites caller spoofing and remains bound across recovery admission. - `openai.stream_evidence_gate.enabled` defaults to false and activates configured semantic policy only. Supported OpenAI response/liveness ownership remains in the request runtime in both states; the same config also supplies the 0..3 recovery cap and up-to-16-MiB ingress snapshot bound. Changes remain restart-required. -- A typed stall recovery re-enters provider-pool admission with the failed provider avoided. Exact `available` is the sole health classification that allows same-provider fallback when no alternate exists. +- A typed stall recovery re-enters provider-pool admission with the failed provider avoided. No health classification allows the stalled provider to be selected again for the same request; without an alternate, recovery terminates once. - `iop_edge_liveness_recovery_eligibility_total` labels are `execution_path`, `provider_health`, `commit_state`, and `eligibility`; `iop_edge_liveness_recovery_results_total` labels are `execution_path`, `provider_health`, and `recovery_result`. All are closed vocabularies and exclude request/attempt/provider/model identifiers and content. - When `repeat_guard` is configured, Chat accepts plain `content`, `reasoning_content`, `reasoning`, and `reasoning_text` provenance for fingerprinting; Responses accepts its own text/reasoning/function-call item provenance. Signed, encrypted, and unknown values are canonical-only and never sanitation or observation payloads. - Completed action/result fingerprints provide the only request-history progress boundary. An identical consecutive action/result is no-progress; a changed completed result is progress, while a different action alone is insufficient. No caller product, session metadata, inferred TTL, or cross-request cache participates. diff --git a/agent-spec/runtime/edge-node-execution.md b/agent-spec/runtime/edge-node-execution.md index e8fb0451..640bcdfe 100644 --- a/agent-spec/runtime/edge-node-execution.md +++ b/agent-spec/runtime/edge-node-execution.md @@ -233,13 +233,13 @@ The shared `packages/go/execution` package contains provider lifecycle, registry | Work stage | The `ornith-fast` Work runner reads the closed PLAN artifact, projects only the admitted workspace tools, and resumes the same frozen provider route after exactly correlated Node results. It rejects any Work `reasoning_effort`, malformed or multiple tool calls, and empty completion or verification evidence. | | request-owned cleanup | Node creates and inventories only `.iop/job/` internal state, cancels and waits for all active command groups, validates the exact tree without following entries, and removes matching artifacts deepest-first with non-recursive descriptor operations. Symlinks, special files, foreign devices, identity replacements, and unowned entries fail closed. User results and sibling request state are preserved. Concurrent cleanup callers receive one bounded cached typed result. | | provider raw tunnel | 선택된 provider의 HTTP/SSE를 `ProviderTunnelRequest`/`ProviderTunnelFrame`으로 relay하며 순서와 단일 terminal outcome을 보장한다. | -| response-stall activity contract | 선택된 provider의 response-stall timeout을 normalized/tunnel request에 보존한다. Node는 wire zero를 `300000ms`로 해석하고 invalid raw value를 adapter 호출 전에 거부한다. Runtime event의 terminal type은 payload/usage보다 우선하며 non-terminal usage는 progress다. | +| response-stall activity contract | 선택된 provider의 response-stall timeout을 normalized/tunnel request에 보존한다. Node는 wire zero를 `60000ms`로 해석하고 invalid raw value를 adapter 호출 전에 거부한다. Runtime event의 terminal type은 payload/usage보다 우선하며 non-terminal usage는 progress다. | | Node stall watchdog | Node가 normalized run과 raw tunnel에 하나의 activity watchdog을 적용한다. progress만 timer를 reset하며, stall은 `response_stalled` terminal 하나와 Node-owned safe metadata를 만들어 normalized `RunEvent`와 raw `ProviderTunnelFrame` wire의 optional typed `ExecutionFailure` 필드에 싣는다. stall claim 뒤에는 bounded close grace fence와 독립 exact-target health probe를 직렬 확장 없이 join한다. close grace 안에 provider return이 확인된 경우만 `Retryable` capability hint를 준다. | | Node health evidence join | stall terminal에 three-way health evidence를 싣는다: `provider_health` status와 `liveness_classification` normalization이 `available`/`request_stalled`, `unavailable`/`provider_unhealthy`, `unknown`/`health_unknown` 쌍으로 fail-closed된다. probe 성공은 progress reset·fence 변경·retry authority가 아니며 late output은 fenced 상태를 유지한다. | | health observation sequence | transport Session이 connection-scoped monotonic `health_observation_seq`를 소유한다. 새 connection은 0에서 시작해 첫 finalized observation이 1이며, 같은 connection의 normalized/tunnel observation이 source를 공유해 동시에도 유일 증가값을 받는다. internal/unbound 경로는 key를 생략한다. | | Edge terminal health handoff | Edge validates authoritative reception node/generation plus the immutable provider/adapter/target lease before applying typed stall evidence. Every validated current bound stall receives `provider_id`, validated health, and `recovery_handoff=confirmed`, while only fresh unavailable evidence lowers a separate runtime overlay; the token never grants replay eligibility. Every valid current terminal still releases its lease exactly once. | | CAPABILITIES recovery | Node runs the same bounded exact-target `ProbeHealth` and returns stable adapter/target/status plus the next Session sequence. Edge recovers exactly one matching current-generation unavailable provider only from a strictly newer `available` result; malformed, ambiguous, stale, unknown, and unavailable responses are no-ops. | -| recovery candidate preference | `ProviderPoolDispatchRequest` carries `AvoidProviderID` and `AllowAvoidedProviderFallback`. Every admission (initial and queued re-resolution) prefers a runtime-eligible alternate over the avoided provider; only the explicit fallback flag (derived from exact probe-backed `available` evidence) permits re-selecting the avoided provider when no alternate exists. Zero values preserve current selection. This is selection policy only: no retry loop, slot reservation, priority change, persistence, or retry counter. | +| recovery candidate preference | `ProviderPoolDispatchRequest` carries `AvoidProviderID` and `AllowAvoidedProviderFallback`. Every admission (initial and queued re-resolution) requires a runtime-eligible alternate over the avoided provider for liveness recovery. `available` probe evidence describes endpoint health but never permits re-selecting the stalled provider for the same request. Liveness replay is request-locally capped at one, so a replacement-attempt stall terminates. Zero values preserve current selection. This is selection policy only: no retry loop, slot reservation, priority change, persistence, or retry counter. | | OpenAI typed-stall consumption | Every supported Chat/Responses normalized or tunnel request has one unconditional runtime liveness owner, independent of configured semantic activation. It converts only the Edge-confirmed typed stall handoff into a raw-free StreamGate event, owns pre-commit eligibility, and closes the already fenced old transport before re-admission; Node does not grant replay authority. | | tunnel-tolerant liveness | Edge와 Node는 30초 heartbeat interval과 45초 response wait를 공통으로 사용해 긴 prompt prefill이나 streaming backpressure 중의 정상 connection을 조기에 끊지 않는다. | | reconnect/generation fencing | 현재 connection이 종료되면 해당 generation만 fence하고 Node supervisor가 reconnect한다. Heartbeat wait를 넘긴 경우의 close reason은 `heartbeat_timeout`이다. | diff --git a/agent-spec/runtime/provider-pool-config-refresh.md b/agent-spec/runtime/provider-pool-config-refresh.md index cd43bae9..10848cb8 100644 --- a/agent-spec/runtime/provider-pool-config-refresh.md +++ b/agent-spec/runtime/provider-pool-config-refresh.md @@ -115,7 +115,7 @@ Edge 설정에서 provider-pool이 어떻게 모델 실행 후보를 고르고, | usage attribution policy | `models[].usage_attribution`은 `provider|model_group`만 허용하고 생략 시 provider 귀속으로 해석한다. model-group 귀속은 운영자의 명시적 opt-in이다. | | provider mapping | `models[].providers`는 provider id를 실제 served model name으로 매핑한다. | | node provider catalog | `nodes[].providers[]`는 Node 아래 resource/provider catalog이며 provider id는 Edge config에서 전역 유일해야 한다. | -| response-stall timeout | `response_stall_timeout_ms`는 provider별 response-stall timeout이다. zero/omitted는 `300000ms`, invalid negative/overflow 값은 validation error이며 selected candidate의 effective 값은 normalized/tunnel request에 보존된다. | +| response-stall timeout | `response_stall_timeout_ms`는 provider별 response-stall timeout이다. zero/omitted는 `60000ms`, invalid negative/overflow 값은 validation error이며 selected candidate의 effective 값은 normalized/tunnel request에 보존된다. | | config validation | config load가 provider id 참조, served model membership, numeric bounds, long-context budget을 검증한다. | | provider 후보 필터링 | dispatch는 dispatch-ready connection을 가진 Node의 provider 후보 중 catalog match, enabled, healthy/available, capacity 조건을 만족하는 후보만 사용한다. protocol profile capability(`messages`, `chat`, `responses`, `streaming`, `tool_calling`, `count_tokens`, `models`)는 operation별 admission에 사용된다. | | provider 전역 capacity/priority dispatch | `node_id + provider_id` lease가 여러 model group의 일반·long in-flight를 합산한다. available 후보 중 낮은 in-flight를 고르고 동률이면 낮은 `priority`와 round-robin을 적용한다. | @@ -188,7 +188,7 @@ sequenceDiagram - Node managed mode requires Edge transport TLS, `recipient_key_id`/recipient private-key path, issuer key id/public-key path, and a bounded replay cache. All cert/key/keyring values are external file references and credential-plane changes are restart-required. - `protocol_profiles` is the top-level catalog of custom overlays. A `ProtocolProfileConf` supplies `base`, `driver`, `base_url`, operation paths, `auth`, `capabilities`, `model_mapping`, and `extensions`; `base` inheritance is separate from legacy provider-type normalization. - `nodes[].providers[].profile` selects a catalog entry. Config normalization resolves that selection (or a legacy type alias) into the runtime-only `RuntimeProfile` snapshot; the source YAML remains a selector plus catalog, not a per-model overlay. -- `nodes[].providers[].response_stall_timeout_ms` is validated at config load: zero/omitted resolves to `300000ms`; safe positive values are retained; negative and duration-overflow values are rejected. Its effective value is immutable for the selected provider attempt and survives queue re-resolution for both execution paths. +- `nodes[].providers[].response_stall_timeout_ms` is validated at config load: zero/omitted resolves to `60000ms`; safe positive values are retained; negative and duration-overflow values are rejected. Its effective value is immutable for the selected provider attempt and survives queue re-resolution for both execution paths. - Profile catalog and provider-selector changes are restart-required. Snapshot immutability describes loaded runtime state and does not make those changes live-applicable. - `ConcreteProtocolProfile.MapModel(model)`은 provider의 model alias 정규화를 수행한다. provider가 model mapping을 정의하면 IOP external `model` key를 provider served target으로 변환한다. - `ConcreteProtocolProfile.ResolveOperationURL(op)` returns the complete resolved upstream URL. Absolute operation URLs are returned unchanged, while relative operation paths are joined once to the normalized base URL; the listed `/v1/...` values are operation-path inputs, not return values. diff --git a/apps/edge/internal/openai/stream_gate_dispatcher_test.go b/apps/edge/internal/openai/stream_gate_dispatcher_test.go index ff3dba3c..bef8950f 100644 --- a/apps/edge/internal/openai/stream_gate_dispatcher_test.go +++ b/apps/edge/internal/openai/stream_gate_dispatcher_test.go @@ -235,7 +235,7 @@ func TestOpenAIAttemptDispatcherStalledProvider(t *testing.T) { t.Fatalf("dispatch recovery: %v", err) } defer binding.Controller().AbortAttempt(context.Background()) - if service.lastPool.AvoidProviderID != "provider.stalled" || !service.lastPool.AllowAvoidedProviderFallback { + if service.lastPool.AvoidProviderID != "provider.stalled" || service.lastPool.AllowAvoidedProviderFallback { t.Fatalf("recovery pool hints = %#v", service.lastPool) } } diff --git a/apps/edge/internal/openai/stream_gate_filters.go b/apps/edge/internal/openai/stream_gate_filters.go index c7fcabea..f249af9d 100644 --- a/apps/edge/internal/openai/stream_gate_filters.go +++ b/apps/edge/internal/openai/stream_gate_filters.go @@ -759,7 +759,7 @@ func (f *openAIStallRecoveryFilter) Evaluate(_ context.Context, fctx streamgate. var intent *streamgate.RecoveryIntent if descriptor == "response_stalled_confirmed" { unsafe := fctx.CommitState() != streamgate.CommitStateTransportUncommitted || fctx.HasToolSideEffect() || f.requestRef == "" || batchHasToolEvidence(batch) - if unsafe { + if unsafe || !f.state.claimRecovery() { descriptor = "response_stalled_ineligible" } else { directive, err := streamgate.NewRecoveryDirectiveExact(f.requestRef) diff --git a/apps/edge/internal/openai/stream_gate_ingress.go b/apps/edge/internal/openai/stream_gate_ingress.go index 83cbdc60..9694b478 100644 --- a/apps/edge/internal/openai/stream_gate_ingress.go +++ b/apps/edge/internal/openai/stream_gate_ingress.go @@ -45,6 +45,20 @@ type openAIStallRecoveryState struct { providerID string health string confirmedForClose bool + recoveryClaimed bool +} + +func (s *openAIStallRecoveryState) claimRecovery() bool { + if s == nil { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + if s.recoveryClaimed { + return false + } + s.recoveryClaimed = true + return true } func (s *openAIStallRecoveryState) arm(attemptID, providerID, health string) { @@ -81,7 +95,10 @@ func (s *openAIStallRecoveryState) consumeAdmission() (providerID string, allowF if !s.confirmedForClose || s.providerID == "" { return "", false, false } - providerID, allowFallback = s.providerID, s.health == "available" + // Health-probe availability proves only that the endpoint is reachable. It + // never makes the exact request that just stalled safe to replay on the same + // provider. + providerID, allowFallback = s.providerID, false s.attemptID, s.providerID, s.health = "", "", "" s.confirmedForClose = false return providerID, allowFallback, true diff --git a/apps/edge/internal/openai/stream_gate_stall_recovery_test.go b/apps/edge/internal/openai/stream_gate_stall_recovery_test.go index 14b8d04a..1aa725c8 100644 --- a/apps/edge/internal/openai/stream_gate_stall_recovery_test.go +++ b/apps/edge/internal/openai/stream_gate_stall_recovery_test.go @@ -106,7 +106,7 @@ func TestOpenAIStallRecoveryFilter(t *testing.T) { t.Fatal("confirmed state was not armed") } provider, fallback, ok := state.consumeAdmission() - if !ok || provider != "provider-a" || fallback != (health == "available") { + if !ok || provider != "provider-a" || fallback { t.Fatalf("admission hint = %q/%t/%t", provider, fallback, ok) } }) @@ -140,6 +140,34 @@ func TestOpenAIStallRecoveryIneligibleAfterCommitOrTool(t *testing.T) { } } +func TestOpenAIStallRecoveryIsBoundedToOneReplay(t *testing.T) { + state := &openAIStallRecoveryState{} + filter, err := newOpenAIStallRecoveryFilter("openai.ingress.1", state) + if err != nil { + t.Fatal(err) + } + event, err := newOpenAIProviderErrorEventFromFailure(confirmedStallFailure("available"), streamGateErrorRunFailed) + if err != nil { + t.Fatal(err) + } + ctx := stallFilterContext(t, streamgate.CommitStateTransportUncommitted, false) + batch := stallBatch(t, event, streamgate.CommitStateTransportUncommitted) + first, err := filter.Evaluate(context.Background(), ctx, batch) + if err != nil { + t.Fatal(err) + } + second, err := filter.Evaluate(context.Background(), ctx, batch) + if err != nil { + t.Fatal(err) + } + if first.Kind() != streamgate.FilterDecisionKindViolation || first.RecoveryIntent() == nil { + t.Fatalf("first stall decision = %#v", first) + } + if second.Kind() != streamgate.FilterDecisionKindPass || second.RecoveryIntent() != nil { + t.Fatalf("second stall decision = %#v", second) + } +} + func stallMatrixSuccessAttempt(endpoint, path string, stream bool, runID, provider, marker string) scriptedPoolAttempt { attempt := scriptedPoolAttempt{path: path, runID: runID, provider: provider, target: "served-" + provider} if path == string(edgeservice.ProviderPoolPathNormalized) { @@ -304,7 +332,7 @@ func TestOpenAIStallAfterLogicalFinishMatrix(t *testing.T) { t.Fatalf("raw failure data leaked: %q", body) } requests := stallPoolRequests(service) - if len(requests) != 2 || requests[1].AvoidProviderID != "provider-a" || !requests[1].AllowAvoidedProviderFallback { + if len(requests) != 2 || requests[1].AvoidProviderID != "provider-a" || requests[1].AllowAvoidedProviderFallback { t.Fatalf("pre-commit replay requests=%+v", requests) } if strings.Count(body, recovered) != 1 { @@ -557,7 +585,7 @@ func TestOpenAIStallRecoveryMatrix(t *testing.T) { }) } - t.Run("same-provider fallback requires available evidence", func(t *testing.T) { + t.Run("stall replay never grants same-provider fallback", func(t *testing.T) { path := string(edgeservice.ProviderPoolPathNormalized) service := newScriptedPoolRunService( stallMatrixFailureAttempt(path, "available-a", "provider-a", "available"), @@ -565,7 +593,7 @@ func TestOpenAIStallRecoveryMatrix(t *testing.T) { ) w := runStallMatrixHandler(t, stallMatrixServer(service, false, 1), openAIRebuildEndpointChat, false, nil) requests := stallPoolRequests(service) - if w.Code != http.StatusOK || len(requests) != 2 || requests[1].AvoidProviderID != "provider-a" || !requests[1].AllowAvoidedProviderFallback { + if w.Code != http.StatusOK || len(requests) != 2 || requests[1].AvoidProviderID != "provider-a" || requests[1].AllowAvoidedProviderFallback { t.Fatalf("available fallback response=%d/%q requests=%+v", w.Code, w.Body.String(), requests) } }) diff --git a/apps/edge/internal/service/provider_pool.go b/apps/edge/internal/service/provider_pool.go index 98d3185a..075d6b0a 100644 --- a/apps/edge/internal/service/provider_pool.go +++ b/apps/edge/internal/service/provider_pool.go @@ -95,9 +95,9 @@ func (e *ProviderPoolOperationUnsupportedError) Unwrap() error { // alternate provider over the avoided one. The avoided provider is only // retained when no alternate exists AND AllowAvoidedProviderFallback is // true AND the provider is still runtime eligible — the explicit fallback -// permission is the only way to re-select the avoided provider, and it is -// always derived from exact probe-backed available evidence by the caller -// (never from current overlay state). +// permission is the only way to re-select the avoided provider. Liveness +// recovery never grants that permission: a health probe proves endpoint +// availability, not safety of replaying the request that just stalled. // // Zero values (empty AvoidProviderID, false AllowAvoidedProviderFallback) // preserve the current candidate selection behavior. diff --git a/configs/edge.yaml b/configs/edge.yaml index ab6630e1..0e3a73f2 100644 --- a/configs/edge.yaml +++ b/configs/edge.yaml @@ -166,8 +166,8 @@ openai: stream_evidence_gate: enabled: false environment: dev # dev | dev-corp; request-start selector snapshot - max_request_fault_recovery: 3 - max_strategy_fault_recovery: 3 + max_request_fault_recovery: 0 + max_strategy_fault_recovery: 0 max_ingress_snapshot_bytes: 16777216 # filters are disabled by omission. Each policy has one unique filter kind: # repeat_guard (request-local history plus Unicode rolling/current-stream @@ -417,7 +417,7 @@ nodes: health: "healthy" capacity: 1 priority: 50 - # response_stall_timeout_ms: 300000 # omitted → uses documented default + # response_stall_timeout_ms: 60000 # omitted → uses documented default # Seulgivibe OpenAI-compatible provider examples. Keep endpoint values # illustrative and provide user tokens per request via openai.provider_auth. # - id: "seulgivibe-claude" diff --git a/docs/edge-local-dev-guide.md b/docs/edge-local-dev-guide.md index d2ead4cd..9a569923 100644 --- a/docs/edge-local-dev-guide.md +++ b/docs/edge-local-dev-guide.md @@ -77,11 +77,14 @@ nodes: dev-runtime에서 긴 응답의 정지 판정은 아래 순서를 유지한다. ```text -Node provider response_stall_timeout_ms = 120000 -external caller boundary ≈ 180000 -Edge request hard timeout > 180000 +Node provider response_stall_timeout_ms = 60000 +Node close/probe join ceiling = 5000 +Agent dispatcher silence safety = 70000 +Edge request hard timeout > 70000 ``` +dev-runtime은 `max_request_fault_recovery: 0`과 `max_strategy_fault_recovery: 0`을 명시하고 Pi agent retry도 비활성화한다. 따라서 provider 무진행은 Node가 60초에 한 번만 종료하며 Pi·Edge·Dispatcher가 같은 요청을 다시 중첩 실행하지 않는다. + `response_stall_timeout_ms` 변경은 restart-required다. `config check`와 `config refresh --mode dry-run`에서 이를 확인한 뒤 Edge와 Node를 같은 source ref로 rebuild/restart하고, 각 binary의 build identity와 실행 중인 process identity를 다시 대조한다. tracked 검증 근거에는 source/build/config 식별자, 단조 시간, terminal 개수와 결과 분류만 남기며 prompt, output, token, credential 원문은 기록하지 않는다. 확인: diff --git a/packages/go/config/provider_types.go b/packages/go/config/provider_types.go index 2d8a9645..8e27ce9f 100644 --- a/packages/go/config/provider_types.go +++ b/packages/go/config/provider_types.go @@ -100,7 +100,7 @@ type NodeProviderConf struct { // ResponseStallTimeoutMS is the provider-originated response-stall timeout // in milliseconds. It is carried on every provider-first and legacy route // request so the downstream watchdog has one effective value per dispatched - // attempt. Zero is treated as the documented default (300000 ms). Negative + // attempt. Zero is treated as the documented default (60000 ms). Negative // values and positive values that cannot safely become a time.Duration are // rejected by Validate. The effective value is returned by // EffectiveResponseStallTimeoutMS. diff --git a/packages/go/execution/liveness.go b/packages/go/execution/liveness.go index 07d85a11..91f6ea48 100644 --- a/packages/go/execution/liveness.go +++ b/packages/go/execution/liveness.go @@ -10,7 +10,7 @@ import ( // DefaultResponseStallTimeoutMS is the default response-stall timeout in // milliseconds. It is used when no provider-configured value is available // (zero wire value, omitted config, direct/legacy dispatch). -const DefaultResponseStallTimeoutMS = 300000 +const DefaultResponseStallTimeoutMS = 60000 // maxSafeStallTimeoutMS is the largest millisecond value that can safely // become a time.Duration without overflow. Values above this bound are diff --git a/packages/go/execution/liveness_test.go b/packages/go/execution/liveness_test.go index 8534aa7e..cf15b1dc 100644 --- a/packages/go/execution/liveness_test.go +++ b/packages/go/execution/liveness_test.go @@ -25,7 +25,6 @@ func TestResolveStallTimeoutMS(t *testing.T) { {"exact safe boundary passes through", math.MaxInt64 / int64(time.Millisecond), math.MaxInt64 / int64(time.Millisecond), false}, {"first overflowing millisecond rejected", math.MaxInt64/int64(time.Millisecond) + 1, 0, true}, {"negative rejected", -1, 0, true}, - {"overflow rejected", execution.DefaultResponseStallTimeoutMS * 100000000, 0, true}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -53,7 +52,7 @@ func TestValidateStallTimeoutMS(t *testing.T) { if err := execution.ValidateStallTimeoutMS(-1); err == nil { t.Error("expected error for -1") } - if err := execution.ValidateStallTimeoutMS(execution.DefaultResponseStallTimeoutMS * 100000000); err == nil { + if err := execution.ValidateStallTimeoutMS(math.MaxInt64/int64(time.Millisecond) + 1); err == nil { t.Error("expected error for overflow value") } }