diff --git a/agent-ops/.version b/agent-ops/.version index c1dddabb..0799b6de 100644 --- a/agent-ops/.version +++ b/agent-ops/.version @@ -1 +1 @@ -1.1.197 +1.1.198 diff --git a/agent-ops/skills/common/orchestrate-agent-task-loop/SKILL.md b/agent-ops/skills/common/orchestrate-agent-task-loop/SKILL.md index b3c97414..e6978c6f 100644 --- a/agent-ops/skills/common/orchestrate-agent-task-loop/SKILL.md +++ b/agent-ops/skills/common/orchestrate-agent-task-loop/SKILL.md @@ -53,7 +53,7 @@ Each target has: - optional `runtime.resume_command`, `preflight_command`, `environment`, `session_path`, `native_session_monitor`, `terminal_success`, and `auxiliary_logs`; - optional `runtime.output_format`: `text` or `jsonl`. -Command templates may use only `{agent}`, `{model}`, `{reasoning_effort}`, `{target_id}`, `{workspace}`, `{attempt_dir}`, `{session_id}`, `{resume_session}`, `{resume_session_dir}`, and `{prompt}`. A target with `reasoning_effort` must use `{reasoning_effort}` in its command and resume command when present; a target without the field cannot use that placeholder. `native_session_monitor=true` requires both `resume_command` and `session_path`. `terminal_success=agent_end` requires JSONL output and accepts only a non-retrying final `agent_end` whose last assistant message has `stopReason=stop`; `error`, `aborted`, a missing event, or another stop reason fails closed. The catalog must not embed repository secrets; environment values should refer only to runtime-provided non-secret configuration. +Command templates may use only `{agent}`, `{model}`, `{reasoning_effort}`, `{target_id}`, `{workspace}`, `{attempt_dir}`, `{session_id}`, `{resume_session}`, `{resume_session_dir}`, and `{prompt}`. A target with `reasoning_effort` must use `{reasoning_effort}` in its command and resume command when present; a target without the field cannot use that placeholder. `native_session_monitor=true` requires both `resume_command` and `session_path`. `terminal_success=agent_end` requires JSONL output and accepts only a non-retrying final `agent_end` whose last assistant message has `stopReason=stop`. `terminal_success=turn_completed` requires JSONL output and accepts only final `turn.completed`; `turn.failed` or a missing terminal event fails closed. When either declared success event is observed with exit 0, earlier recovered transport diagnostics do not turn the attempt into a failure. The catalog must not embed repository secrets; environment values should refer only to runtime-provided non-secret configuration. Each route owns its ordered `candidates` plus optional `rule_id`, `policy_priority`, and `reason_codes`. A route may use catalog-owned `windows` instead of a fixed candidate list; every window supplies an IANA timezone, start/end time, and candidates. Exactly one window must match. diff --git a/agent-ops/skills/common/orchestrate-agent-task-loop/assets/default-execution-catalog.json b/agent-ops/skills/common/orchestrate-agent-task-loop/assets/default-execution-catalog.json index 2b8eae35..448f4980 100644 --- a/agent-ops/skills/common/orchestrate-agent-task-loop/assets/default-execution-catalog.json +++ b/agent-ops/skills/common/orchestrate-agent-task-loop/assets/default-execution-catalog.json @@ -148,7 +148,8 @@ "--dangerously-bypass-approvals-and-sandbox", "{prompt}" ], - "output_format": "jsonl" + "output_format": "jsonl", + "terminal_success": "turn_completed" } }, "codex-sol-high": { @@ -171,7 +172,8 @@ "--dangerously-bypass-approvals-and-sandbox", "{prompt}" ], - "output_format": "jsonl" + "output_format": "jsonl", + "terminal_success": "turn_completed" } }, "codex-sol-xhigh": { @@ -194,7 +196,8 @@ "--dangerously-bypass-approvals-and-sandbox", "{prompt}" ], - "output_format": "jsonl" + "output_format": "jsonl", + "terminal_success": "turn_completed" } }, "codex-terra-high": { @@ -217,7 +220,8 @@ "--dangerously-bypass-approvals-and-sandbox", "{prompt}" ], - "output_format": "jsonl" + "output_format": "jsonl", + "terminal_success": "turn_completed" } } }, diff --git a/agent-ops/skills/common/orchestrate-agent-task-loop/scripts/dispatch.py b/agent-ops/skills/common/orchestrate-agent-task-loop/scripts/dispatch.py index 58cba403..46cb33e4 100644 --- a/agent-ops/skills/common/orchestrate-agent-task-loop/scripts/dispatch.py +++ b/agent-ops/skills/common/orchestrate-agent-task-loop/scripts/dispatch.py @@ -2662,11 +2662,22 @@ def failure_report_lines(failure: str, locator: Path) -> list[str]: return lines -def json_agent_terminal_outcome(value: object) -> str | None: - """Return the final agent outcome encoded by a generic JSONL event.""" +def json_agent_terminal_outcome( + value: object, + terminal_success_contract: str | None = None, +) -> str | None: + """Return the authoritative final outcome encoded by a JSONL event.""" if not isinstance(value, dict): return None - if str(value.get("type", "")).lower() != "agent_end": + event_type = str(value.get("type", "")).lower() + if terminal_success_contract in {None, "turn_completed"}: + if event_type == "turn.completed": + return "succeeded" + if event_type == "turn.failed": + return "failed" + if terminal_success_contract not in {None, "agent_end"}: + return None + if event_type != "agent_end": return None if value.get("willRetry") is True: return None @@ -2689,9 +2700,14 @@ def json_agent_terminal_outcome(value: object) -> str | None: return None -def json_agent_terminal_outcome_from_line(line: str) -> str | None: +def json_agent_terminal_outcome_from_line( + line: str, + terminal_success_contract: str | None = None, +) -> str | None: try: - return json_agent_terminal_outcome(json.loads(line)) + return json_agent_terminal_outcome( + json.loads(line), terminal_success_contract + ) except json.JSONDecodeError: return None @@ -3629,7 +3645,7 @@ async def invoke( control_violation: str | None = None session_stall_seconds: float | None = None terminal_success_contract = spec.runtime.get("terminal_success") - terminal_success_seen = False + terminal_outcome_seen: str | None = None try: runtime_values = { "agent": spec.cli, @@ -3898,11 +3914,12 @@ async def invoke( record["active_command_execution_ids"] = sorted( active_command_execution_ids ) - if ( - channel == "stdout" - and json_agent_terminal_outcome_from_line(line) == "succeeded" - ): - terminal_success_seen = True + if channel == "stdout" and terminal_success_contract: + terminal_outcome = json_agent_terminal_outcome_from_line( + line, str(terminal_success_contract) + ) + if terminal_outcome is not None: + terminal_outcome_seen = terminal_outcome diagnostic = terminal_diagnostic(spec.cli, channel, line) if diagnostic: diagnostics.append(diagnostic) @@ -4016,6 +4033,16 @@ async def invoke( failure_class = "process-terminated" failure_source = "process-termination" record["termination_initiator"] = "unknown" + elif ( + return_code == 0 + and terminal_success_contract is not None + and terminal_outcome_seen == "succeeded" + ): + # A declared terminal-success event is authoritative. Earlier JSONL + # error events and stderr diagnostics may describe a recovered + # transport (for example Codex WebSocket -> HTTPS fallback), not the + # final attempt outcome. + failure_class = None else: classified_failure, classified_evidence = classify_failure_with_evidence( "\n".join(diagnostics[-50:]) @@ -4034,11 +4061,12 @@ async def invoke( elif stdout_diagnostic_index is not None: failure_class = "generic-error" failure_evidence = diagnostics[stdout_diagnostic_index] - elif terminal_success_contract == "agent_end" and not terminal_success_seen: + elif terminal_success_contract is not None and terminal_outcome_seen != "succeeded": failure_class = "generic-error" failure_source = "dispatcher-terminal-contract" failure_evidence = ( - "required agent_end terminal success event was not observed" + f"required {terminal_success_contract} terminal success event " + "was not observed" ) failure_evidence_source = "dispatcher:terminal-contract" else: diff --git a/agent-ops/skills/common/orchestrate-agent-task-loop/scripts/execution_target_policy.py b/agent-ops/skills/common/orchestrate-agent-task-loop/scripts/execution_target_policy.py index 2b9d5f9c..3cb8158b 100644 --- a/agent-ops/skills/common/orchestrate-agent-task-loop/scripts/execution_target_policy.py +++ b/agent-ops/skills/common/orchestrate-agent-task-loop/scripts/execution_target_policy.py @@ -127,9 +127,10 @@ def _validate_runtime(value: object, label: str) -> dict[str, Any]: ) terminal_success = value.get("terminal_success") if terminal_success is not None: - if terminal_success != "agent_end": + if terminal_success not in {"agent_end", "turn_completed"}: raise CatalogError( - f"{label}.terminal_success must be 'agent_end'" + f"{label}.terminal_success must be 'agent_end' or " + "'turn_completed'" ) if runtime["output_format"] != "jsonl": raise CatalogError( diff --git a/agent-ops/skills/common/orchestrate-agent-task-loop/tests/test_dispatch.py b/agent-ops/skills/common/orchestrate-agent-task-loop/tests/test_dispatch.py index 4960d0d0..559870ea 100644 --- a/agent-ops/skills/common/orchestrate-agent-task-loop/tests/test_dispatch.py +++ b/agent-ops/skills/common/orchestrate-agent-task-loop/tests/test_dispatch.py @@ -451,26 +451,30 @@ class RuntimeCatalogDispatcherTests(unittest.TestCase): def _invoke_fake_json_event( self, - event: dict, + event: dict | list[dict], *, stderr_line: str | None = None, + terminal_success: str = "agent_end", ) -> tuple[int, str | None, dict, str]: with TemporaryDirectory() as tmp: root = Path(tmp) plan = write_plan(root) task = task_from_plan(root, plan) runner = root / "fake_json_runner.py" + events = event if isinstance(event, list) else [event] runner.write_text( "import json\n" "import sys\n" - f"print(json.dumps({event!r}))\n" + f"events = {events!r}\n" + "for event in events:\n" + " print(json.dumps(event))\n" f"print({stderr_line!r}, file=sys.stderr) if {stderr_line!r} else None\n", encoding="utf-8", ) runtime = { "command": [sys.executable, str(runner)], "output_format": "jsonl", - "terminal_success": "agent_end", + "terminal_success": terminal_success, } agent = dispatch.AgentSpec( "fake-json-runner", @@ -571,6 +575,58 @@ class RuntimeCatalogDispatcherTests(unittest.TestCase): self.assertEqual(record["status"], "succeeded") self.assertIn("succeeded:0", work_log) + def test_turn_completed_overrides_recovered_transport_diagnostics(self): + events = [ + { + "type": "error", + "message": "Reconnecting after websocket certificate failure", + }, + { + "type": "item.completed", + "item": { + "type": "error", + "message": "Falling back from WebSockets to HTTPS transport", + }, + }, + {"type": "turn.completed", "usage": {"input_tokens": 1}}, + ] + + return_code, failure, record, work_log = self._invoke_fake_json_event( + events, + stderr_line="ERROR websocket UnknownIssuer before HTTPS fallback", + terminal_success="turn_completed", + ) + + self.assertEqual(return_code, 0) + self.assertIsNone(failure) + self.assertEqual(record["status"], "succeeded") + self.assertIn("succeeded:0", work_log) + + def test_turn_failed_remains_terminal_failure(self): + return_code, failure, record, work_log = self._invoke_fake_json_event( + [ + {"type": "error", "message": "request failed"}, + {"type": "turn.failed", "error": {"message": "terminal"}}, + ], + terminal_success="turn_completed", + ) + + self.assertEqual(return_code, 0) + self.assertEqual(failure, "generic-error") + self.assertEqual(record["status"], "failed") + self.assertNotIn("succeeded:0", work_log) + + def test_required_turn_completed_event_fails_closed_when_missing(self): + return_code, failure, record, work_log = self._invoke_fake_json_event( + {"type": "message", "text": "partial output only"}, + terminal_success="turn_completed", + ) + + self.assertEqual(return_code, 0) + self.assertEqual(failure, "generic-error") + self.assertEqual(record["failure_source"], "dispatcher-terminal-contract") + self.assertNotIn("succeeded:0", work_log) + def test_zero_exit_worker_without_implementation_progress_is_generic_error(self): with TemporaryDirectory() as tmp: root = Path(tmp) diff --git a/agent-ops/skills/common/orchestrate-agent-task-loop/tests/test_execution_target_policy.py b/agent-ops/skills/common/orchestrate-agent-task-loop/tests/test_execution_target_policy.py index 9043a805..28b92e3a 100644 --- a/agent-ops/skills/common/orchestrate-agent-task-loop/tests/test_execution_target_policy.py +++ b/agent-ops/skills/common/orchestrate-agent-task-loop/tests/test_execution_target_policy.py @@ -210,6 +210,8 @@ class ExecutionTargetPolicyTests(unittest.TestCase): def test_terminal_success_contract_requires_jsonl_agent_end(self): valid = catalog_value() valid["targets"]["target-a"]["runtime"]["terminal_success"] = "agent_end" + valid_turn = catalog_value() + valid_turn["targets"]["target-a"]["runtime"]["terminal_success"] = "turn_completed" invalid_name = catalog_value() invalid_name["targets"]["target-a"]["runtime"]["terminal_success"] = "message_end" invalid_format = catalog_value() @@ -222,7 +224,12 @@ class ExecutionTargetPolicyTests(unittest.TestCase): loaded.targets["target-a"].runtime["terminal_success"], "agent_end", ) - with self.assertRaisesRegex(policy.CatalogError, "must be 'agent_end'"): + loaded_turn = policy.load_catalog(write_catalog(root, valid_turn)) + self.assertEqual( + loaded_turn.targets["target-a"].runtime["terminal_success"], + "turn_completed", + ) + with self.assertRaisesRegex(policy.CatalogError, "must be 'agent_end' or"): policy.load_catalog(write_catalog(root, invalid_name)) with self.assertRaisesRegex(policy.CatalogError, "requires output_format='jsonl'"): policy.load_catalog(write_catalog(root, invalid_format)) diff --git a/agent-ops/skills/common/orchestrate-agent-task-loop/tests/test_select_execution_target.py b/agent-ops/skills/common/orchestrate-agent-task-loop/tests/test_select_execution_target.py index 1289b6c9..a218724a 100644 --- a/agent-ops/skills/common/orchestrate-agent-task-loop/tests/test_select_execution_target.py +++ b/agent-ops/skills/common/orchestrate-agent-task-loop/tests/test_select_execution_target.py @@ -188,8 +188,10 @@ class SelectorTests(unittest.TestCase): ) sol_high = catalog.targets["codex-sol-high"] self.assertEqual(sol_high.reasoning_effort, "high") + self.assertEqual(sol_high.runtime["terminal_success"], "turn_completed") sol_xhigh = catalog.targets["codex-sol-xhigh"] self.assertEqual(sol_xhigh.reasoning_effort, "xhigh") + self.assertEqual(sol_xhigh.runtime["terminal_success"], "turn_completed") def test_initial_decision_contains_catalog_evidence_and_no_quota(self): with TemporaryDirectory() as tmp: