diff --git a/agent-ops/skills/project/orchestrate-agent-task-loop/SKILL.md b/agent-ops/skills/project/orchestrate-agent-task-loop/SKILL.md index 9b0ec64a..8d1000e3 100644 --- a/agent-ops/skills/project/orchestrate-agent-task-loop/SKILL.md +++ b/agent-ops/skills/project/orchestrate-agent-task-loop/SKILL.md @@ -77,7 +77,7 @@ Dispatcher-child re-entry boundary: | `local-G01`–`local-G06` | Pi `iop/ornith:35b`, thinking high | | `local-G07`–`local-G08` | KST `[07:00,23:00)` agy `Gemini 3.6 Flash (Medium)`; `[23:00,07:00)` Pi `iop/laguna-s:2.1` | | `local-G09`–`local-G10` | Claude `claude-opus-4-8`, effort xhigh | -| `cloud-G01`–`cloud-G02` | agy `Gemini 3.6 Flash (Low)` | +| `cloud-G01`–`cloud-G02` | Codex `gpt-5.3-codex-spark`, reasoning xhigh → on quota/failover agy `Gemini 3.6 Flash (Low)` → Claude `claude-haiku-4-5`, effort xhigh | | `cloud-G03`–`cloud-G04` | agy `Gemini 3.6 Flash (Medium)` | | `cloud-G05`–`cloud-G06` | agy `Gemini 3.6 Flash (High)` | | `cloud-G07`–`cloud-G08` | Claude `claude-opus-4-8`, effort xhigh | @@ -234,7 +234,7 @@ When recovering a KST-night `local-G07`–`local-G08` Laguna locator or a termin - Archive `WORK_LOG.md` as `work_log_N.log` only after the final task review process exits, the dispatcher appends `FINISH`, and a complete scan finds no active/running task in that group. Accept the log at either the active group path or the verified completed single-task archive; do not impose either location contract on common plan/code-review. 3. **Escalate and recover context.** - - Escalate `agy -> Claude -> Codex` or `Claude -> Codex` only on terminal provider error events or stderr evidence of context/output limits, provider quota/rate limits, unavailable models, or confirmed provider transport errors. For AGY, accept top-level `error`, `fatal`, `request.failed`, or `turn.failed` events; failed/rejected status with a top-level error/code; stderr; or strong `RESOURCE_EXHAUSTED`, HTTP 429, quota, or rate-limit evidence in `agy-cli.log`. For Claude, classify a `rate_limit_event` with `rate_limit_info.status=rejected`, an error `result` with `api_error_status=429` or `error=rate_limit`, or a `You've hit your session limit · resets ...` terminal diagnostic as `provider-quota`. Never escalate from an assistant message, source text, tool/test output, or a plain quota-configuration string in an AGY log. + - For `cloud-G01`–`cloud-G02`, fail over in the pinned candidate order `gpt-5.3-codex-spark -> Gemini 3.6 Flash (Low) -> claude-haiku-4-5`. For single-candidate routes, escalate `agy -> Claude -> Codex` or `Claude -> Codex`. Apply either transition only on terminal provider error events or stderr evidence of context/output limits, provider quota/rate limits, unavailable models, or confirmed provider transport errors. For AGY, accept top-level `error`, `fatal`, `request.failed`, or `turn.failed` events; failed/rejected status with a top-level error/code; stderr; or strong `RESOURCE_EXHAUSTED`, HTTP 429, quota, or rate-limit evidence in `agy-cli.log`. For Claude, classify a `rate_limit_event` with `rate_limit_info.status=rejected`, an error `result` with `api_error_status=429` or `error=rate_limit`, or a `You've hit your session limit · resets ...` terminal diagnostic as `provider-quota`. Never escalate from an assistant message, source text, tool/test output, or a plain quota-configuration string in an AGY log. - Target Codex `gpt-5.6-terra` with reasoning `high` when escalating from Claude to Codex. - If Codex returns the same error, retry in a fresh Codex session using the locator while preserving the previous Codex model/reasoning and sharing the same stage's 10-consecutive-failure limit. Continue dispatching other tasks during recovery. - When current source reads a locator blocked 10 times as `generic-error` by older dispatcher source, collapse those 10 failures into one terminal error and clear only that task's blocker only if all 10 terminal-evidence records for the same task/plan/role/source/execution target reclassify to the same escalatable error. Include `stream.log` and the attempt's `agy-cli.log` for AGY. Do not adjust automatically when any history is missing or mixed, or when the locator dispatcher source hash equals the current source hash. Dry-run must display this escalation recovery and next model without writing state. Live execution must choose the higher target from the locator's actual failed target, not the initial PLAN route, inherit locator context, and restore the same escalation target and locator from persisted reclassification metadata after immediate restart. diff --git a/agent-ops/skills/project/orchestrate-agent-task-loop/scripts/execution_target_policy.py b/agent-ops/skills/project/orchestrate-agent-task-loop/scripts/execution_target_policy.py index f9519703..6028d622 100644 --- a/agent-ops/skills/project/orchestrate-agent-task-loop/scripts/execution_target_policy.py +++ b/agent-ops/skills/project/orchestrate-agent-task-loop/scripts/execution_target_policy.py @@ -44,6 +44,12 @@ AGY_GEMINI_HIGH = RouteTarget( ) PI_LAGUNA = RouteTarget("pi", "iop/laguna-s:2.1", "local_model", True) CLAUDE_OPUS = RouteTarget("claude", "claude-opus-4-8", "cloud_model", False) +CLAUDE_HAIKU_XHIGH = RouteTarget( + "claude", "claude-haiku-4-5", "cloud_model", False +) +CODEX_SPARK_XHIGH = RouteTarget( + "codex", "gpt-5.3-codex-spark", "cloud_model", False +) CODEX_SOL_XHIGH = RouteTarget("codex", "gpt-5.6-sol", "cloud_model", False) CODEX_TERRA_HIGH = RouteTarget("codex", "gpt-5.6-terra", "cloud_model", False) @@ -55,6 +61,8 @@ CANONICAL_TARGETS = ( AGY_GEMINI_HIGH, PI_LAGUNA, CLAUDE_OPUS, + CLAUDE_HAIKU_XHIGH, + CODEX_SPARK_XHIGH, CODEX_SOL_XHIGH, CODEX_TERRA_HIGH, ) @@ -180,23 +188,27 @@ def select_policy( ) if grade <= 2: - target = AGY_GEMINI_LOW + candidates = ( + CODEX_SPARK_XHIGH, + AGY_GEMINI_LOW, + CLAUDE_HAIKU_XHIGH, + ) rule_id = "worker-cloud-g01-g02" - reason_code = "cloud_gemini_low_grade" + reason_code = "cloud_spark_priority_grade" elif grade <= 4: - target = AGY_GEMINI_MEDIUM + candidates = (AGY_GEMINI_MEDIUM,) rule_id = "worker-cloud-g03-g04" reason_code = "cloud_gemini_medium_grade" elif grade <= 6: - target = AGY_GEMINI_HIGH + candidates = (AGY_GEMINI_HIGH,) rule_id = "worker-cloud-g05-g06" reason_code = "cloud_gemini_high_grade" elif grade <= 8: - target = CLAUDE_OPUS + candidates = (CLAUDE_OPUS,) rule_id = "worker-cloud-g07-g08" reason_code = "cloud_opus_grade" else: - target = CODEX_SOL_XHIGH + candidates = (CODEX_SOL_XHIGH,) rule_id = "worker-cloud-g09-g10" reason_code = "cloud_codex_grade" return PolicyDecision( @@ -204,5 +216,5 @@ def select_policy( policy_priority=30, reason_codes=(reason_code,), time_window="not_applicable", - candidates=(target,), + candidates=candidates, ) diff --git a/agent-ops/skills/project/orchestrate-agent-task-loop/tests/test_dispatch.py b/agent-ops/skills/project/orchestrate-agent-task-loop/tests/test_dispatch.py index cc1d362a..d243345b 100644 --- a/agent-ops/skills/project/orchestrate-agent-task-loop/tests/test_dispatch.py +++ b/agent-ops/skills/project/orchestrate-agent-task-loop/tests/test_dispatch.py @@ -3106,13 +3106,25 @@ class ReviewControlTest(unittest.TestCase): "gpt-5.6-sol", "codex/gpt-5.6-sol xhigh", ) + spark = dispatch.AgentSpec( + "codex", + "gpt-5.3-codex-spark", + "codex/gpt-5.3-codex-spark xhigh", + ) claude = dispatch.AgentSpec( "claude", "claude-opus-4-8", "claude/claude-opus-4-8 xhigh", ) + haiku = dispatch.AgentSpec( + "claude", + "claude-haiku-4-5", + "claude/claude-haiku-4-5 xhigh", + ) self.assertEqual(dispatch.effective_reasoning_effort(codex), "xhigh") + self.assertEqual(dispatch.effective_reasoning_effort(spark), "xhigh") self.assertEqual(dispatch.effective_reasoning_effort(claude), "xhigh") + self.assertEqual(dispatch.effective_reasoning_effort(haiku), "xhigh") def test_classifies_provider_tunnel_connection_refusal(self): provider_line = ( @@ -7878,6 +7890,112 @@ class DispatcherCanonicalFailoverIntegrationTest(unittest.IsolatedAsyncioTestCas locator.write_text(json.dumps(record), encoding="utf-8") return locator + async def test_cloud_g01_g02_quota_failover_runs_spark_gemini_haiku(self): + daytime = datetime( + 2026, 7, 26, 14, 0, 0, tzinfo=timezone(timedelta(hours=9)) + ) + with tempfile.TemporaryDirectory() as temporary: + workspace = Path(temporary) + (workspace / ".git").mkdir() + task = self.make_task(workspace, lane="cloud", grade=1) + store = dispatch.StateStore(workspace) + selector = dispatch._selector_module() + + def unknown_quota_probe(*args, **kwargs): + adapter = kwargs["adapter"] + target = kwargs["target"] + return { + "schema_version": "1.0", + "snapshot_id": f"unknown-{adapter}-{target}", + "source": "iop-node quota-probe", + "checked_at": kwargs["checked_at"].isoformat(), + "targets": [ + { + "adapter": adapter, + "target": target, + "status": "unknown", + } + ], + "required_caps": [], + "reason_codes": ["checker_error"], + } + + try: + with mock.patch.object( + selector, + "probe_candidate_quota", + side_effect=unknown_quota_probe, + ): + _, initial_spec = dispatch.persisted_execution_decision( + store, + task, + stage="worker", + evaluated_at=daytime, + ) + specs = { + "codex": initial_spec, + "agy": dispatch.AgentSpec( + "agy", + "Gemini 3.6 Flash (Low)", + "agy/Gemini 3.6 Flash (Low)", + ), + "claude": dispatch.AgentSpec( + "claude", + "claude-haiku-4-5", + "claude/claude-haiku-4-5 xhigh", + ), + } + locators = { + cli: self.make_attempt_locator(workspace, task, spec) + for cli, spec in specs.items() + } + invoked_specs = [] + + async def mock_invoke(*args, **kwargs): + spec = args[4] + invoked_specs.append(spec) + if spec.cli == "claude": + return 0, None, locators[spec.cli] + return 1, "provider-quota", locators[spec.cli] + + with ( + mock.patch.object(dispatch, "invoke", new=mock_invoke), + mock.patch.object( + dispatch.asyncio, + "sleep", + new=mock.AsyncMock(), + ), + ): + success, final_locator = await dispatch.run_escalating( + workspace, + store, + task, + "worker", + initial_spec, + ) + + self.assertTrue(success) + self.assertEqual(final_locator, locators["claude"]) + self.assertEqual( + [(spec.cli, spec.model) for spec in invoked_specs], + [ + ("codex", "gpt-5.3-codex-spark"), + ("agy", "Gemini 3.6 Flash (Low)"), + ("claude", "claude-haiku-4-5"), + ], + ) + decision = store.task_state(task)["execution_decisions"]["worker"] + self.assertEqual( + decision["used_candidates"], + [ + {"adapter": "codex", "target": "gpt-5.3-codex-spark"}, + {"adapter": "agy", "target": "Gemini 3.6 Flash (Low)"}, + {"adapter": "claude", "target": "claude-haiku-4-5"}, + ], + ) + finally: + store.close() + async def test_invalid_logical_context_does_not_commit_or_promote(self): daytime = datetime(2026, 7, 26, 14, 0, 0, tzinfo=timezone(timedelta(hours=9))) diff --git a/agent-ops/skills/project/orchestrate-agent-task-loop/tests/test_execution_target_policy.py b/agent-ops/skills/project/orchestrate-agent-task-loop/tests/test_execution_target_policy.py index 417f6175..cefa9eac 100644 --- a/agent-ops/skills/project/orchestrate-agent-task-loop/tests/test_execution_target_policy.py +++ b/agent-ops/skills/project/orchestrate-agent-task-loop/tests/test_execution_target_policy.py @@ -68,7 +68,7 @@ class ExecutionTargetPolicyTests(unittest.TestCase): }, "cloud": { **{ - grade: ("agy", "Gemini 3.6 Flash (Low)", False) + grade: ("codex", "gpt-5.3-codex-spark", False) for grade in range(1, 3) }, **{ @@ -103,6 +103,28 @@ class ExecutionTargetPolicyTests(unittest.TestCase): route, ) + def test_cloud_g01_g02_uses_ordered_spark_gemini_haiku_candidates(self): + for grade in (1, 2): + with self.subTest(grade=grade): + decision = policy.select_policy( + stage="worker", + lane="cloud", + grade=grade, + evaluated_at=at_utc(3), + ) + self.assertEqual( + decision.candidates, + ( + policy.CODEX_SPARK_XHIGH, + policy.AGY_GEMINI_LOW, + policy.CLAUDE_HAIKU_XHIGH, + ), + ) + self.assertEqual( + decision.reason_codes, + ("cloud_spark_priority_grade",), + ) + def test_review_matrix_is_fixed_to_codex(self): for lane in ("local", "cloud"): for grade in range(1, 11): @@ -166,6 +188,8 @@ class ExecutionTargetPolicyTests(unittest.TestCase): (policy.AGY_GEMINI_MEDIUM, policy.CLAUDE_OPUS), (policy.AGY_GEMINI_HIGH, policy.CLAUDE_OPUS), (policy.CLAUDE_OPUS, policy.CODEX_TERRA_HIGH), + (policy.CLAUDE_HAIKU_XHIGH, None), + (policy.CODEX_SPARK_XHIGH, None), (policy.CODEX_SOL_XHIGH, None), (policy.CODEX_TERRA_HIGH, None), (policy.PI_ORNITH, None), @@ -203,6 +227,14 @@ class ExecutionTargetPolicyTests(unittest.TestCase): policy.CLAUDE_OPUS, policy.QuotaProbeSpec("claude", "claude-opus-4-8", ("overall",)), ), + ( + policy.CLAUDE_HAIKU_XHIGH, + policy.QuotaProbeSpec("claude", "claude-haiku-4-5", ("overall",)), + ), + ( + policy.CODEX_SPARK_XHIGH, + policy.QuotaProbeSpec("codex", "gpt-5.3-codex-spark", ("overall",)), + ), ( policy.CODEX_SOL_XHIGH, policy.QuotaProbeSpec("codex", "gpt-5.6-sol", ("overall",)), diff --git a/agent-ops/skills/project/orchestrate-agent-task-loop/tests/test_select_execution_target.py b/agent-ops/skills/project/orchestrate-agent-task-loop/tests/test_select_execution_target.py index f44816d2..91a10380 100644 --- a/agent-ops/skills/project/orchestrate-agent-task-loop/tests/test_select_execution_target.py +++ b/agent-ops/skills/project/orchestrate-agent-task-loop/tests/test_select_execution_target.py @@ -376,8 +376,8 @@ class SelectorRouteMatrixTests(unittest.TestCase): 10: ("claude", "claude-opus-4-8", "cloud_model", False), }, "cloud": { - 1: ("agy", "Gemini 3.6 Flash (Low)", "cloud_model", False), - 2: ("agy", "Gemini 3.6 Flash (Low)", "cloud_model", False), + 1: ("codex", "gpt-5.3-codex-spark", "cloud_model", False), + 2: ("codex", "gpt-5.3-codex-spark", "cloud_model", False), 3: ("agy", "Gemini 3.6 Flash (Medium)", "cloud_model", False), 4: ("agy", "Gemini 3.6 Flash (Medium)", "cloud_model", False), 5: ("agy", "Gemini 3.6 Flash (High)", "cloud_model", False), @@ -1063,6 +1063,69 @@ class SelectorIdentityAndQuotaRoundtripTests(unittest.TestCase): class SelectorFailoverContractTests(unittest.TestCase): + def test_cloud_g01_g02_quota_failover_follows_spark_gemini_haiku_order(self): + with TemporaryDirectory() as tmp: + task_file = write_task_file(Path(tmp), "PLAN", "cloud", 1) + initial = selector.select_execution_target( + task_file, + evaluated_at=kst(12), + quota_probe_command="missing-probe", + ) + gemini = selector.select_execution_target( + task_file, + evaluated_at=kst(12), + transition="failover", + prior_decision=initial, + failure_class="provider-quota", + quota_probe_command="missing-probe", + ) + haiku = selector.select_execution_target( + task_file, + evaluated_at=kst(12), + transition="failover", + prior_decision=gemini, + failure_class="provider-quota", + quota_probe_command="missing-probe", + ) + + self.assertEqual( + [ + (candidate["adapter"], candidate["target"]) + for candidate in initial["candidates"] + ], + [ + ("codex", "gpt-5.3-codex-spark"), + ("agy", "Gemini 3.6 Flash (Low)"), + ("claude", "claude-haiku-4-5"), + ], + ) + self.assertEqual( + (gemini["selected"]["adapter"], gemini["selected"]["target"]), + ("agy", "Gemini 3.6 Flash (Low)"), + ) + self.assertEqual( + (haiku["selected"]["adapter"], haiku["selected"]["target"]), + ("claude", "claude-haiku-4-5"), + ) + self.assertEqual( + haiku["used_candidates"], + [ + {"adapter": "codex", "target": "gpt-5.3-codex-spark"}, + {"adapter": "agy", "target": "Gemini 3.6 Flash (Low)"}, + {"adapter": "claude", "target": "claude-haiku-4-5"}, + ], + ) + with self.assertRaises(selector.SelectorInputError) as exhausted: + selector.select_execution_target( + task_file, + evaluated_at=kst(12), + transition="failover", + prior_decision=haiku, + failure_class="provider-quota", + quota_probe_command="missing-probe", + ) + self.assertEqual(exhausted.exception.code, "no_failover_candidate") + def test_qualified_failover_uses_only_unused_eligible_candidate(self): with TemporaryDirectory() as tmp: task_file = write_task_file(Path(tmp), "PLAN", "local", 8)