sync: agent-ops from agentic-framework v1.1.204
This commit is contained in:
parent
7a4c74c81f
commit
85fc7d70db
3 changed files with 91 additions and 39 deletions
|
|
@ -111,7 +111,7 @@ Accept self-check completion only when `## Implementation Checklist` or its supp
|
|||
- Record the target id, opaque agent/model identity, execution class, runtime contract, catalog evidence, process identity, workspace identity, timestamps, result, and exact failure evidence.
|
||||
- Treat stderr as terminal diagnostic evidence. For JSONL, recognize generic terminal event fields such as error/fatal type or severity, rejected/failed status with an error code, explicit error flags, and a non-retrying `agent_end` whose last assistant message ends with `error` or `aborted`.
|
||||
- Determine liveness from PID/start-token/process-marker evidence and actual stream or native-session progress. Heartbeat mtime is never agent progress. For Codex JSONL, an unmatched `item.started` `command_execution` is an active tool interval: suspend the model-response silence timer until its matching `item.completed`, then restore normal stall detection.
|
||||
- The dispatcher model-silence safety net is 70 seconds. Downstream provider runtimes should emit their bounded terminal before that deadline; do not extend the dispatcher budget per target to cover nested retries.
|
||||
- The dispatcher model-silence safety net is 310 seconds. The dev Ornith provider's bounded response-stall terminal is 300 seconds, so the dispatcher remains slightly above it and observes that terminal instead of killing the caller first. Do not extend the dispatcher budget per target to cover nested retries.
|
||||
- Treat a confirmed provider transport terminal as the end of the current dispatch. Do not resume or automatically resend the same native session; an operator may start a fresh dispatch after the provider/runtime state is corrected.
|
||||
- When the selected target declares `session_stall_resume=true` and its JSONL emitted a runtime session id, terminate the silent process and invoke the catalog `resume_command` once for that exact same target and session with a continuation message. Do not inject a second continuation into the same stalled session; return to the existing bounded fresh-conversation retry and failover route. If the capability or runtime session id is absent, preserve workspace changes and logical locator evidence but retry with a fresh conversation. Never apply same-session continuation to provider transport terminals.
|
||||
- Never start a duplicate attempt while owned live evidence remains.
|
||||
|
|
@ -135,6 +135,8 @@ python3 agent-ops/skills/common/orchestrate-agent-task-loop/scripts/dispatch.py
|
|||
|
||||
Remove `--dry-run` to start execution. Add `--execution-catalog <path>` only to override the bundled default. Add `--task-group <name>`, `--max-parallel <n>`, or `--retry-blocked` only when requested by the workflow.
|
||||
|
||||
`--retry-blocked` is a forced fresh restart, never a continuation. Before using it, stop the dispatcher and confirm that no owned agent process is live. It preserves workspace edits, the active PLAN/CODE_REVIEW files, and failed run logs, but clears the scoped unfinished task's attempt counters, prior errors and blocker evidence, active locator/native-session linkage, recovery and generic failure budgets, persisted execution decisions, and route transition history. The next worker/reviewer must receive a newly generated session and an `initial` selector transition; it must not resume or inherit any earlier conversation. If owned live evidence remains, refuse the reset.
|
||||
|
||||
After an intentional catalog replacement invalidates a persisted incomplete worker decision, preview and accept it explicitly:
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -179,7 +179,10 @@ def validated_max_parallel(value: int) -> int:
|
|||
|
||||
|
||||
STREAM_HEARTBEAT_SECONDS = 30
|
||||
MODEL_RESPONSE_STALL_SECONDS = 70
|
||||
# The dev Ornith route allows five minutes for provider prefill/first output.
|
||||
# Keep the dispatcher safety net slightly above that downstream terminal so it
|
||||
# observes the provider result instead of terminating the caller first.
|
||||
MODEL_RESPONSE_STALL_SECONDS = 310
|
||||
RECOVERY_FAILURE_LIMIT = 10
|
||||
GENERIC_FAILURE_LIMIT_PER_TARGET = 3
|
||||
SELF_CHECK_UNCHECKED_RETRY_LIMIT = 10
|
||||
|
|
@ -1141,8 +1144,15 @@ class StateStore:
|
|||
self.save()
|
||||
return accepted
|
||||
|
||||
def mark_retry_failover(self, task_group: str | None = None, workspace: Path | None = None) -> None:
|
||||
def reset_for_fresh_restart(self, task_group: str | None = None) -> None:
|
||||
"""Reset unfinished dispatcher state without resuming prior attempts.
|
||||
|
||||
Operator-requested restart is a fresh execution boundary. Failed run
|
||||
artifacts stay on disk as evidence, but no locator, native session,
|
||||
selector transition, attempt number, or failure budget crosses it.
|
||||
"""
|
||||
prefix = f"{task_group}/" if task_group else None
|
||||
reset_tasks: set[str] = set()
|
||||
for task_name, value in self.data.get("tasks", {}).items():
|
||||
if (
|
||||
task_group is not None
|
||||
|
|
@ -1150,45 +1160,58 @@ class StateStore:
|
|||
and not task_name.startswith(prefix)
|
||||
):
|
||||
continue
|
||||
if not value.get("blocked"):
|
||||
continue
|
||||
blocker_evidence = value.get("blocker_evidence") if isinstance(value.get("blocker_evidence"), dict) else {}
|
||||
decisions = value.get("execution_decisions", {})
|
||||
worker_decision = decisions.get("worker") if isinstance(decisions, dict) else None
|
||||
role = blocker_evidence.get("role")
|
||||
failure_class = blocker_evidence.get("failure_class")
|
||||
locator = blocker_evidence.get("locator")
|
||||
selected = blocker_evidence.get("selected")
|
||||
work_unit_id = blocker_evidence.get("work_unit_id")
|
||||
qualified = (
|
||||
role == "worker"
|
||||
and failure_class in QUALIFIED_FAILOVER_FAILURES
|
||||
and isinstance(locator, str)
|
||||
and locator.strip()
|
||||
and isinstance(selected, dict)
|
||||
and isinstance(work_unit_id, str)
|
||||
and isinstance(worker_decision, dict)
|
||||
and worker_decision.get("work_unit_id") == work_unit_id
|
||||
unfinished = (
|
||||
not value.get("worker_done")
|
||||
or bool(value.get("blocked"))
|
||||
or bool(value.get("active_locator"))
|
||||
or bool(value.get("retry_failover_pending"))
|
||||
)
|
||||
handoff_id = str(uuid.uuid4())
|
||||
retry_context = ({
|
||||
"role": role,
|
||||
"failure_class": failure_class,
|
||||
"locator": locator,
|
||||
"selected": selected,
|
||||
"work_unit_id": work_unit_id,
|
||||
"handoff_id": handoff_id,
|
||||
} if qualified else None)
|
||||
if not unfinished:
|
||||
continue
|
||||
live, detail = external_active_is_live(
|
||||
value,
|
||||
expected_workspace=self.workspace,
|
||||
expected_workspace_id=self.workspace_id,
|
||||
expected_runs_root=self.runs,
|
||||
)
|
||||
if live:
|
||||
raise DispatcherTerminalStateError(
|
||||
"fresh restart 전에 실행 중 agent를 중단해야 한다: "
|
||||
f"task={task_name} detail={detail}"
|
||||
)
|
||||
|
||||
value["blocked"] = None
|
||||
value["blocker_evidence"] = None
|
||||
value["active_stage"] = None
|
||||
value["active_locator"] = None
|
||||
value["active_started_at"] = None
|
||||
value["review_no_progress"] = 0
|
||||
value["selfcheck_incomplete"] = 0
|
||||
value["selfcheck_context_locator"] = None
|
||||
value["recovery_failures"] = {}
|
||||
value["stage_failure_budgets"] = {}
|
||||
value["retry_failover_pending"] = qualified
|
||||
value["retry_failover_context"] = retry_context
|
||||
value["blocker_evidence"] = None
|
||||
value["generic_failure_budgets"] = {}
|
||||
value["retry_failover_pending"] = False
|
||||
value["retry_failover_context"] = None
|
||||
value["execution_decisions"] = {}
|
||||
value["route_transition_history"] = []
|
||||
if not value.get("worker_done"):
|
||||
value["worker_cli"] = None
|
||||
value["worker_model"] = None
|
||||
value["selfcheck_done"] = False
|
||||
reset_tasks.add(task_name)
|
||||
|
||||
counters = self.data.setdefault("attempt_counters", {})
|
||||
for key in list(counters):
|
||||
task_name = key.split("|", 1)[0]
|
||||
if task_name in reset_tasks:
|
||||
del counters[key]
|
||||
|
||||
claims = self.data.setdefault("write_claims", {})
|
||||
for task_name in reset_tasks:
|
||||
claims.pop(task_name, None)
|
||||
if reset_tasks:
|
||||
self.write_claim_snapshot()
|
||||
self.save()
|
||||
|
||||
|
||||
|
|
@ -6143,7 +6166,7 @@ async def dispatch_with_store(
|
|||
) -> int:
|
||||
orchestration_scope = args.task_group or "__all__"
|
||||
if args.retry_blocked and not args.dry_run:
|
||||
store.mark_retry_failover(args.task_group)
|
||||
store.reset_for_fresh_restart(args.task_group)
|
||||
running: dict[str, asyncio.Task[str | None]] = {}
|
||||
last_wait: dict[str, str] = {}
|
||||
completed_tasks: dict[str, str] = {}
|
||||
|
|
@ -6922,7 +6945,14 @@ def parse_args() -> argparse.Namespace:
|
|||
),
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true", help="classify and print without launching CLIs")
|
||||
parser.add_argument("--retry-blocked", action="store_true", help="clear dispatcher-local blocked state")
|
||||
parser.add_argument(
|
||||
"--retry-blocked",
|
||||
action="store_true",
|
||||
help=(
|
||||
"fresh-restart unfinished tasks: clear attempt counters, prior "
|
||||
"errors, locators/sessions, failure budgets, and selector history"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--accept-catalog-revision",
|
||||
action="store_true",
|
||||
|
|
|
|||
|
|
@ -356,7 +356,7 @@ class RuntimeCatalogDispatcherTests(unittest.TestCase):
|
|||
decision["selected"],
|
||||
)
|
||||
|
||||
def test_retry_blocked_marks_failover_without_quota_state(self):
|
||||
def test_retry_blocked_resets_to_fresh_attempt_without_prior_context(self):
|
||||
with TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
catalog = write_catalog(root)
|
||||
|
|
@ -370,6 +370,8 @@ class RuntimeCatalogDispatcherTests(unittest.TestCase):
|
|||
state = store.task_state(task)
|
||||
state.update(
|
||||
blocked="runtime failure",
|
||||
active_stage=None,
|
||||
active_locator=None,
|
||||
blocker_evidence={
|
||||
"role": "worker",
|
||||
"failure_class": "provider-quota",
|
||||
|
|
@ -377,13 +379,31 @@ class RuntimeCatalogDispatcherTests(unittest.TestCase):
|
|||
"selected": decision["selected"],
|
||||
"work_unit_id": decision["work_unit_id"],
|
||||
},
|
||||
recovery_failures={"worker": 3},
|
||||
stage_failure_budgets={"unit|worker": {"count": 3}},
|
||||
generic_failure_budgets={"unit|worker|target": {"count": 2}},
|
||||
route_transition_history=[{"transition": "resume"}],
|
||||
retry_failover_pending=True,
|
||||
retry_failover_context={"locator": "/tmp/locator.json"},
|
||||
)
|
||||
counter_key = f"{task.name}|{task.plan_hash}|worker"
|
||||
store.data.setdefault("attempt_counters", {})[counter_key] = 5
|
||||
store.save()
|
||||
store.mark_retry_failover("group")
|
||||
store.reset_for_fresh_restart("group")
|
||||
state = store.task_state(task)
|
||||
counters = dict(store.data["attempt_counters"])
|
||||
finally:
|
||||
store.close()
|
||||
self.assertTrue(state["retry_failover_pending"])
|
||||
self.assertFalse(state["retry_failover_pending"])
|
||||
self.assertIsNone(state["retry_failover_context"])
|
||||
self.assertIsNone(state["blocker_evidence"])
|
||||
self.assertIsNone(state["blocked"])
|
||||
self.assertEqual(state["execution_decisions"], {})
|
||||
self.assertEqual(state["route_transition_history"], [])
|
||||
self.assertEqual(state["recovery_failures"], {})
|
||||
self.assertEqual(state["stage_failure_budgets"], {})
|
||||
self.assertEqual(state["generic_failure_budgets"], {})
|
||||
self.assertNotIn(counter_key, counters)
|
||||
self.assertNotIn("quota_snapshot", state)
|
||||
self.assertNotIn("retry_quota_refresh_pending", state)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue