#!/usr/bin/env python3 """Dispatch every independently-ready agent-task pair until the task group completes.""" from __future__ import annotations import argparse import asyncio import fcntl import importlib.util import hashlib import json import os import re import signal import shutil import subprocess import sys import uuid from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any _OBSERVATION_MODULE_NAME = "agent_task_dispatcher_observation" def load_sibling_observation_module(): loaded = sys.modules.get(_OBSERVATION_MODULE_NAME) if loaded is not None: return loaded spec = importlib.util.spec_from_file_location( _OBSERVATION_MODULE_NAME, Path(__file__).with_name("dispatcher_observation.py"), ) if spec is None or spec.loader is None: raise RuntimeError("failed to load dispatcher observation module") module = importlib.util.module_from_spec(spec) sys.modules[_OBSERVATION_MODULE_NAME] = module try: spec.loader.exec_module(module) except BaseException: sys.modules.pop(_OBSERVATION_MODULE_NAME, None) raise return module observation = load_sibling_observation_module() SEP = observation.SEP banner = observation.banner attempt_event = observation.attempt_event validation_claim = observation.validation_claim PLAN_RE = re.compile(r"^PLAN-(local|cloud)-G(0[1-9]|10)\.md$") REVIEW_RE = re.compile(r"^CODE_REVIEW-(local|cloud)-G(0[1-9]|10)\.md$") PLAN_LOG_RE = re.compile( r"^plan_(local|cloud)_G(0[1-9]|10)_(0|[1-9][0-9]*)\.log$" ) REVIEW_LOG_RE = re.compile( r"^code_review_(local|cloud)_G(0[1-9]|10)_(0|[1-9][0-9]*)\.log$" ) SUBTASK_RE = re.compile(r"^(?P\d{2})(?:\+(?P\d{2}(?:,\d{2})*))?_[a-z0-9_]+$") MODIFIED_FILES_HEADINGS = ("Modified Files Summary", "수정 파일 요약") MODIFIED_FILES_HEADER_CELLS = frozenset({"file", "files", "path", "paths", "파일", "경로"}) PLACEHOLDER_PATH_RE = re.compile( r"(?:[<>{}]|\.\.\.|(?:^|[/_.-])(?:tbd|todo|placeholder)(?:$|[/_.-]))", re.IGNORECASE, ) IMPLEMENTATION_CHECKLIST_HEADINGS = ("Implementation Checklist", "구현 체크리스트") # The canonical English and legacy Korean verdict contracts are paired: a # heading only accepts the verdict label of its own schema. Mixed pairs are not # a documented schema and must fail closed. CODE_REVIEW_RESULT_SCHEMAS = ( ("Code Review Result", "Overall Verdict"), ("코드리뷰 결과", "종합 판정"), ) VERDICT_SCHEMA_MATCHERS = tuple( ( re.compile(rf"^##\s*{re.escape(heading)}[ \t]*$", re.MULTILINE), re.compile( rf"^(?:-\s*)?(?:\*\*)?{re.escape(label)}(?:\*\*)?\s*:\s*(PASS|WARN|FAIL)[ \t]*$", re.MULTILINE, ), re.compile( rf"^###\s+{re.escape(label)}[ \t]*$\s*^(?:\*\*)?(PASS|WARN|FAIL)(?:\*\*)?[ \t]*$", re.MULTILINE, ), ) for heading, label in CODE_REVIEW_RESULT_SCHEMAS ) PLAN_IDENTITY_RE = re.compile( r"" ) IMPLEMENTATION_CHECKBOX_RE = re.compile( r"^-\s+\[([^\]\r\n]*)\]", re.MULTILINE ) WORK_LOG_NAME = "WORK_LOG.md" WORK_LOG_ARCHIVE_RE = re.compile(r"^work_log_(\d+)\.log$") AGENT_PROCESS_MARKER_ENV = "IOP_AGENT_TASK_EXECUTION_ID" KST = timezone(timedelta(hours=9), name="KST") def validated_max_parallel(value: int) -> int: """Validate and return a non-negative integer for --max-parallel. Rejects negative values and non-integer types. Used both for CLI argument parsing and for programmatic callers that may pass arbitrary namespaces. """ if not isinstance(value, int) or isinstance(value, bool): raise ValueError( f"--max-parallel must be an integer >= 0, got {value!r}" ) if value < 0: raise ValueError( f"--max-parallel must be >= 0, got {value}" ) return value STREAM_HEARTBEAT_SECONDS = 30 PI_MODEL_RESPONSE_STALL_SECONDS = 3 * 60 PI_SESSION_SCHEMA_VERSION = 3 RECOVERY_FAILURE_LIMIT = 10 SELF_CHECK_INCOMPLETE_LIMIT = 10 REVIEW_NO_PROGRESS_LIMIT = 10 PROVIDER_TRANSPORT_FAILURES = frozenset( {"provider-connection", "provider-stream-disconnect"} ) FAILURE_EVIDENCE_LIMIT = 2000 # Used only to reject a stale locator whose dispatcher and agent PIDs are both # gone. A live process is inspected after silence; it is never killed solely by # this fallback clock. CODEX_STREAM_STALL_SECONDS = 5 * 60 PROMOTABLE_PATTERNS = { "context-limit": [ r"context (?:length|window)", r"maximum context", r"prompt is too long", r"too many tokens", r"token limit", r"exceeded.{0,40}token", r"output (?:token )?limit", r"maximum output", r"\bmax_tokens\b", r"response (?:is )?too long", ], "provider-quota": [ r"rate.?limit", r"\bquota\b", r"resource[_ ]?exhausted", r"\b429\b", r"too many requests", r"usage limit", r"capacity limit", r"\bsession limit\b", ], "model-unavailable": [ r"model.{0,40}(?:not found|unavailable)", r"overloaded", r"temporarily unavailable", ], "provider-connection": [ r"\bprovider[_ -]?tunnel[_ -]?error\b", ( r"(?:provider|backend|/v1/chat/completions|/v1/responses)" r".{0,160}(?:connection refused|dial tcp)" ), ], "provider-stream-disconnect": [ r"backend connection failed during streaming request", r"sse stream before done", r"llama-server was unresponsive", r"backend watchdog", r"model will be reloaded automatically on retry", ( r"(?:provider|backend|sse).{0,160}" r"curl error: failure when receiving data from the peer" ), ], } PROMOTABLE_FAILURES = frozenset( {"context-limit", "provider-quota", "model-unavailable"} ) CLOUD_PROMOTION_FAILURES = PROMOTABLE_FAILURES | PROVIDER_TRANSPORT_FAILURES QUALIFIED_FAILOVER_FAILURES = frozenset( {"provider-quota", "context-limit", "model-unavailable", "provider-stream-disconnect"} ) class DispatcherAlreadyRunning(RuntimeError): """A live dispatcher owns the workspace; this is non-terminal tracking state.""" class DispatcherTerminalStateError(RuntimeError): """Persistent workspace state prevents safe dispatch before work can start.""" class DispatcherInterruptedWithActiveWork(RuntimeError): """A control-plane error occurred after one or more agent tasks had started.""" class ExecutionDecisionError(RuntimeError): """A selector decision is invalid for this task and must fail closed.""" def now_iso() -> str: return datetime.now(timezone.utc).isoformat() def work_log_now_kst() -> str: return datetime.now(KST).strftime("%y-%m-%d %H:%M:%S") def sha256_file(path: Path | None) -> str: if path is None or not path.exists(): return "none" digest = hashlib.sha256() with path.open("rb") as stream: for chunk in iter(lambda: stream.read(65536), b""): digest.update(chunk) return digest.hexdigest() DISPATCHER_SOURCE_PATH = Path(__file__).resolve() DISPATCHER_SOURCE_SHA256 = sha256_file(DISPATCHER_SOURCE_PATH) DISPATCHER_PROCESS_STARTED_AT = now_iso() def dispatcher_source_provenance() -> dict[str, Any]: current_sha256 = sha256_file(DISPATCHER_SOURCE_PATH) return { "dispatcher_pid": os.getpid(), "dispatcher_process_start_token": process_start_token(os.getpid()), "dispatcher_process_started_at": DISPATCHER_PROCESS_STARTED_AT, "dispatcher_source_path": str(DISPATCHER_SOURCE_PATH), "dispatcher_source_sha256": DISPATCHER_SOURCE_SHA256, "dispatcher_source_current_sha256": current_sha256, "dispatcher_source_matches_loaded": current_sha256 == DISPATCHER_SOURCE_SHA256, } def plan_identity(path: Path | None) -> str: if path is None or not path.exists(): return "none" text = path.read_text(encoding="utf-8", errors="replace")[:1024] match = PLAN_IDENTITY_RE.search(text) if not match: return sha256_file(path) identity = "\0".join(match.group(name) for name in ("task", "plan", "tag")) return "meta:" + hashlib.sha256(identity.encode()).hexdigest() def write_json(path: Path, value: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + ".tmp") temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") temporary.replace(path) def milestone_work_log_path(task: Task) -> Path: return ( task.directory.parent / WORK_LOG_NAME if "/" in task.name else task.directory / WORK_LOG_NAME ) def append_work_log_event( path: Path, *, task_name: str, event: str, execution_id: str, role: str, attempt: int, model: str, result: str, locator: Path, ) -> Path: path.parent.mkdir(parents=True, exist_ok=True) with path.open("a+", encoding="utf-8") as stream: fcntl.flock(stream.fileno(), fcntl.LOCK_EX) try: stream.seek(0) text = stream.read() if not text: stream.write( "# Milestone Work Log\n\n" "> Dispatcher-owned execution timeline. Workers and reviewers do not edit this file.\n\n" "| seq | time | event | task | role | attempt | model | result | locator |\n" "|---:|---|---|---|---|---:|---|---|---|\n" ) sequence = 1 else: stream.seek(0, os.SEEK_END) if "| seq | time | event | task | role | attempt | model | result | locator |" not in text: if not text.endswith("\n"): stream.write("\n") stream.write( "\n## Dispatcher Timeline\n\n" "> Dispatcher-owned. Workers and reviewers do not edit this section.\n\n" "| seq | time | event | task | role | attempt | model | result | locator |\n" "|---:|---|---|---|---|---:|---|---|---|\n" ) sequence = 1 + max( ( int(match.group(1)) for match in re.finditer(r"^\|\s*(\d+)\s*\|", text, re.MULTILINE) ), default=0, ) if not text.endswith("\n"): stream.write("\n") def cell(value: Any) -> str: return str(value).replace("|", r"\|").replace("\n", " ") stream.write( f"| {sequence} | {work_log_now_kst()} | {cell(event)} | " f"{cell(task_name)} | " f"{cell(role)} | {attempt} | {cell(model)} | {cell(result)} | " f"{cell(locator.resolve())} |\n" ) stream.flush() finally: fcntl.flock(stream.fileno(), fcntl.LOCK_UN) return path def append_milestone_event( task: Task, *, event: str, execution_id: str, role: str, attempt: int, model: str, result: str, locator: Path, ) -> Path: return append_work_log_event( milestone_work_log_path(task), task_name=task.name, event=event, execution_id=execution_id, role=role, attempt=attempt, model=model, result=result, locator=locator, ) def safe_name(value: str) -> str: return re.sub(r"[^A-Za-z0-9_.-]+", "__", value).strip("_") or "task" @dataclass(frozen=True) class AgentSpec: cli: str model: str display: str local_pi: bool = False reasoning_effort: str | None = None def effective_reasoning_effort(spec: AgentSpec) -> str | None: if spec.cli in {"codex", "claude"}: return spec.reasoning_effort or "xhigh" return None def agent_spec_from_record(record: dict[str, Any]) -> AgentSpec | None: cli = str(record.get("cli") or "") model = str(record.get("model") or "") if not cli or not model: return None reasoning_effort = record.get("reasoning_effort") if reasoning_effort is not None: reasoning_effort = str(reasoning_effort) local_pi = cli == "pi" if cli in {"codex", "claude"}: effort = reasoning_effort or "xhigh" display = f"{cli}/{model} {effort}" elif cli == "pi": display = f"pi/iop/{model}" else: display = f"{cli}/{model}" return AgentSpec( cli, model, display, local_pi=local_pi, reasoning_effort=reasoning_effort, ) def agent_spec_from_locator(locator: Path | None) -> AgentSpec | None: if locator is None: return None try: record = json.loads(locator.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return None if not isinstance(record, dict): return None return agent_spec_from_record(record) @dataclass(frozen=True) class PiSessionState: phase: str expected_tool_call_ids: tuple[str, ...] = () completed_tool_call_ids: tuple[str, ...] = () pending_tool_call_ids: tuple[str, ...] = () reason: str = "" @dataclass(frozen=True) class LegacyPromotionRecovery: locator: Path role: str failure_class: str evidence: str evidence_source: str prior_dispatcher_sha256: str failed_cli: str failed_model: str failed_reasoning_effort: str | None def failed_spec_from_recovery( recovery: LegacyPromotionRecovery, ) -> AgentSpec: record = { "cli": recovery.failed_cli, "model": recovery.failed_model, "reasoning_effort": recovery.failed_reasoning_effort, } spec = agent_spec_from_record(record) if spec is None: raise ValueError("legacy promotion recovery에 failed agent identity가 없다") return spec @dataclass class Task: name: str directory: Path plan: Path | None review: Path | None user_review: Path | None recovery: bool errors: list[str] = field(default_factory=list) index: int = 0 deps: tuple[str, ...] = () write_set: set[str] = field(default_factory=set) write_set_known: bool = False plan_hash: str = "none" lane: str | None = None grade: int | None = None def next_execution_identity( store: StateStore, task: Task, role: str, ) -> tuple[int, str]: attempt = store.next_attempt(task, role) identity = ( f"{safe_name(task.name)}__p{plan_number(task)}__{role}__a{attempt:02d}" ) return attempt, identity class StateStore: def __init__(self, workspace: Path): self.workspace = workspace.resolve() self.workspace_id = hashlib.sha256( str(self.workspace).encode() ).hexdigest()[:16] git_marker = self.workspace / ".git" git_directory: Path | None = None if git_marker.is_dir(): git_directory = git_marker elif git_marker.is_file(): marker = git_marker.read_text(encoding="utf-8", errors="replace").strip() if marker.startswith("gitdir:"): candidate = Path(marker.split(":", 1)[1].strip()) git_directory = ( candidate if candidate.is_absolute() else (self.workspace / candidate).resolve() ) candidates = [] if git_directory is not None: candidates.append(git_directory / "agent-task-dispatcher") state_base = Path(os.environ.get("XDG_STATE_HOME", str(Path.home() / ".local" / "state"))) candidates.append(state_base / "agent-task-dispatcher" / self.workspace_id) self.root = candidates[-1] last_error: OSError | None = None for candidate in candidates: try: candidate.mkdir(parents=True, exist_ok=True) self.root = candidate last_error = None break except OSError as exc: last_error = exc if last_error is not None: raise DispatcherTerminalStateError( f"dispatcher state 디렉터리를 만들 수 없다: {candidates}" ) from last_error self.path = self.root / "state.json" self.runs = self.root / "runs" self.runs.mkdir(exist_ok=True) self.lock_stream = (self.root / "dispatcher.lock").open("a+", encoding="utf-8") try: fcntl.flock(self.lock_stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError as exc: self.lock_stream.seek(0) owner = self.lock_stream.read().strip() or "owner metadata unavailable" self.lock_stream.close() raise DispatcherAlreadyRunning( f"같은 workspace의 dispatcher가 이미 실행 중이다: " f"{self.root}; owner={owner}" ) from exc try: self.lock_stream.seek(0) self.lock_stream.truncate() self.lock_stream.write( json.dumps(dispatcher_source_provenance(), ensure_ascii=False) + "\n" ) self.lock_stream.flush() except OSError as exc: self.lock_stream.close() raise DispatcherTerminalStateError( f"dispatcher lock owner metadata를 기록할 수 없다: {self.root}" ) from exc if self.path.exists(): try: self.data = json.loads(self.path.read_text(encoding="utf-8")) except (json.JSONDecodeError, OSError) as exc: self.lock_stream.close() raise DispatcherTerminalStateError( f"dispatcher state를 읽을 수 없다: {self.path}" ) from exc if not isinstance(self.data, dict): self.lock_stream.close() raise DispatcherTerminalStateError( f"dispatcher state가 object가 아니다: {self.path}" ) else: self.data = {"tasks": {}, "attempt_counters": {}} try: self._bind_workspace_identity() self.write_claim_snapshot() except DispatcherTerminalStateError: self.lock_stream.close() raise def _bind_workspace_identity(self) -> None: expected = { "id": self.workspace_id, "root": str(self.workspace), } current = self.data.get("workspace_identity") if current is None: self.data["workspace_identity"] = expected return if not isinstance(current, dict): raise DispatcherTerminalStateError( f"dispatcher workspace identity가 object가 아니다: {self.path}" ) if ( current.get("id") != expected["id"] or current.get("root") != expected["root"] ): raise DispatcherTerminalStateError( "dispatcher state의 workspace identity가 현재 checkout과 다르다: " f"state={current} current={expected}" ) def write_claim_snapshot(self) -> dict[str, dict[str, Any]]: raw = self.data.setdefault("write_claims", {}) if not isinstance(raw, dict): raise DispatcherTerminalStateError( f"dispatcher write_claims가 object가 아니다: {self.path}" ) snapshot: dict[str, dict[str, Any]] = {} for owner, value in raw.items(): if not isinstance(owner, str) or not owner or not isinstance(value, dict): raise DispatcherTerminalStateError( f"dispatcher write claim 형식이 유효하지 않다: owner={owner!r}" ) paths = value.get("paths") exclusive = value.get("exclusive", False) if not isinstance(paths, list) or not isinstance(exclusive, bool): raise DispatcherTerminalStateError( f"dispatcher write claim 경로 형식이 유효하지 않다: owner={owner}" ) if value.get("workspace_id") != self.workspace_id: raise DispatcherTerminalStateError( "dispatcher write claim의 workspace identity가 다르다: " f"owner={owner}" ) canonical: list[str] = [] for raw_path in paths: if not isinstance(raw_path, str) or not raw_path: raise DispatcherTerminalStateError( f"dispatcher write claim 경로가 유효하지 않다: owner={owner}" ) path = Path(raw_path) resolved = path.resolve() try: resolved.relative_to(self.workspace) except ValueError as exc: raise DispatcherTerminalStateError( "dispatcher write claim이 workspace 밖을 가리킨다: " f"owner={owner} path={raw_path}" ) from exc if not path.is_absolute() or str(resolved) != raw_path or resolved == self.workspace: raise DispatcherTerminalStateError( "dispatcher write claim 경로가 canonical file이 아니다: " f"owner={owner} path={raw_path}" ) canonical.append(raw_path) if (not canonical and not exclusive) or len(canonical) != len(set(canonical)): raise DispatcherTerminalStateError( f"dispatcher write claim 경로 집합이 유효하지 않다: owner={owner}" ) record = dict(value) record["paths"] = sorted(canonical) snapshot[owner] = record return snapshot def replace_write_claims( self, claims: dict[str, dict[str, Any]], *, persist: bool, ) -> None: previous = self.data.get("write_claims", {}) self.data["write_claims"] = claims try: self.write_claim_snapshot() if persist and previous != claims: self.save() except BaseException: self.data["write_claims"] = previous raise def adopt_active_write_claim(self, task: Task) -> None: claims = self.write_claim_snapshot() if task.name in claims: return timestamp = now_iso() claims[task.name] = { "task": task.name, "plan_hash": task.plan_hash, "paths": sorted(task.write_set) if task.write_set_known else [], "exclusive": not task.write_set_known, "workspace_id": self.workspace_id, "acquired_at": timestamp, "updated_at": timestamp, "source": "active-recovery", } self.replace_write_claims(claims, persist=True) def release_write_claim(self, task_name: str, *, persist: bool = True) -> bool: claims = self.write_claim_snapshot() if task_name not in claims: return False del claims[task_name] self.replace_write_claims(claims, persist=persist) return True def save(self) -> None: write_json(self.path, self.data) def close(self) -> None: if not self.lock_stream.closed: self.lock_stream.close() def task_state(self, task: Task) -> dict[str, Any]: tasks = self.data.setdefault("tasks", {}) current = tasks.get(task.name) if not current or current.get("plan_hash") != task.plan_hash: current = { "plan_hash": task.plan_hash, "worker_done": False, "worker_cli": None, "worker_model": None, "selfcheck_done": False, "blocked": None, "active_stage": None, "active_locator": None, "review_no_progress": 0, "selfcheck_incomplete": 0, "recovery_failures": {}, "execution_decisions": {}, "route_transition_history": [], "stage_failure_budgets": {}, "retry_quota_refresh_pending": False, "retry_quota_refresh_context": None, "blocker_evidence": None, } tasks[task.name] = current self.save() return current def peek_task_state(self, task: Task) -> dict[str, Any]: current = self.data.get("tasks", {}).get(task.name) if current and current.get("plan_hash") == task.plan_hash: return dict(current) return { "plan_hash": task.plan_hash, "worker_done": False, "worker_cli": None, "worker_model": None, "selfcheck_done": False, "blocked": None, "active_stage": None, "active_locator": None, "review_no_progress": 0, "selfcheck_incomplete": 0, "recovery_failures": {}, "execution_decisions": {}, "route_transition_history": [], "retry_quota_refresh_pending": False, "retry_quota_refresh_context": None, "blocker_evidence": None, } def update_task(self, task: Task, **values: Any) -> None: state = self.task_state(task) state.update(values) self.save() def mark_active(self, task: Task, stage: str, locator: Path | None = None) -> None: self.update_task( task, active_stage=stage, active_locator=str(locator) if locator else None, active_started_at=now_iso(), ) def clear_active(self, task: Task) -> None: self.update_task( task, active_stage=None, active_locator=None, active_started_at=None, ) def consume_matching_retry_handoff(self, task: Task, locator_path: str) -> bool: """Atomically consume a pending retry handoff when a matching locator exists. When a worker writes its locator and sets active_locator, the pending retry_quota_refresh state must be cleared in the same transaction. This prevents a crash window where a restart sees the pending handoff and creates a duplicate invocation. Returns True if the pending handoff was consumed, False if no matching locator was found (active_locator is None or differs from locator_path). """ state = self.task_state(task) active = state.get("active_locator") if active != locator_path: return False pending = state.get("retry_quota_refresh_pending") if not pending: return False context = state.get("retry_quota_refresh_context") if not isinstance(context, dict): return False context_locator = context.get("locator") if context_locator != locator_path: return False # Snapshot current state to restore on save failure. This ensures the # crash window is not widened by a partial consume: if the save fails, # the pending handoff remains intact both in-memory and on-disk. pre_state = dict(state) pre_keys = set(state.keys()) pre_values = {k: state.get(k) for k in ["retry_quota_refresh_pending", "retry_quota_refresh_context"]} try: self.update_task( task, retry_quota_refresh_pending=False, retry_quota_refresh_context=None, ) except Exception: # Restore the pre-consume state on any failure. for k, v in pre_values.items(): state[k] = v # Restore key existence: if a key existed before, restore its value; # if a key did not exist before, ensure it is not present. for k in list(state.keys()): if k not in pre_keys: del state[k] for k, v in pre_values.items(): if k not in state: state[k] = v raise return True def commit_retry_handoff_locator( self, task: Task, handoff_id: str, locator_path: str, ) -> bool: """Atomically commit a new locator and consume a matching pending retry handoff. This is the durable one-save transition for retry handoff. It matches the pending handoff by stable handoff_id (not by locator path, which changes on each attempt) and atomically updates active_locator, clears the pending flag, and clears the context in a single save. On save failure the pre-state is fully restored both in-memory and on disk so the crash window is not widened. Returns True if a matching pending handoff was consumed, False if no pending handoff with the given handoff_id was found. """ state = self.task_state(task) pending = state.get("retry_quota_refresh_pending") if not pending: return False context = state.get("retry_quota_refresh_context") if not isinstance(context, dict): return False if context.get("handoff_id") != handoff_id: return False # Snapshot current state to restore on save failure. pre_state = dict(state) pre_keys = set(state.keys()) pre_values = { k: state.get(k) for k in [ "retry_quota_refresh_pending", "retry_quota_refresh_context", "active_locator", ] } try: self.update_task( task, active_locator=locator_path, retry_quota_refresh_pending=False, retry_quota_refresh_context=None, ) except Exception: for k, v in pre_values.items(): state[k] = v for k in list(state.keys()): if k not in pre_keys: del state[k] for k, v in pre_values.items(): if k not in state: state[k] = v raise return True def next_attempt(self, task: Task, role: str) -> int: key = f"{task.name}|{task.plan_hash}|{role}" counters = self.data.setdefault("attempt_counters", {}) number = int(counters.get(key, 0)) counters[key] = number + 1 self.save() return number def clear_blocked(self, task_group: str | None = None) -> None: prefix = f"{task_group}/" if task_group else None for task_name, value in self.data.get("tasks", {}).items(): if ( task_group is not None and task_name != task_group and not task_name.startswith(prefix) ): continue value["blocked"] = None value["review_no_progress"] = 0 value["selfcheck_incomplete"] = 0 value["recovery_failures"] = {} value["stage_failure_budgets"] = {} value["retry_quota_refresh_pending"] = False self.save() def mark_retry_quota_refresh(self, task_group: str | None = None, workspace: Path | None = None) -> None: prefix = f"{task_group}/" if task_group else None for task_name, value in self.data.get("tasks", {}).items(): if ( task_group is not None and task_name != task_group and not task_name.startswith(prefix) ): continue if not value.get("blocked"): continue blocker_evidence = value.get("blocker_evidence") if isinstance(value.get("blocker_evidence"), dict) else {} decisions = value.get("execution_decisions", {}) worker_decision = decisions.get("worker") if isinstance(decisions, dict) else None role = blocker_evidence.get("role") failure_class = blocker_evidence.get("failure_class") locator = blocker_evidence.get("locator") selected = blocker_evidence.get("selected") work_unit_id = blocker_evidence.get("work_unit_id") qualified = ( role == "worker" and failure_class in QUALIFIED_FAILOVER_FAILURES and isinstance(locator, str) and locator.strip() and isinstance(selected, dict) and isinstance(work_unit_id, str) and isinstance(worker_decision, dict) and worker_decision.get("work_unit_id") == work_unit_id ) handoff_id = str(uuid.uuid4()) retry_context = ({ "role": role, "failure_class": failure_class, "locator": locator, "selected": selected, "work_unit_id": work_unit_id, "handoff_id": handoff_id, } if qualified else None) value["blocked"] = None value["review_no_progress"] = 0 value["selfcheck_incomplete"] = 0 value["recovery_failures"] = {} value["stage_failure_budgets"] = {} value["retry_quota_refresh_pending"] = qualified value["retry_quota_refresh_context"] = retry_context value["blocker_evidence"] = None self.save() def prepare_orchestration( self, scope: str, tasks: list[Task], workspace: Path, ) -> None: orchestrations = self.data.setdefault("orchestrations", {}) current = orchestrations.get(scope) if current is None or (current.get("status") == "complete" and tasks): current = {"status": "running", "tasks": {}} orchestrations[scope] = current changed = False tracked = current.setdefault("tasks", {}) for task in tasks: record = tracked.get(task.name) if record is None: tracked[task.name] = { "status": "active", "archive": None, "archive_baseline": [ str(path.resolve()) for path in matching_archive_directories_by_name( workspace, task.name, require_complete=False, ) ], } changed = True continue if record.get("status") != "complete" and ( record.get("status") != "active" or "reason" in record ): record["status"] = "active" record.pop("reason", None) changed = True if changed or current.get("status") != "running": current["status"] = "running" self.save() def mark_orchestration_task_complete( self, scope: str, task_name: str, archive: str | Path, ) -> None: archive_path = Path(archive).resolve() if not archive_path.is_dir() or not (archive_path / "complete.log").is_file(): raise RuntimeError( f"완료 archive에 complete.log가 없다: task={task_name} archive={archive_path}" ) current = self.data.setdefault("orchestrations", {}).setdefault( scope, {"status": "running", "tasks": {}} ) tracked = current.setdefault("tasks", {}) record = tracked.setdefault( task_name, {"status": "active", "archive": None, "archive_baseline": []}, ) record.update(status="complete", archive=str(archive_path)) record.pop("reason", None) self.release_write_claim(task_name, persist=False) self.save() cleanup_completed_task_attempt_logs(self.runs, task_name) def mark_orchestration_blocked( self, scope: str, outcomes: dict[str, tuple[str, str]], ) -> None: current = self.data.setdefault("orchestrations", {}).setdefault( scope, {"status": "running", "tasks": {}} ) current["status"] = "blocked" tracked = current.setdefault("tasks", {}) for task_name, (status, reason) in outcomes.items(): record = tracked.setdefault( task_name, { "status": "active", "archive": None, "archive_baseline": [], }, ) if record.get("status") == "complete": continue record.update(status=status, reason=reason) self.save() def reconcile_orchestration( self, scope: str, workspace: Path, active_or_running: set[str], ) -> tuple[dict[str, str], dict[str, str]]: current = self.data.setdefault("orchestrations", {}).setdefault( scope, {"status": "running", "tasks": {}} ) completed: dict[str, str] = {} errors: dict[str, str] = {} changed = False for task_name, record in current.setdefault("tasks", {}).items(): if record.get("status") == "complete": archive = str(record.get("archive") or "") if archive and (Path(archive) / "complete.log").is_file(): completed[task_name] = archive if task_name not in active_or_running: changed = ( self.release_write_claim(task_name, persist=False) or changed ) else: errors[task_name] = "persisted complete archive가 유효하지 않다" continue if task_name in active_or_running: continue baseline = set(str(path) for path in record.get("archive_baseline", [])) candidates = [ path for path in matching_archive_directories_by_name(workspace, task_name) if str(path.resolve()) not in baseline ] if len(candidates) == 1: archive = str(candidates[0].resolve()) record.update(status="complete", archive=archive) completed[task_name] = archive changed = self.release_write_claim(task_name, persist=False) or changed changed = True elif not candidates: errors[task_name] = ( "관찰된 task가 active와 새 complete.log archive 모두에서 사라졌다" ) else: errors[task_name] = ( "새 complete.log archive가 여러 개라 완료 경로를 확정할 수 없다: " + ",".join(str(path) for path in candidates) ) if changed: self.save() for task_name in completed: if task_name not in active_or_running: cleanup_completed_task_attempt_logs(self.runs, task_name) return completed, errors def orchestration_tasks(self, scope: str) -> set[str]: current = self.data.get("orchestrations", {}).get(scope, {}) return set(current.get("tasks", {})) def mark_orchestration_complete(self, scope: str) -> None: current = self.data.setdefault("orchestrations", {}).setdefault( scope, {"status": "running", "tasks": {}} ) current["status"] = "complete" self.save() def orchestration_live_agent_processes( store: StateStore, scope: str, ) -> dict[str, str]: """Return observed tasks with live or conservatively active evidence.""" task_states = store.data.get("tasks", {}) live: dict[str, str] = {} for task_name in store.orchestration_tasks(scope): state = task_states.get(task_name) if not isinstance(state, dict): continue is_live, detail = external_active_is_live( state, expected_workspace=store.workspace, expected_workspace_id=store.workspace_id, expected_runs_root=store.runs, ) if is_live: live[task_name] = detail return live def workspace_live_agent_processes( store: StateStore, ) -> dict[str, str]: """Return observed tasks across the entire physical workspace with live or conservatively active evidence.""" task_states = store.data.get("tasks", {}) live: dict[str, str] = {} if isinstance(task_states, dict): for task_name, state in task_states.items(): if not isinstance(state, dict): continue is_live, detail = external_active_is_live( state, expected_workspace=store.workspace, expected_workspace_id=store.workspace_id, expected_runs_root=store.runs, ) if is_live: live[task_name] = detail return live def parse_route(plan: Path | None) -> tuple[str | None, int | None]: if plan is None: return None, None match = PLAN_RE.match(plan.name) if not match: return None, None return match.group(1), int(match.group(2)) def parse_task_name(task_root: Path, directory: Path) -> str: return directory.relative_to(task_root).as_posix() def inspect_write_set( plan: Path | None, workspace: Path, ) -> tuple[set[str], list[str]]: if plan is None: return set(), ["PLAN 경로가 없다"] if not plan.is_file(): return set(), [f"PLAN 파일이 없다: {plan}"] workspace = workspace.resolve() try: text = plan.read_text(encoding="utf-8", errors="replace") except OSError as exc: return set(), [f"PLAN 파일을 읽을 수 없다: {plan}: {exc}"] matches = [] for heading in MODIFIED_FILES_HEADINGS: pattern = rf"^##\s*{re.escape(heading)}[ \t]*$([\s\S]*?)(?=^##\s|\Z)" for m in re.finditer(pattern, text, re.MULTILINE): matches.append(m) if not matches: return set(), ["Modified Files Summary 섹션이 없다"] if len(matches) != 1: return set(), [ f"Modified Files Summary 섹션은 정확히 1개여야 한다: count={len(matches)}" ] match = matches[0] result: set[str] = set() diagnostics: list[str] = [] for line in match.group(1).splitlines(): if not line.lstrip().startswith("|"): continue cells = [cell.strip() for cell in line.strip().strip("|").split("|")] if not cells: continue if all(set(cell) <= {":", "-"} for cell in cells): continue if cells[0].casefold() in MODIFIED_FILES_HEADER_CELLS: continue claims = re.findall(r"`([^`]+)`", cells[0]) if not claims: diagnostics.append( f"정확한 backtick workspace 파일 경로가 없는 claim 행: {cells[0]}" ) continue for value in claims: normalized = re.sub(r":\d+(?::\d+)?$", "", value.strip()) if not normalized: diagnostics.append("빈 경로 claim은 허용되지 않는다") continue if PLACEHOLDER_PATH_RE.search(normalized): diagnostics.append( f"placeholder 또는 malformed path claim은 허용되지 않는다: {normalized}" ) continue if normalized.startswith(("http://", "https://")): diagnostics.append(f"URL claim은 허용되지 않는다: {normalized}") continue if "\\" in normalized: diagnostics.append( f"malformed path claim은 허용되지 않는다: {normalized}" ) continue if any(character in normalized for character in "*?[]"): diagnostics.append( f"glob 또는 broad path claim은 허용되지 않는다: {normalized}" ) continue if normalized.endswith(("/", "\\")): diagnostics.append( f"디렉터리 claim은 허용되지 않는다: {normalized}" ) continue candidate = Path(normalized) try: resolved = ( candidate.resolve() if candidate.is_absolute() else (workspace / candidate).resolve() ) except (OSError, RuntimeError) as exc: diagnostics.append( f"경로를 canonicalize할 수 없다: {normalized}: {exc}" ) continue try: resolved.relative_to(workspace) except ValueError: diagnostics.append( f"workspace 밖 claim은 허용되지 않는다: {normalized}" ) continue if resolved == workspace: diagnostics.append("workspace root claim은 허용되지 않는다") continue if resolved.is_dir(): diagnostics.append( f"디렉터리 claim은 허용되지 않는다: {normalized}" ) continue result.add(str(resolved)) if not result: diagnostics.append("정확한 workspace 파일 claim이 하나 이상 필요하다") return result, diagnostics def extract_write_set(plan: Path | None, workspace: Path) -> tuple[set[str], bool]: write_set, diagnostics = inspect_write_set(plan, workspace) if diagnostics: return set(), False return write_set, True def latest_verdict_log(directory: Path) -> Path | None: candidates: list[tuple[int, Path]] = [] for path in directory.glob("code_review_*.log"): match = REVIEW_LOG_RE.fullmatch(path.name) if match is None or not path.is_file() or read_verdict(path) is None: continue candidates.append((int(match.group(3)), path)) if not candidates: return None return max( candidates, key=lambda candidate: (candidate[0], candidate[1].name), )[1] def matching_plan_log(directory: Path, review_log: Path | None) -> Path | None: if review_log is None: return None review_identity = plan_identity(review_log) matches = [ path for path in directory.glob("plan_*.log") if PLAN_LOG_RE.fullmatch(path.name) is not None and path.is_file() and plan_identity(path) == review_identity ] return max(matches, key=lambda path: path.stat().st_mtime_ns) if matches else None def read_task_directory(workspace: Path, directory: Path) -> Task | None: """Read one already-known task directory without scanning the task group.""" task_root = workspace / "agent-task" if not directory.is_dir(): return None plans = sorted(p for p in directory.iterdir() if p.is_file() and PLAN_RE.match(p.name)) reviews = sorted(p for p in directory.iterdir() if p.is_file() and REVIEW_RE.match(p.name)) users = sorted(directory.glob("USER_REVIEW.md")) complete = directory / "complete.log" recovery_log = latest_verdict_log(directory) if not plans and not reviews and not users and not complete.exists() and recovery_log is None: return None name = parse_task_name(task_root, directory) errors: list[str] = [] if len(plans) > 1: errors.append(f"active PLAN이 {len(plans)}개다") if len(reviews) > 1: errors.append(f"active CODE_REVIEW가 {len(reviews)}개다") if len(users) > 1: errors.append(f"USER_REVIEW가 {len(users)}개다") if users and (plans or reviews): errors.append("USER_REVIEW stop state와 active PLAN/CODE_REVIEW가 공존한다") plan = plans[0] if len(plans) == 1 else None review = reviews[0] if len(reviews) == 1 else None recovery = complete.exists() or recovery_log is not None if bool(plan) != bool(review) and not recovery: errors.append("active PLAN/CODE_REVIEW pair가 불완전하다") relative = directory.relative_to(task_root) subtask = relative.parts[1] if len(relative.parts) == 2 else None index = 0 deps: tuple[str, ...] = () if subtask: match = SUBTASK_RE.match(subtask) if match: index = int(match.group("index")) deps = tuple((match.group("deps") or "").split(",")) if match.group("deps") else () else: errors.append(f"split subtask 이름이 계약과 다르다: {subtask}") lane, grade = parse_route(plan) recovery_plan = matching_plan_log(directory, recovery_log) write_set_source = plan or recovery_plan write_set: set[str] = set() write_set_known = False if recovery_log is not None and recovery_plan is None: errors.append( "PLAN Modified Files Summary를 복구할 matching PLAN log가 없다" ) elif write_set_source is not None: write_set, write_set_diagnostics = inspect_write_set( write_set_source, workspace, ) write_set_known = bool(write_set) and not write_set_diagnostics errors.extend( f"PLAN Modified Files Summary가 유효하지 않다: {diagnostic}" for diagnostic in write_set_diagnostics ) if plan is not None: metadata = PLAN_IDENTITY_RE.search( plan.read_text(encoding="utf-8", errors="replace")[:1024] ) if metadata is None: errors.append("PLAN task/plan/tag metadata를 판별할 수 없다") elif metadata.group("task") != name: errors.append( f"PLAN task metadata가 디렉터리와 다르다: {metadata.group('task')}" ) return Task( name=name, directory=directory, plan=plan, review=review, user_review=users[0] if len(users) == 1 else None, recovery=recovery, errors=errors, index=index, deps=deps, write_set=write_set, write_set_known=write_set_known, plan_hash=( plan_identity(plan) if plan else sha256_file( recovery_log or (users[0] if len(users) == 1 else complete) ) ), lane=lane, grade=grade, ) @dataclass class StageFailureBudget: """Persistent failure counter shared by one work unit and stage.""" store: StateStore task: Task work_unit_id: str stage: str @classmethod def from_decision(cls, store: StateStore, task: Task, decision: dict[str, Any]) -> "StageFailureBudget": work_unit_id = decision.get("work_unit_id") stage = decision.get("stage") if not isinstance(work_unit_id, str) or not work_unit_id or not isinstance(stage, str) or not stage: raise ExecutionDecisionError("stage failure budget identity가 유효하지 않다") return cls(store, task, work_unit_id, stage) @property def key(self) -> str: return f"{self.work_unit_id}|{self.stage}" def _budgets(self) -> dict[str, Any]: state = self.store.task_state(self.task) budgets = state.get("stage_failure_budgets", {}) if not isinstance(budgets, dict): raise ExecutionDecisionError("persisted stage failure budgets schema가 유효하지 않다") return dict(budgets) def count(self) -> int: entry = self._budgets().get(self.key, {}) if not isinstance(entry, dict): raise ExecutionDecisionError("persisted stage failure budget entry가 유효하지 않다") return int(entry.get("count", 0)) def record_failure(self, *, target: dict[str, Any], transition: str) -> int: budgets = self._budgets() entry = dict(budgets.get(self.key, {})) count = int(entry.get("count", 0)) + 1 entry.update( work_unit_id=self.work_unit_id, stage=self.stage, count=count, last_target={"adapter": target.get("adapter"), "target": target.get("target")}, last_transition=transition, ) budgets[self.key] = entry self.store.update_task(self.task, stage_failure_budgets=budgets) return count def reset_on_success(self) -> None: budgets = self._budgets() budgets.pop(self.key, None) self.store.update_task(self.task, stage_failure_budgets=budgets) def scan_tasks( workspace: Path, task_group: str | None, *, exclude_names: set[str] | None = None, ) -> list[Task]: task_root = workspace / "agent-task" if not task_root.is_dir(): raise DispatcherTerminalStateError( f"agent-task 디렉터리가 없다: {task_root}" ) directories: list[Path] = [] try: groups = [task_root / task_group] if task_group else sorted( p for p in task_root.iterdir() if p.is_dir() and p.name != "archive" ) except FileNotFoundError: return [] for group in groups: if not group.is_dir(): continue directories.append(group) try: directories.extend(sorted(p for p in group.iterdir() if p.is_dir())) except FileNotFoundError: continue tasks = [ task for directory in directories if ( exclude_names is None or parse_task_name(task_root, directory) not in exclude_names ) if (task := read_task_directory(workspace, directory)) is not None ] return sorted(tasks, key=lambda task: (task.index, task.name)) def dependency_candidates(workspace: Path, task: Task, predecessor: str) -> list[Path]: parts = task.name.split("/") if len(parts) != 2: return [] group = parts[0] task_root = workspace / "agent-task" found: list[Path] = [] active_group = task_root / group for pattern in (f"{predecessor}_*/complete.log", f"{predecessor}+*/complete.log"): found.extend(active_group.glob(pattern)) archive = task_root / "archive" if archive.is_dir(): try: years = list(archive.iterdir()) except FileNotFoundError: years = [] for year in years: if not year.is_dir(): continue try: months = list(year.iterdir()) except FileNotFoundError: continue for month in months: archived_group = month / group if not archived_group.is_dir(): continue for pattern in (f"{predecessor}_*/complete.log", f"{predecessor}+*/complete.log"): found.extend(archived_group.glob(pattern)) return sorted(set(path.resolve() for path in found)) def dependency_state(workspace: Path, task: Task) -> tuple[bool, str]: missing: list[str] = [] ambiguous: list[str] = [] for predecessor in task.deps: candidates = dependency_candidates(workspace, task, predecessor) if not candidates: missing.append(predecessor) elif len(candidates) > 1: ambiguous.append(f"{predecessor}={','.join(str(p) for p in candidates)}") if ambiguous: return False, "dependency ambiguity: " + "; ".join(ambiguous) if missing: return False, "predecessor complete.log 대기: " + ",".join(missing) return True, "ready" def live_predecessors( task: Task, active_task_names: set[str], ) -> list[str]: parts = task.name.split("/") if len(parts) != 2 or not task.deps: return [] group = parts[0] live: list[str] = [] for predecessor in task.deps: prefix = re.compile(rf"^{re.escape(predecessor)}(?:[+_])") if any( name.startswith(f"{group}/") and prefix.match(name.split("/", 1)[1]) for name in active_task_names ): live.append(predecessor) return live def _selector_module(): if "agent_task_execution_target_selector" in sys.modules: return sys.modules["agent_task_execution_target_selector"] path = Path(__file__).resolve().parent / "select_execution_target.py" spec = importlib.util.spec_from_file_location("agent_task_execution_target_selector", path) if spec is None or spec.loader is None: raise ExecutionDecisionError(f"selector load 실패: {path}") module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _decision_file(task: Task, stage: str) -> Path: path = task.plan if stage == "worker" else task.review if path is None or not path.is_file(): raise ExecutionDecisionError(f"{stage} selector 입력 파일이 없다") return path def agent_spec_from_decision(decision: dict[str, Any]) -> AgentSpec: selected = decision.get("selected") if not isinstance(selected, dict): raise ExecutionDecisionError("selector selected가 object가 아니다") adapter, target = selected.get("adapter"), selected.get("target") local_pi = selected.get("selfcheck_required") execution_class = selected.get("execution_class") if (not isinstance(adapter, str) or not isinstance(target, str) or not target or not isinstance(local_pi, bool) or execution_class not in {"local_model", "cloud_model"}): raise ExecutionDecisionError("selector selected schema가 유효하지 않다") try: selector = _selector_module() selector._validate_prior_decision(decision) decision_info = decision["decision"] evaluated_at = selector.datetime.fromisoformat(decision_info["evaluated_at"]) policy_targets = selector.policy.select_policy( stage=decision["stage"], lane=decision["lane"], grade=decision["grade"], evaluated_at=evaluated_at, ).candidates selector._validate_prior_candidate_identity( decision, stage=decision["stage"], lane=decision["lane"], grade=decision["grade"], ) canonical = selector.policy.canonical_target(adapter, target) except Exception as exc: raise ExecutionDecisionError(f"selector policy validation 실패: {exc}") from exc if canonical is None or ( canonical.execution_class != execution_class or canonical.selfcheck_required != local_pi ): raise ExecutionDecisionError("selector selected가 canonical policy target이 아니다") initial_keys = {(item.adapter, item.target) for item in policy_targets} if (adapter, target) not in initial_keys: promotion_path = decision.get("promotion_path") if not isinstance(promotion_path, list) or len(promotion_path) < 2: raise ExecutionDecisionError("selector promotion path가 없다") resolved_path = [] for index, entry in enumerate(promotion_path): if not isinstance(entry, dict): raise ExecutionDecisionError( f"selector promotion path[{index}]가 object가 아니다" ) resolved = selector.policy.canonical_target( entry.get("adapter"), entry.get("target") ) if resolved is None: raise ExecutionDecisionError( f"selector promotion path[{index}] target이 canonical이 아니다" ) resolved_path.append(resolved) if (resolved_path[0].adapter, resolved_path[0].target) not in initial_keys: raise ExecutionDecisionError("selector promotion path 시작 target이 잘못됐다") if any( selector.policy.promotion_target(previous) != current for previous, current in zip(resolved_path, resolved_path[1:]) ): raise ExecutionDecisionError("selector promotion path 순서가 잘못됐다") if resolved_path[-1] != canonical: raise ExecutionDecisionError("selector promotion path tail이 selected와 다르다") if adapter == "pi": if not target.startswith("iop/") or not local_pi: raise ExecutionDecisionError("Pi selector target/schema가 유효하지 않다") return AgentSpec("pi", target.removeprefix("iop/"), f"pi/{target}", local_pi=True) if adapter not in {"agy", "claude", "codex"} or local_pi: raise ExecutionDecisionError(f"selector adapter/schema가 유효하지 않다: {adapter!r}") reasoning_effort = "high" if canonical == selector.policy.CODEX_TERRA_HIGH else None suffix = " high" if reasoning_effort == "high" else ( " xhigh" if adapter in {"claude", "codex"} else "" ) display = f"{adapter}/{target}{suffix}" if reasoning_effort is not None: return AgentSpec( adapter, target, display, reasoning_effort=reasoning_effort, ) return AgentSpec(adapter, target, display) def _spec_from_completing_decision(decision: dict[str, Any]) -> AgentSpec: """Lightweight AgentSpec extraction from a persisted completing decision. Unlike `agent_spec_from_decision`, this does not re-validate against the selector policy. The completing decision is already authoritative evidence of the target that succeeded, so re-running policy is unnecessary and would defeat the purpose of pinning the selfcheck target. """ selected = decision.get("selected") if not isinstance(selected, dict): raise ExecutionDecisionError( "completing decision selected schema가 유효하지 않다" ) adapter = selected.get("adapter") target = selected.get("target") execution_class = selected.get("execution_class") selfcheck_required = selected.get("selfcheck_required") if not all(isinstance(value, str) and value for value in (adapter, target, execution_class)): raise ExecutionDecisionError( "completing decision selected의 adapter/target/execution_class는 빈 문자열이 아닌 string이어야 한다" ) if execution_class not in {"local_model", "cloud_model"}: raise ExecutionDecisionError( f"completing decision execution_class이 유효하지 않다: {execution_class}" ) if not isinstance(selfcheck_required, bool): raise ExecutionDecisionError( "completing decision selected.selfcheck_required must be a boolean" ) if adapter == "pi": if not target.startswith("iop/"): raise ExecutionDecisionError( f"Pi completing decision target이 iop/ prefix가 아니다: {target}" ) if execution_class != "local_model": raise ExecutionDecisionError( f"Pi completing decision execution_class이 local_model이 아니다: {execution_class}" ) if not selfcheck_required: raise ExecutionDecisionError( "Pi completing decision selfcheck_required가 False이다" ) model = target.removeprefix("iop/") display = f"pi/{target}" return AgentSpec(adapter, model, display, local_pi=True) if adapter not in {"agy", "claude", "codex"}: raise ExecutionDecisionError( f"completing decision adapter가 유효하지 않다: {adapter!r}" ) if execution_class != "cloud_model": raise ExecutionDecisionError( f"cloud completing decision execution_class이 cloud_model이 아니다: {adapter}/{execution_class}" ) if selfcheck_required: raise ExecutionDecisionError( f"cloud completing decision selfcheck_required가 True이다: {adapter}/{target}" ) display = f"{adapter}/{target}" return AgentSpec(adapter, target, display, local_pi=False) def select_execution_decision( task: Task, *, stage: str, prior_decision: dict[str, Any] | None = None, quota_snapshot: dict[str, Any] | None = None, evaluated_at: datetime | None = None, transition: str | None = None, failure_class: str | None = None, ) -> dict[str, Any]: try: selector = _selector_module() except Exception as exc: code = getattr(exc, "code", exc.__class__.__name__) raise ExecutionDecisionError( f"{stage} selector load 실패 [{code}]: {exc}" ) from exc try: if transition is None: if prior_decision is not None and stage == "worker" and task.plan and task.plan.is_file(): current_id = work_unit_id_from_file(task.plan) prior_id = prior_decision.get("work_unit_id") if isinstance(prior_decision, dict) else None if current_id and isinstance(prior_id, str) and prior_id and prior_id != current_id: prior_decision = None transition = "resume" if prior_decision is not None else "initial" return selector.select_execution_target( _decision_file(task, stage), stage=stage, evaluated_at=evaluated_at or datetime.now(KST), transition=transition, prior_decision=prior_decision, quota_snapshot=quota_snapshot, failure_class=failure_class, ) except (OSError, ValueError, selector.SelectorInputError) as exc: code = getattr(exc, "code", exc.__class__.__name__) raise ExecutionDecisionError( f"{stage} selector decision 실패 [{code}]: {exc}" ) from exc def work_unit_id_from_file(path: Path) -> str | None: if not path.is_file(): return None try: head = path.read_text(encoding="utf-8", errors="replace")[:1024] match = PLAN_IDENTITY_RE.search(head) if match: return f"{match.group('task')}::plan-{match.group('plan')}::tag-{match.group('tag')}" except Exception: pass return None def official_review_plan_source(task: Task) -> Path: """Resolve the authoritative PLAN generation for an official review.""" if task.plan is not None or task.review is not None: if task.plan is None or task.review is None: raise ExecutionDecisionError( "official review active PLAN/CODE_REVIEW pair가 불완전하다" ) return task.plan recovery_log = latest_verdict_log(task.directory) recovery_plan = matching_plan_log(task.directory, recovery_log) if recovery_log is None or recovery_plan is None: raise ExecutionDecisionError( "official review recovery의 matching archived PLAN identity를 복구할 수 없다" ) return recovery_plan def official_review_source_identity(task: Task) -> tuple[str, int, str]: source = official_review_plan_source(task) route_match = PLAN_RE.match(source.name) or PLAN_LOG_RE.match(source.name) if route_match is None: raise ExecutionDecisionError( f"official review PLAN route를 복구할 수 없다: {source.name}" ) try: head = source.read_text(encoding="utf-8", errors="replace")[:1024] except OSError as exc: raise ExecutionDecisionError( f"official review PLAN source를 읽을 수 없다: {source}" ) from exc metadata = PLAN_IDENTITY_RE.search(head) if metadata is None or metadata.group("task") != task.name: raise ExecutionDecisionError( f"official review PLAN work-unit identity를 복구할 수 없다: {source}" ) work_unit_id = ( f"{metadata.group('task')}::plan-{metadata.group('plan')}::" f"tag-{metadata.group('tag')}" ) return route_match.group(1), int(route_match.group(2)), work_unit_id def synthesized_official_review_decision( task: Task, *, evaluated_at: datetime | None = None ) -> dict[str, Any]: lane, grade, work_unit_id = official_review_source_identity(task) evaluated = evaluated_at or datetime.now(KST) if evaluated.tzinfo is None or evaluated.utcoffset() is None: raise ExecutionDecisionError( "official review evaluated_at이 timezone-aware가 아니다" ) recovery_from_archive = task.plan is None and task.review is None selector = _selector_module() policy_decision = selector.policy.select_policy( stage="review", lane=lane, grade=grade, evaluated_at=evaluated ) selected_target = policy_decision.candidates[0] target_ref = { "adapter": selected_target.adapter, "target": selected_target.target, } candidate = { "candidate_rank": 1, "adapter": selected_target.adapter, "target": selected_target.target, "execution_class": selected_target.execution_class, "selfcheck_required": selected_target.selfcheck_required, "quota_mode": "bounded", "quota_status": "unknown", "eligibility": "eligible", "rejection_reason": None, } return { "schema_version": selector.SCHEMA_VERSION, "work_unit_id": work_unit_id, "stage": "review", "lane": lane, "grade": grade, "selected": { "adapter": selected_target.adapter, "target": selected_target.target, "execution_class": selected_target.execution_class, "selfcheck_required": selected_target.selfcheck_required, }, "candidates": [candidate], "decision": { "rule_id": policy_decision.rule_id, "policy_priority": policy_decision.policy_priority, "reason_codes": list(policy_decision.reason_codes), "evaluated_at": evaluated.astimezone(KST).isoformat(), "timezone": selector.TIMEZONE_NAME, "time_window": policy_decision.time_window, "pinned": recovery_from_archive, }, "quota": { "snapshot_id": None, "mode": "bounded", "status": "unknown", "source": "official_review_fixed_policy", "checked_at": None, "targets": [], }, "transition": { "previous_target": dict(target_ref) if recovery_from_archive else None, "next_target": dict(target_ref) if recovery_from_archive else None, "trigger": "resume" if recovery_from_archive else "initial", "context_transfer": "none", }, } def read_or_preview_stage_decision( task: Task, state: dict[str, Any], *, stage: str, dry_run: bool = False, evaluated_at: datetime | None = None, quota_snapshot: dict[str, Any] | None = None, ) -> dict[str, Any]: decisions = state.get("execution_decisions", {}) if isinstance(state, dict) else {} prior = decisions.get(stage) if isinstance(decisions, dict) else None if stage == "review": lane, grade, work_unit_id = official_review_source_identity(task) if ( isinstance(prior, dict) and isinstance(prior.get("decision"), dict) and isinstance(prior.get("quota"), dict) ): if ( prior.get("work_unit_id") != work_unit_id or prior.get("stage") != "review" or prior.get("lane") != lane or prior.get("grade") != grade ): raise ExecutionDecisionError( "persisted official review decision이 recovery source identity/route와 다르다" ) agent_spec_from_decision(prior) return prior return synthesized_official_review_decision( task, evaluated_at=evaluated_at ) if isinstance(prior, dict) and task.plan and task.plan.is_file(): work_unit_id = work_unit_id_from_file(task.plan) if work_unit_id and prior.get("work_unit_id") == work_unit_id: return prior if quota_snapshot is None: quota_snapshot = state.get("quota_snapshot") if isinstance(state, dict) else None return select_execution_decision( task, stage=stage, prior_decision=prior, quota_snapshot=quota_snapshot, evaluated_at=evaluated_at, ) def selector_evidence_lines(decision: dict[str, Any] | None) -> list[str]: if not isinstance(decision, dict): return [] selected = decision.get("selected", {}) if not isinstance(selected, dict): return [] work_unit = decision.get("work_unit_id", "none") decision_info = decision.get("decision", {}) if not isinstance(decision_info, dict): decision_info = {} rule_id = decision_info.get("rule_id", decision.get("rule_id", "none")) priority = decision_info.get( "policy_priority", decision.get("priority", "none") ) transition = decision.get("transition", {}) trigger = transition.get("trigger", "none") if isinstance(transition, dict) else "none" quota = decision.get("quota", decision.get("quota_snapshot", {})) quota_status = quota.get("status", "none") if isinstance(quota, dict) else "none" candidates = decision.get("candidates", []) cand_strs = [] if isinstance(candidates, list): for c in candidates: if isinstance(c, dict): rank = c.get("candidate_rank", "?") adapter = c.get("adapter", "?") target = c.get("target", "?") elig = c.get("eligibility", "?") cand_strs.append(f"#{rank}:{adapter}/{target}({elig})") reasons = decision_info.get( "reason_codes", selected.get("reason_codes", []) ) reason_str = ",".join(reasons) if isinstance(reasons, list) else str(reasons) lines = [ f"work_unit_id={work_unit}", f"rule_id={rule_id}", f"priority={priority}", f"transition={trigger}", f"quota_status={quota_status}", ] if cand_strs: lines.append(f"candidates={';'.join(cand_strs)}") if reason_str: lines.append(f"reason_codes={reason_str}") return lines def selector_runtime_evidence(decision: dict[str, Any]) -> dict[str, Any]: """Return canonical selector fields persisted in runtime audit records.""" return { "work_unit_id": decision.get("work_unit_id"), "candidates": decision.get("candidates"), "selected": decision.get("selected"), "decision": decision.get("decision"), "quota": decision.get("quota"), "transition": decision.get("transition"), } def commit_execution_decision( store: StateStore, task: Task, stage: str, decision: dict[str, Any], quota_snapshot: dict[str, Any] | None = None, ) -> None: state = store.task_state(task) decisions, history = state.get("execution_decisions", {}), state.get("route_transition_history", []) if not isinstance(decisions, dict) or not isinstance(history, list): raise ExecutionDecisionError("persisted selector state schema가 유효하지 않다") decisions = dict(decisions) decisions[stage] = decision stage_budget_count = 0 try: stage_budget_count = StageFailureBudget.from_decision(store, task, decision).count() except Exception: pass selected = decision.get("selected", {}) history_entry = { "stage": stage, "transition": decision.get("transition", {}).get("trigger") if isinstance(decision.get("transition"), dict) else None, "work_unit_id": decision.get("work_unit_id"), "candidates": decision.get("candidates"), "selected": selected, "decision": decision.get("decision"), "reason_codes": decision.get("decision", {}).get("reason_codes", []) if isinstance(decision.get("decision"), dict) else [], "quota": decision.get("quota"), "stage_budget": stage_budget_count, } history = [*history, history_entry] # Preserve retry handoff state whenever a pending retry is in flight. # The retry handoff (stable handoff_id, pending flag, context) must survive # the decision commit so that the subsequent production invoke() can read # it, embed the handoff_id in the new locator record, and atomically # consume the pending handoff via commit_retry_handoff_locator(). # Clearing it here would force invoke() to fall back to a generic # active-locator update and lose the crash-safe handoff identity. # invoke() handles consumption regardless of failover or resume transition. is_retry_in_flight = bool(state.get("retry_quota_refresh_pending")) update_kwargs = { "execution_decisions": decisions, "route_transition_history": history, "blocked": None, "blocker_evidence": None, } if not is_retry_in_flight: update_kwargs["retry_quota_refresh_pending"] = False update_kwargs["retry_quota_refresh_context"] = None if quota_snapshot is not None: update_kwargs["quota_snapshot"] = quota_snapshot store.update_task(task, **update_kwargs) def persisted_execution_decision( store: StateStore, task: Task, *, stage: str, transition: str | None = None, failure_class: str | None = None, evaluated_at: datetime | None = None, quota_snapshot: dict[str, Any] | None = None, ) -> tuple[dict[str, Any], AgentSpec]: state = store.task_state(task) decisions = state.get("execution_decisions", {}) if not isinstance(decisions, dict): raise ExecutionDecisionError("persisted selector state schema가 유효하지 않다") is_retry = retry_quota_refresh_pending(state) and stage == "worker" if quota_snapshot is None and not is_retry: quota_snapshot = state.get("quota_snapshot") if quota_snapshot is not None and not isinstance(quota_snapshot, dict): raise ExecutionDecisionError("persisted quota snapshot schema가 유효하지 않다") prior_decision = decisions.get(stage) retry_ctx = state.get("retry_quota_refresh_context") if isinstance(state.get("retry_quota_refresh_context"), dict) else {} if stage == "review": decision = read_or_preview_stage_decision( task, state, stage=stage, evaluated_at=evaluated_at, quota_snapshot=quota_snapshot ) else: if transition is None: if is_retry: transition = "failover" failure_class = failure_class or retry_ctx.get("failure_class") or "provider-quota" elif prior_decision is not None and stage == "worker" and task.plan and task.plan.is_file(): current_id = work_unit_id_from_file(task.plan) prior_id = prior_decision.get("work_unit_id") if isinstance(prior_decision, dict) else None if current_id and isinstance(prior_id, str) and prior_id and prior_id != current_id: prior_decision = None transition = "resume" if prior_decision is not None else "initial" else: transition = "resume" if prior_decision is not None else "initial" try: decision = select_execution_decision( task, stage=stage, prior_decision=prior_decision, quota_snapshot=quota_snapshot, transition=transition, failure_class=failure_class, evaluated_at=evaluated_at, ) except ExecutionDecisionError as exc: # No persisted unused quota target is an explicit resume case. A # failed qualified failover with a fresh snapshot must not consume # the retry intent before its decision can commit successfully. if is_retry and transition == "failover" and quota_snapshot is None: decision = select_execution_decision( task, stage=stage, prior_decision=prior_decision, quota_snapshot=quota_snapshot, transition="resume", evaluated_at=evaluated_at, ) else: raise spec = agent_spec_from_decision(decision) commit_execution_decision(store, task, stage, decision, quota_snapshot=quota_snapshot) return decision, spec def has_persisted_worker_decision(state: dict[str, Any], task: Task | None = None) -> bool: decisions = state.get("execution_decisions", {}) if not isinstance(decisions, dict): return False prior = decisions.get("worker") if prior is None: return False if task is not None and task.plan and task.plan.is_file(): current_id = work_unit_id_from_file(task.plan) if current_id and prior.get("work_unit_id") != current_id: return False return True def retry_quota_refresh_pending(state: dict[str, Any]) -> bool: return bool(state.get("retry_quota_refresh_pending")) def derive_work_unit_quota_evidence( decision: dict[str, Any] | None, status: str = "exhausted", reason: str = "confirmed_runtime_provider_quota", ) -> dict[str, Any]: selector = _selector_module() func = getattr(selector, "derive_work_unit_quota_evidence", None) if func is not None: return func(decision, status=status, reason=reason) return { "schema_version": "1.0", "snapshot_id": None, "source": "derived_work_unit_quota", "checked_at": datetime.now(KST).isoformat(), "targets": [], "required_caps": [], "reason_codes": [reason], } def build_admission_batch_snapshot( store: StateStore, ready_items: list[tuple[Task, str]], admission_time: datetime, quota_probe_command: str = "iop-node quota-probe", ) -> dict[str, Any] | None: selector = _selector_module() policy_mod = selector.policy unique_keys = [] seen_keys = set() for task, stage in ready_items: if stage != "worker": continue state = store.peek_task_state(task) if has_persisted_worker_decision(state, task) and not retry_quota_refresh_pending(state): continue is_retry = retry_quota_refresh_pending(state) if is_retry: decisions = state.get("execution_decisions", {}) prior = decisions.get("worker") if isinstance(decisions, dict) else None candidates = prior.get("candidates", []) if isinstance(prior, dict) else [] used = prior.get("used_candidates", []) if isinstance(prior, dict) else [] selected = prior.get("selected") if isinstance(prior, dict) else None used_keys = { (entry.get("adapter"), entry.get("target")) for entry in used if isinstance(entry, dict) } if isinstance(selected, dict): used_keys.add((selected.get("adapter"), selected.get("target"))) candidates_to_probe = [ type("Target", (), candidate)() for candidate in candidates if isinstance(candidate, dict) and candidate.get("execution_class") != "local_model" and (candidate.get("adapter"), candidate.get("target")) not in used_keys ] else: lane, grade = task.lane, task.grade if not lane or not grade: continue try: pol_dec = policy_mod.select_policy( stage="worker", lane=lane, grade=grade, evaluated_at=admission_time ) except ValueError: continue candidates_to_probe = pol_dec.candidates for cand in candidates_to_probe: if cand.execution_class == "local_model" and not is_retry: break spec = policy_mod.quota_probe_spec(cand) if spec is not None: key = (cand.adapter, cand.target, spec.command, tuple(spec.required_caps)) if key not in seen_keys: seen_keys.add(key) unique_keys.append(key) if not unique_keys: return None batch_id = f"batch-quota-{uuid.uuid4().hex[:12]}" batch_provider_cls = getattr(selector, "QuotaBatchProvider", None) if batch_provider_cls is None: return None batch_provider = batch_provider_cls(quota_probe_command=quota_probe_command) return batch_provider.aggregate( snapshot_id=batch_id, checked_at=admission_time, keys=unique_keys, ) def plan_number(task: Task) -> int: if task.plan and task.plan.exists(): match = PLAN_IDENTITY_RE.search( task.plan.read_text(encoding="utf-8", errors="replace")[:1024] ) if match: return int(match.group("plan")) return 0 def completing_decision_requires_selfcheck(state: dict[str, Any]) -> bool: completing = state.get("completing_decision") if not isinstance(completing, dict): return False selected = completing.get("selected") if not isinstance(selected, dict): return False return selected.get("execution_class") == "local_model" def _validated_completing_decision( task: Task, decision: dict[str, Any] ) -> tuple[dict[str, Any], AgentSpec]: """Strictly validate a completing decision against the task contract. Enforces that the decision's stage is "worker", its work_unit_id matches the task's PLAN identity, and its selected fields pass the canonical adapter/class/selfcheck normalization through `_spec_from_completing_decision`. Returns the validated decision and its normalized AgentSpec. Raises ExecutionDecisionError on any contract violation so that callers can fail closed rather than advancing to an inconsistent stage. """ if not isinstance(decision, dict): raise ExecutionDecisionError( "completing decision이 dict가 아니다" ) if decision.get("stage") != "worker": raise ExecutionDecisionError( f"completing decision stage가 worker가 아니다: {decision.get('stage')!r}" ) expected_work_unit_id = work_unit_id_from_file(task.plan) if decision.get("work_unit_id") != expected_work_unit_id: raise ExecutionDecisionError( f"completing decision work_unit_id 불일치: " f"persisted={decision.get('work_unit_id')!r} " f"plan={expected_work_unit_id!r}" ) spec = _spec_from_completing_decision(decision) return decision, spec def _completing_decision_is_valid( task: Task, state: dict[str, Any] ) -> bool: """Check whether the persisted completing decision satisfies the task contract. Validates stage, work_unit_id, and selected adapter/class/selfcheck combination. Used by task_stage to prevent a worker_done state with no authoritative completing decision from advancing to review. """ completing = state.get("completing_decision") if not isinstance(completing, dict): return False try: _validated_completing_decision(task, completing) except ExecutionDecisionError: return False return True def concrete_user_review_value(value: str) -> bool: normalized = value.strip().strip("`").strip() normalized = re.sub(r"^-\s*", "", normalized).strip() if not normalized or re.search(r"\{[^}]+\}|<[^>]+>", normalized): return False return normalized.casefold() not in { "-", "n/a", "na", "none", "unknown", "미정", "없음", "해당 없음", } def user_review_blocker_state(path: Path) -> tuple[bool, str]: if not path.is_file(): return False, "파일이 없다" try: text = path.read_text(encoding="utf-8", errors="replace") except OSError as exc: return False, f"파일을 읽을 수 없다: {exc}" status = markdown_section(text, "상태").strip().strip("`") if status != "USER_REVIEW": return False, "상태가 USER_REVIEW가 아니다" reason = markdown_section(text, "사유") gate_type_matches = re.findall( r"(?m)^-\s*유형:\s*(milestone-lock|external-execution)\s*$", reason, ) if len(gate_type_matches) != 1: return False, "지원하는 user-review 유형이 정확히 하나가 아니다" gate_type = gate_type_matches[0] target = re.search(r"(?m)^-\s*연결 대상:\s*(.+?)\s*$", reason) target_value = target.group(1) if target else "" if not concrete_user_review_value(target_value): return False, "구체적인 연결 대상이 없다" if gate_type == "milestone-lock" and ( "agent-roadmap/" not in target_value or "/milestones/" not in target_value or ".md" not in target_value ): return False, "구체적인 Milestone 연결 대상이 없다" evidence = markdown_section(text, "차단 근거") evidence_line = re.search(r"(?m)^-\s*차단 판단 근거:\s*(.+?)\s*$", evidence) if evidence_line is None or not concrete_user_review_value( evidence_line.group(1) ): return False, "구체적인 차단 판단 근거가 없다" decision = markdown_section(text, "사용자 조치 또는 결정") legacy_decision = markdown_section(text, "연결 결정 필요") if decision.strip() and legacy_decision.strip(): return False, "사용자 조치 또는 결정 섹션이 중복됐다" if not decision.strip(): decision = legacy_decision unresolved = [ value for value in re.findall(r"(?m)^-\s*\[\s\]\s+(.+?)\s*$", decision) if concrete_user_review_value(value) ] if not unresolved: return False, "미해결 사용자 조치 또는 결정 항목이 없다" resume = markdown_section(text, "재개 조건") resume_conditions = [ line for line in resume.splitlines() if concrete_user_review_value(line) ] if not resume_conditions: return False, "구체적인 재개 조건이 없다" return True, f"unresolved {gate_type} user action or decision" def task_stage(task: Task, state: dict[str, Any]) -> str: if task.errors: return "blocked" if task.user_review: if task.plan is not None or task.review is not None: return "blocked" blocking, _ = user_review_blocker_state(task.user_review) return "user-review" if blocking else "blocked" if task.recovery and (task.plan is None or task.review is None): return "review" if task.review and task.review.exists(): text = task.review.read_text(encoding="utf-8", errors="replace") if verdict_from_text(text): return "review" if state.get("worker_done"): if not _completing_decision_is_valid(task, state): return "blocked" if completing_decision_requires_selfcheck(state) and not state.get("selfcheck_done"): return "selfcheck" return "review" return "worker" def markdown_section(text: str, heading: str | tuple[str, ...]) -> str: headings = (heading,) if isinstance(heading, str) else heading matches = [] for h in headings: for m in re.finditer(rf"^##\s*{re.escape(h)}[ \t]*$", text, re.MULTILINE): matches.append(m) if len(matches) != 1: return "" match = matches[0] next_heading = re.search(r"^##\s+", text[match.end():], re.MULTILINE) end = match.end() + next_heading.start() if next_heading else len(text) return text[match.end():end].strip() def implementation_review_errors(task: Task) -> list[str]: if task.review is None or not task.review.is_file(): return ["CODE_REVIEW 파일 없음"] text = task.review.read_text(encoding="utf-8", errors="replace") checklist = markdown_section(text, IMPLEMENTATION_CHECKLIST_HEADINGS) checkbox_values = IMPLEMENTATION_CHECKBOX_RE.findall(checklist) if not checkbox_values or any(not value.strip() for value in checkbox_values): return ["구현 체크리스트 미완료"] return [] def classify_failure_with_evidence(output: str) -> tuple[str, str | None]: lines = output.splitlines() for category, patterns in PROMOTABLE_PATTERNS.items(): for line in reversed(lines): lowered = line.lower() if any(re.search(pattern, lowered, re.DOTALL) for pattern in patterns): return category, line return "generic-error", None def classify_failure(output: str) -> str: return classify_failure_with_evidence(output)[0] def termination_signal(return_code: int) -> tuple[str, bool] | None: signal_number: int | None = None inferred = False if return_code < 0: signal_number = -return_code elif return_code > 128: signal_number = return_code - 128 inferred = True if signal_number is None: return None try: return signal.Signals(signal_number).name, inferred except ValueError: return None def failure_report_lines(failure: str, locator: Path) -> list[str]: record: dict[str, Any] = {} try: record = json.loads(locator.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): pass failure_class = str(record.get("failure_class") or failure) source = str(record.get("failure_source") or "unverified") provider_confirmed = bool( record.get("provider_transport_failure_confirmed", False) ) lines = [ f"failure_class={failure_class}", f"failure_source={source}", "provider_transport_failure_confirmed=" f"{str(provider_confirmed).lower()}", ] if record.get("dispatcher_pid") is not None: lines.append(f"dispatcher_pid={record['dispatcher_pid']}") if record.get("agent_pid") is not None: lines.append(f"agent_pid={record['agent_pid']}") if record.get("dispatcher_source_sha256"): lines.append( f"dispatcher_source_sha256={record['dispatcher_source_sha256']}" ) source_matches_loaded = record.get("dispatcher_source_matches_loaded") if source_matches_loaded is not None: lines.append( "dispatcher_source_matches_loaded=" f"{str(bool(source_matches_loaded)).lower()}" ) if ( source_matches_loaded is False and record.get("dispatcher_source_current_sha256") ): lines.append( "dispatcher_source_current_sha256=" f"{record['dispatcher_source_current_sha256']}" ) if provider_confirmed: evidence_source = record.get("failure_evidence_source") evidence = record.get("failure_evidence_excerpt") if evidence_source: lines.append(f"provider_evidence_source={evidence_source}") if evidence: rendered = str(evidence).replace("\r", r"\r").replace("\n", r"\n") lines.append(f"provider_evidence={rendered}") if failure_class == "session-stall": lines.extend( [ f"timeout_phase={record.get('pi_session_phase') or 'unknown'}", f"timeout_seconds={record.get('session_stall_seconds') or 'unknown'}", "termination_initiator=" f"{record.get('termination_initiator') or 'dispatcher'}", ] ) elif failure_class == "process-terminated": lines.extend( [ f"termination_signal={record.get('termination_signal') or 'unknown'}", "termination_initiator=" f"{record.get('termination_initiator') or 'unknown'}", ] ) lines.append(f"locator={locator}") return lines def terminal_diagnostic(cli: str, channel: str, line: str) -> str | None: if channel == "stderr": return line try: value = json.loads(line) except json.JSONDecodeError: if cli == "agy" and re.match( r"^\s*(?:error|fatal|provider error|model error)\b", line, re.IGNORECASE ): return line return None event_type = str(value.get("type", "")) if cli == "codex" and event_type in {"turn.failed", "error"}: return json.dumps(value.get("error", value), ensure_ascii=False) if cli == "agy": severity = str(value.get("severity") or value.get("level") or "") status = str(value.get("status") or "") non_terminal_event = event_type.lower() in { "assistant", "message", "tool", "tool.result", "tool_result", } if ( not non_terminal_event and ( event_type.lower() in { "error", "fatal", "request.failed", "turn.failed", } or severity.lower() in {"error", "fatal"} or ( status.lower() in {"failed", "rejected"} and any( field in value for field in ( "code", "error", "error_code", "status_code", ) ) ) ) ): return json.dumps(value, ensure_ascii=False) if cli == "claude": subtype = str(value.get("subtype", "")) if event_type == "rate_limit_event": rate_limit_info = value.get("rate_limit_info") if isinstance(rate_limit_info, dict) and str( rate_limit_info.get("status", "") ).lower() == "rejected": return json.dumps(value, ensure_ascii=False) if event_type == "result" and ( value.get("is_error") or subtype.startswith("error") ): # Preserve typed terminal fields such as api_error_status=429 and # error=rate_limit. The human-readable result alone is not the # failure contract and may change between Claude CLI releases. return json.dumps(value, ensure_ascii=False) if event_type == "system" and subtype.startswith("error"): return json.dumps(value, ensure_ascii=False) return None def legacy_promotion_recovery( runs_root: Path, task: Task, state: dict[str, Any], ) -> LegacyPromotionRecovery | None: """Reclassify only an older dispatcher's exhausted generic terminal failure.""" blocked = str(state.get("blocked") or "") recovery_failures = state.get("recovery_failures") if not blocked or not isinstance(recovery_failures, dict): return None locator_match = re.search(r"(?:^|\s)locator=(.+?)\s*$", blocked) if locator_match is None: return None locator = Path(locator_match.group(1)) try: locator = locator.resolve(strict=True) runs_root = runs_root.resolve(strict=True) if not locator.is_relative_to(runs_root) or locator.name != "locator.json": return None latest_record = json.loads(locator.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return None role = str(latest_record.get("role") or "") try: failure_count = int(recovery_failures.get(role, 0)) except (TypeError, ValueError): return None prior_sha256 = str(latest_record.get("dispatcher_source_sha256") or "") failed_spec = agent_spec_from_record(latest_record) if ( latest_record.get("task") != task.name or latest_record.get("status") != "failed" or latest_record.get("failure_class") != "generic-error" or role not in {"worker", "selfcheck", "review"} or failure_count < RECOVERY_FAILURE_LIMIT or not prior_sha256 or prior_sha256 == DISPATCHER_SOURCE_SHA256 or failed_spec is None or promoted_spec(failed_spec, 0) is None ): return None plan = latest_record.get("plan_number") latest_attempt = latest_record.get("attempt") if not isinstance(latest_attempt, int): return None classified_attempts: list[ tuple[int, Path, str, str, str] ] = [] for attempt_directory in runs_root.iterdir(): candidate = attempt_directory / "locator.json" if not candidate.is_file(): continue try: record = json.loads(candidate.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue if ( record.get("task") != task.name or record.get("role") != role or record.get("plan_number") != plan or record.get("status") != "failed" or record.get("failure_class") != "generic-error" or record.get("dispatcher_source_sha256") != prior_sha256 or record.get("cli") != failed_spec.cli or record.get("model") != failed_spec.model or effective_reasoning_effort( agent_spec_from_record(record) or failed_spec ) != effective_reasoning_effort(failed_spec) or not isinstance(record.get("attempt"), int) ): continue diagnostics = attempt_terminal_diagnostics( attempt_directory, record, ) if not diagnostics: continue failure_class, evidence = classify_failure_with_evidence( "\n".join(diagnostic for _, diagnostic in diagnostics[-50:]) ) if failure_class not in CLOUD_PROMOTION_FAILURES or evidence is None: continue evidence_source = next( ( source for source, diagnostic in reversed(diagnostics) if diagnostic == evidence ), f"{failed_spec.cli}:terminal", ) classified_attempts.append( ( int(record["attempt"]), candidate.resolve(), failure_class, evidence, evidence_source, ) ) classified_attempts.sort(key=lambda item: item[0]) expected_attempts = list( range(latest_attempt - failure_count + 1, latest_attempt + 1) ) matching_attempts = [ item for item in classified_attempts if item[0] in expected_attempts ] if ( [item[0] for item in matching_attempts] != expected_attempts or matching_attempts[-1][1] != locator or len({item[2] for item in matching_attempts}) != 1 ): # Do not collapse a mixed or incomplete failure history to one retry. return None _, _, failure_class, evidence, evidence_source = matching_attempts[-1] return LegacyPromotionRecovery( locator=locator, role=role, failure_class=failure_class, evidence=evidence, evidence_source=evidence_source, prior_dispatcher_sha256=prior_sha256, failed_cli=failed_spec.cli, failed_model=failed_spec.model, failed_reasoning_effort=failed_spec.reasoning_effort, ) def persisted_legacy_promotion_recovery( task: Task, state: dict[str, Any], locator: Path, role: str, ) -> LegacyPromotionRecovery | None: metadata = state.get("legacy_terminal_reclassification") if not isinstance(metadata, dict): return None try: recorded_locator = Path(str(metadata["locator"])).resolve(strict=True) record = json.loads(recorded_locator.read_text(encoding="utf-8")) except (KeyError, OSError): return None except json.JSONDecodeError: return None if not isinstance(record, dict): return None failure_class = str(metadata.get("failure_class") or "") failed_spec = agent_spec_from_record(record) recorded_cli = str(metadata.get("failed_cli") or "") recorded_model = str(metadata.get("failed_model") or "") if ( recorded_locator != locator.resolve() or record.get("task") != task.name or record.get("role") != role or record.get("status") != "failed" or record.get("failure_class") != "generic-error" or failure_class not in CLOUD_PROMOTION_FAILURES or str(metadata.get("current_dispatcher_sha256") or "") != DISPATCHER_SOURCE_SHA256 or failed_spec is None or promoted_spec(failed_spec, 0) is None or (recorded_cli and recorded_cli != failed_spec.cli) or (recorded_model and recorded_model != failed_spec.model) ): return None return LegacyPromotionRecovery( locator=recorded_locator, role=role, failure_class=failure_class, evidence=failure_class, evidence_source=str(metadata.get("evidence_source") or "terminal"), prior_dispatcher_sha256=str( metadata.get("prior_dispatcher_sha256") or "unknown" ), failed_cli=failed_spec.cli, failed_model=failed_spec.model, failed_reasoning_effort=( str(metadata["failed_reasoning_effort"]) if metadata.get("failed_reasoning_effort") is not None else failed_spec.reasoning_effort ), ) def pending_persisted_legacy_promotion_recovery( task: Task, state: dict[str, Any], ) -> LegacyPromotionRecovery | None: metadata = state.get("legacy_terminal_reclassification") recovery_failures = state.get("recovery_failures") if not isinstance(metadata, dict) or not isinstance( recovery_failures, dict ): return None role = str(metadata.get("role") or "") if not role: pending_roles = [ str(candidate) for candidate, count in recovery_failures.items() if count ] if len(pending_roles) != 1: return None role = pending_roles[0] try: failure_count = int(recovery_failures.get(role, 0)) locator = Path(str(metadata["locator"])) except (KeyError, TypeError, ValueError): return None if not 0 < failure_count < RECOVERY_FAILURE_LIMIT: return None return persisted_legacy_promotion_recovery( task, state, locator, role, ) def codex_collaboration_tool(line: str) -> str | None: try: value = json.loads(line) except json.JSONDecodeError: return None item = value.get("item") or {} if ( value.get("type") == "item.started" and item.get("type") == "collab_tool_call" and item.get("tool") ): return str(item["tool"]) return None async def terminate_process_group( process: asyncio.subprocess.Process, grace_seconds: float = 5, ) -> None: """Terminate the exact subprocess group and escalate if descendants remain.""" try: os.killpg(process.pid, signal.SIGTERM) except ProcessLookupError: if process.returncode is None: await process.wait() return if process.returncode is None: try: await asyncio.wait_for(process.wait(), timeout=grace_seconds) except TimeoutError: try: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass await process.wait() return try: os.killpg(process.pid, 0) except ProcessLookupError: return try: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass def agy_log_diagnostics(path: Path) -> list[str]: if not path.exists(): return [] diagnostics: list[str] = [] for line in path.read_text(encoding="utf-8", errors="replace").splitlines()[-200:]: failure_class, evidence = classify_failure_with_evidence(line) if ( failure_class not in CLOUD_PROMOTION_FAILURES or evidence is None ): continue if failure_class == "provider-quota" and not re.search( ( r"RESOURCE[_ ]?EXHAUSTED" r"|\b(?:HTTP|status(?: code)?)\s*[:=]?\s*429\b" r"|\btoo many requests\b" r"|(?:rate.?limit|quota|capacity).{0,40}" r"(?:exceed|exhaust|reached|reject)" r"|(?:exceed|exhaust|reached|reject).{0,40}" r"(?:rate.?limit|quota|capacity)" r"|(?:rate.?limit|quota).{0,40}retry after" ), line, re.IGNORECASE, ): continue diagnostics.append(line) return diagnostics def attempt_terminal_diagnostics( attempt_directory: Path, record: dict[str, Any], ) -> list[tuple[str, str]]: spec = agent_spec_from_record(record) if spec is None: return [] try: stream_lines = (attempt_directory / "stream.log").read_text( encoding="utf-8", errors="replace", ).splitlines() except OSError: stream_lines = [] diagnostics: list[tuple[str, str]] = [] for stream_line in stream_lines: match = re.match(r"^\[(stdout|stderr)\]\s?(.*)$", stream_line) if match is None: continue channel, payload = match.groups() diagnostic = terminal_diagnostic(spec.cli, channel, payload) if diagnostic: diagnostics.append((f"{spec.cli}:{channel}", diagnostic)) if spec.cli == "agy": diagnostics.extend( ("agy:cli-log", diagnostic) for diagnostic in agy_log_diagnostics( attempt_directory / "agy-cli.log" ) ) return diagnostics def promoted_spec(spec: AgentSpec, recovery_count: int) -> AgentSpec | None: if spec.cli == "agy": return AgentSpec("claude", "claude-opus-4-8", "claude/claude-opus-4-8 xhigh") if spec.cli == "claude": return AgentSpec( "codex", "gpt-5.6-terra", "codex/gpt-5.6-terra high", reasoning_effort="high", ) if spec.cli == "codex" and recovery_count < 1: return spec return None def render_json_line(cli: str, line: str) -> tuple[list[str], str | None]: try: value = json.loads(line) except json.JSONDecodeError: return [line.rstrip()], None session_id = value.get("thread_id") or value.get("session_id") rendered: list[str] = [] if cli == "codex": if value.get("type") == "thread.started" and session_id: rendered.append(f"session={session_id}") item = value.get("item") or {} item_type = item.get("type") if item_type == "agent_message" and item.get("text"): rendered.extend(str(item["text"]).splitlines()) elif item_type == "command_execution": rendered.append(f"$ {item.get('command', '')} (exit={item.get('exit_code', '?')})") elif item_type in {"mcp_tool_call", "web_search"}: rendered.append(f"{item_type}: {item.get('server', '')} {item.get('tool', item.get('query', ''))}") elif value.get("type") == "turn.failed": rendered.append(str(value.get("error", value))) elif cli == "claude": message = value.get("message") or {} for block in message.get("content") or []: if block.get("type") == "text": rendered.extend(str(block.get("text", "")).splitlines()) elif block.get("type") == "tool_use": rendered.append(f"tool={block.get('name', '')}") if value.get("type") == "result" and value.get("result"): rendered.extend(str(value["result"]).splitlines()) session_id = session_id or value.get("session_id") return rendered, str(session_id) if session_id else None def native_session_path(cli: str, workspace: Path, session_id: str | None, attempt_dir: Path) -> str | None: if cli == "claude" and session_id: encoded = str(workspace).replace("/", "-") return str(Path.home() / ".claude" / "projects" / encoded / f"{session_id}.jsonl") if cli == "pi" and session_id: matches = list((attempt_dir / "pi-sessions").glob(f"*{session_id}*.jsonl")) return str(matches[0]) if matches else str(attempt_dir / "pi-sessions") if cli == "codex" and session_id: matches = list((Path.home() / ".codex" / "sessions").glob(f"**/*{session_id}*.jsonl")) return str(matches[0]) if matches else str(Path.home() / ".codex" / "sessions") return None def native_session_mtime_ns(path: str | None) -> int | None: if not path: return None candidate = Path(path) return candidate.stat().st_mtime_ns if candidate.is_file() else None def reverse_jsonl_lines(path: Path): with path.open("rb") as stream: stream.seek(0, os.SEEK_END) position = stream.tell() buffer = b"" while position > 0: read_size = min(8192, position) position -= read_size stream.seek(position) buffer = stream.read(read_size) + buffer lines = buffer.split(b"\n") buffer = lines[0] for line in reversed(lines[1:]): if line.strip(): yield line if buffer.strip(): yield buffer def pi_session_header_version(path: Path) -> int | None: with path.open("rb") as stream: first_line = stream.readline() if not first_line.strip(): return None header = json.loads(first_line) if not isinstance(header, dict) or header.get("type") != "session": return None version = header.get("version") return version if isinstance(version, int) else None def pi_native_session_state(path: str | None) -> PiSessionState: if not path: return PiSessionState("starting", reason="native-session-path-missing") candidate = Path(path) if not candidate.is_file(): return PiSessionState("starting", reason="native-session-file-missing") completed_ids: list[str] = [] expected_entry_id: str | None = None active_leaf_found = False try: version = pi_session_header_version(candidate) if version != PI_SESSION_SCHEMA_VERSION: return PiSessionState( "unknown", reason=( f"unsupported-session-version:{version}" if version is not None else "session-header-invalid" ), ) for raw_line in reverse_jsonl_lines(candidate): value = json.loads(raw_line) if not isinstance(value, dict): return PiSessionState("unknown", reason="invalid-entry-schema") if value.get("type") == "session": break entry_id = value.get("id") parent_id = value.get("parentId") if ( not isinstance(entry_id, str) or not entry_id or "parentId" not in value or (parent_id is not None and not isinstance(parent_id, str)) ): return PiSessionState("unknown", reason="invalid-entry-identity") if active_leaf_found and entry_id != expected_entry_id: continue active_leaf_found = True expected_entry_id = parent_id if value.get("type") != "message": continue message = value.get("message") if not isinstance(message, dict): return PiSessionState("unknown", reason="invalid-message-schema") role = message.get("role") if role == "toolResult": tool_call_id = message.get("toolCallId") if not isinstance(tool_call_id, str) or not tool_call_id: return PiSessionState( "unknown", reason="tool-result-id-missing" ) if tool_call_id in completed_ids: return PiSessionState( "unknown", reason="duplicate-tool-result-id" ) completed_ids.append(tool_call_id) continue if role == "user": if completed_ids: return PiSessionState( "unknown", reason="tool-results-without-assistant" ) return PiSessionState("awaiting-model", reason="user-message") if role != "assistant": return PiSessionState( "unknown", reason=f"unsupported-message-role:{role}" ) content = message.get("content") if not isinstance(content, list): return PiSessionState( "unknown", reason="assistant-content-not-list" ) if any( not isinstance(block, dict) or block.get("type") not in {"text", "thinking", "toolCall"} for block in content ): return PiSessionState( "unknown", reason="unsupported-assistant-content" ) tool_calls = [ block for block in content if isinstance(block, dict) and block.get("type") == "toolCall" ] if not tool_calls: if completed_ids: return PiSessionState( "unknown", reason="tool-results-without-tool-calls" ) return PiSessionState("finishing", reason="assistant-final") expected_ids: list[str] = [] for tool_call in tool_calls: tool_call_id = tool_call.get("id") if not isinstance(tool_call_id, str) or not tool_call_id: return PiSessionState( "unknown", reason="tool-call-id-missing" ) if tool_call_id in expected_ids: return PiSessionState( "unknown", reason="duplicate-tool-call-id" ) expected_ids.append(tool_call_id) unexpected_ids = [ tool_call_id for tool_call_id in completed_ids if tool_call_id not in expected_ids ] if unexpected_ids: return PiSessionState( "unknown", reason="tool-result-id-not-in-latest-batch" ) completed_set = set(completed_ids) completed = tuple( tool_call_id for tool_call_id in expected_ids if tool_call_id in completed_set ) pending = tuple( tool_call_id for tool_call_id in expected_ids if tool_call_id not in completed_set ) return PiSessionState( "tool-running" if pending else "awaiting-model", expected_tool_call_ids=tuple(expected_ids), completed_tool_call_ids=completed, pending_tool_call_ids=pending, reason=( "pending-tool-results" if pending else "all-tool-results-recorded" ), ) if completed_ids: return PiSessionState( "unknown", reason="tool-results-without-assistant" ) if active_leaf_found and expected_entry_id is not None: return PiSessionState("unknown", reason="active-branch-parent-missing") except (OSError, UnicodeDecodeError, json.JSONDecodeError): return PiSessionState("unknown", reason="unreadable-jsonl") return PiSessionState("starting", reason="no-message-events") def pi_native_session_phase(path: str | None) -> str: return pi_native_session_state(path).phase def log_tail_excerpt(path: Path, *, byte_limit: int = 8192, char_limit: int = 2000) -> str: """Return a bounded recent log excerpt without loading a long reasoning stream.""" try: with path.open("rb") as stream: stream.seek(max(0, path.stat().st_size - byte_limit)) text = stream.read().decode("utf-8", errors="replace") except OSError as exc: return f"" return text[-char_limit:] def process_start_token(value: Any) -> str | None: """Read Linux process start ticks so PID reuse is not treated as liveness.""" try: pid = int(value) text = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8") close = text.rfind(")") fields = text[close + 2 :].split() return fields[19] if close >= 0 and len(fields) > 19 else None except (TypeError, ValueError, OSError): return None def process_is_alive(value: Any, expected_start_token: Any = None) -> bool: """Return whether the same attempt/dispatcher process still exists.""" try: pid = int(value) if pid <= 0: return False os.kill(pid, 0) except (TypeError, ValueError, OSError): return False current_token = process_start_token(pid) if ( expected_start_token is not None and current_token is not None and str(expected_start_token) != current_token ): return False return True def marked_agent_process_pids(marker: str) -> list[int]: """Find live processes carrying the per-attempt environment marker.""" expected = f"{AGENT_PROCESS_MARKER_ENV}={marker}".encode() matches: list[int] = [] for environ in Path("/proc").glob("[0-9]*/environ"): try: values = environ.read_bytes().split(b"\0") pid = int(environ.parent.name) except (OSError, ValueError): continue if expected in values: matches.append(pid) return sorted(matches) def locator_workspace_ownership( locator_path: Path, locator: dict[str, Any], *, expected_workspace: Path | None = None, expected_workspace_id: str | None = None, expected_runs_root: Path | None = None, ) -> tuple[bool, str]: if expected_workspace is None and expected_workspace_id is None: return True, "" try: expected_root = ( expected_workspace.resolve() if expected_workspace is not None else None ) expected_id = expected_workspace_id if expected_id is None and expected_root is not None: expected_id = hashlib.sha256(str(expected_root).encode()).hexdigest()[:16] if expected_runs_root is None: return False, "현재 workspace의 locator runs root가 없다" resolved_runs = expected_runs_root.resolve() resolved_locator = locator_path.resolve() resolved_locator.relative_to(resolved_runs) except (OSError, RuntimeError, ValueError): return ( False, "foreign workspace locator path: " f"locator={locator_path} expected_runs={expected_runs_root}", ) recorded_workspace = locator.get("workspace") recorded_workspace_id = locator.get("workspace_id") if recorded_workspace_id not in (None, "") and ( str(recorded_workspace_id) != str(expected_id) ): return ( False, "foreign workspace locator id: " f"recorded={recorded_workspace_id} expected={expected_id}", ) if recorded_workspace not in (None, ""): try: recorded_root = Path(str(recorded_workspace)).resolve() except (OSError, RuntimeError): return False, "locator workspace 경로를 canonicalize할 수 없다" if expected_root is not None and recorded_root != expected_root: return ( False, "foreign workspace locator root: " f"recorded={recorded_root} expected={expected_root}", ) evidence_fields = ["stream_log"] if locator.get("cli") == "pi": evidence_fields.append("native_session_path") for field in evidence_fields: raw_evidence = locator.get(field) if raw_evidence in (None, ""): continue try: Path(str(raw_evidence)).resolve().relative_to(resolved_runs) except (OSError, RuntimeError, ValueError): return ( False, "foreign workspace locator evidence: " f"field={field} path={raw_evidence}", ) # An identity-less legacy locator is accepted only because physical # containment under the current store's runs root was already proved. return True, "" def external_active_is_live( state: dict[str, Any], *, expected_workspace: Path | None = None, expected_workspace_id: str | None = None, expected_runs_root: Path | None = None, ) -> tuple[bool, str]: raw_locator = state.get("active_locator") if not raw_locator: return False, "active locator 없음" target = Path(str(raw_locator)) locator_path = target if target.name == "locator.json" else target / "locator.json" path_owned, ownership_detail = locator_workspace_ownership( locator_path, {}, expected_workspace=expected_workspace, expected_workspace_id=expected_workspace_id, expected_runs_root=expected_runs_root, ) if not path_owned: return False, ownership_detail locator: dict[str, Any] = {} if locator_path.is_file(): try: locator = json.loads(locator_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return False, f"locator 판독 실패: {locator_path}" if not isinstance(locator, dict): return False, f"locator object 형식이 아니다: {locator_path}" owned, ownership_detail = locator_workspace_ownership( locator_path, locator, expected_workspace=expected_workspace, expected_workspace_id=expected_workspace_id, expected_runs_root=expected_runs_root, ) if not owned: return False, ownership_detail if locator: status = str(locator.get("status") or "") if status and status != "running": return False, f"locator status={status}" # The stream may legitimately remain quiet during long reasoning. A live # process is stronger evidence than a locator or dispatcher heartbeat, and # prevents a second dispatcher from duplicating an active attempt. agent_pid_recorded = locator.get("agent_pid") not in (None, "") for field, token_field in ( ("agent_pid", "agent_process_start_token"), ("dispatcher_pid", "dispatcher_process_start_token"), ): if process_is_alive(locator.get(field), locator.get(token_field)): return True, f"{field}={locator[field]} alive; output stream is monitored" process_marker = str(locator.get("agent_process_marker") or "") if process_marker: marker_pids = marked_agent_process_pids(process_marker) if marker_pids: return ( True, "agent process marker alive: " + ",".join(str(pid) for pid in marker_pids), ) return ( False, "agent process marker is absent from the process table", ) if agent_pid_recorded: return ( False, "recorded agent process identity is no longer alive", ) native_raw = locator.get("native_session_path") native = Path(str(native_raw)) if native_raw else None if native and native.is_dir(): sessions = list(native.glob("*.jsonl")) native = max(sessions, key=lambda path: path.stat().st_mtime_ns) if sessions else None if native is None or not native.is_file(): roots = [target] if target.is_dir() else [target.parent] sessions = [ path for root in roots for path in (*root.glob("*.jsonl"), *root.glob("pi-sessions/*.jsonl")) ] native = max(sessions, key=lambda path: path.stat().st_mtime_ns) if sessions else None now = datetime.now(timezone.utc).timestamp() cli = str(locator.get("cli") or "") stream_progress_at: float | None = None stream_raw = locator.get("stream_log") stream = Path(str(stream_raw)) if stream_raw else None if stream and stream.is_file(): stream_progress_at = stream.stat().st_mtime if native and native.is_file(): native_progress_at = native.stat().st_mtime progress_at = max(native_progress_at, stream_progress_at or 0.0) inactive = max(0.0, now - progress_at) if cli == "pi": phase = pi_native_session_phase(str(native)) # Only an exact incomplete toolCall -> toolResult batch is a tool # execution interval. Unknown/starting/model-reasoning states # must never be treated as a stalled tool merely because their # native event file is quiet. if phase == "tool-running": return ( True, "phase=tool-running with no agent PID evidence; " "time-based duplicate recovery is disabled", ) return ( True, f"phase={phase} native+stream inactive={inactive:.1f}s " "with no agent PID evidence; time-based duplicate recovery is disabled", ) return ( True, "native+stream inactive=" f"{inactive:.1f}s with no agent PID evidence; " "time-based duplicate recovery is disabled", ) if stream_progress_at is not None: inactive = max(0.0, now - stream_progress_at) return ( True, f"stream inactive={inactive:.1f}s with no agent PID evidence; " "time-based duplicate recovery is disabled", ) return False, f"active 증거 없음: {raw_locator}" def laguna_resume_locator( state: dict[str, Any], *, expected_workspace: Path | None = None, expected_workspace_id: str | None = None, expected_runs_root: Path | None = None, ) -> Path | None: raw_locator = state.get("active_locator") if not raw_locator: return None target = Path(str(raw_locator)) locator = target if target.name == "locator.json" else target / "locator.json" path_owned, _ = locator_workspace_ownership( locator, {}, expected_workspace=expected_workspace, expected_workspace_id=expected_workspace_id, expected_runs_root=expected_runs_root, ) if not path_owned: return None try: record = json.loads(locator.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return None if not isinstance(record, dict): return None owned, _ = locator_workspace_ownership( locator, record, expected_workspace=expected_workspace, expected_workspace_id=expected_workspace_id, expected_runs_root=expected_runs_root, ) if not owned: return None if ( record.get("cli") != "pi" or not str(record.get("model", "")).startswith("laguna-s") or record.get("failure_class") not in {"context-limit", "session-stall"} or record.get("status") != "failed" ): return None native_raw = record.get("native_session_path") native = Path(str(native_raw)) if native_raw else None if native is None or not native.exists(): return None if expected_runs_root is not None: try: native.resolve().relative_to(expected_runs_root.resolve()) except (OSError, RuntimeError, ValueError): return None return locator def agy_conversations() -> dict[Path, int]: root = Path.home() / ".gemini" / "antigravity-cli" / "conversations" if not root.is_dir(): return {} return {path: path.stat().st_mtime_ns for path in root.glob("*.db")} def build_command( spec: AgentSpec, prompt: str, workspace: Path, session_id: str, attempt_dir: Path, pi_resume_session: Path | None = None, ) -> list[str]: if spec.cli == "codex": return [ "codex", "exec", "--json", "-C", str(workspace), "-m", spec.model, "-c", f'model_reasoning_effort="{effective_reasoning_effort(spec)}"', "--dangerously-bypass-approvals-and-sandbox", prompt, ] if spec.cli == "claude": return [ "claude", "-p", "--output-format", "stream-json", "--verbose", "--session-id", session_id, "--model", spec.model, "--effort", str(effective_reasoning_effort(spec)), "--dangerously-skip-permissions", prompt, ] if spec.cli == "agy": return [ # `--print` consumes its immediately following argument as the prompt. # Keeping the timeout first makes Gemini answer the literal flag instead. "agy", "--print", prompt, "--print-timeout", "8h", "--model", spec.model, "--dangerously-skip-permissions", "--log-file", str(attempt_dir / "agy-cli.log"), ] if spec.cli == "pi": command = [ "pi", "-p", "--mode", "json", "--approve", "--provider", "iop", "--model", spec.model, "--thinking", "high", ] if pi_resume_session is not None: command.extend( [ "--session", str(pi_resume_session), "--session-dir", str(pi_resume_session.parent), ] ) else: command.extend( [ "--session-id", session_id, "--session-dir", str(attempt_dir / "pi-sessions"), ] ) command.append(prompt) return command raise RuntimeError(f"지원하지 않는 CLI: {spec.cli}") async def invoke( workspace: Path, store: StateStore, task: Task, role: str, spec: AgentSpec, prompt: str, resume_locator: Path | None = None, ) -> tuple[int, str | None, Path]: attempt, identity = next_execution_identity(store, task, role) attempt_dir = store.runs / f"{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}__{identity}" attempt_dir.mkdir(parents=True, exist_ok=False) locator_path = attempt_dir / "locator.json" stream_path = attempt_dir / "stream.log" normalized_output_path = attempt_dir / "normalized-output.log" heartbeat_path = attempt_dir / "heartbeat.log" stream_path.touch() normalized_output_path.touch() heartbeat_path.touch() session_id = str(uuid.uuid4()) process_marker = f"w{store.workspace_id}__{identity}__{uuid.uuid4()}" pi_resume_session: Path | None = None if spec.local_pi and resume_locator and resume_locator.is_file(): resume_locator_path = ( resume_locator if resume_locator.name == "locator.json" else resume_locator / "locator.json" ) path_owned, _ = locator_workspace_ownership( resume_locator_path, {}, expected_workspace=store.workspace, expected_workspace_id=store.workspace_id, expected_runs_root=store.runs, ) if path_owned: try: prior = json.loads(resume_locator_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): prior = {} else: prior = {} owned, _ = locator_workspace_ownership( resume_locator_path, prior if isinstance(prior, dict) else {}, expected_workspace=store.workspace, expected_workspace_id=store.workspace_id, expected_runs_root=store.runs, ) if owned and isinstance(prior, dict): prior_native = prior.get("native_session_path") candidate = Path(str(prior_native)) if prior_native else None if candidate and candidate.is_dir(): sessions = list(candidate.glob("*.jsonl")) candidate = ( max(sessions, key=lambda path: path.stat().st_mtime_ns) if sessions else None ) if candidate and candidate.is_file(): try: candidate.resolve().relative_to(store.runs.resolve()) except (OSError, RuntimeError, ValueError): candidate = None if candidate and candidate.is_file(): pi_resume_session = candidate resume_locator = resume_locator_path session_id = str(prior.get("session_id") or candidate.stem) started_at = now_iso() work_log_path = milestone_work_log_path(task) record: dict[str, Any] = { "execution_id": identity, "task": task.name, "plan_number": plan_number(task), "role": role, "attempt": attempt, "workspace": str(store.workspace), "workspace_id": store.workspace_id, **dispatcher_source_provenance(), "cli": spec.cli, "model": spec.model, "reasoning_effort": effective_reasoning_effort(spec), "agent_process_marker": process_marker, "plan_path": str(task.plan) if task.plan else None, "review_path": str(task.review) if task.review else None, "session_id": session_id if spec.cli in {"claude", "pi"} else None, "native_session_path": ( str(pi_resume_session) if pi_resume_session is not None else native_session_path(spec.cli, workspace, session_id, attempt_dir) ), "output_log": str(stream_path), "stream_log": str(stream_path), "normalized_output_log": str(normalized_output_path), "heartbeat_log": str(heartbeat_path), "cli_log": str(attempt_dir / "agy-cli.log") if spec.cli == "agy" else None, "work_log": str(work_log_path.resolve()), "started_at": started_at, "status": "running", "resumed_from_locator": str(resume_locator) if pi_resume_session else None, } stage_decision = None if isinstance(store, StateStore): decisions = store.task_state(task).get("execution_decisions", {}) if isinstance(decisions, dict): stage_decision = decisions.get(role) if isinstance(stage_decision, dict): record.update(selector_runtime_evidence(stage_decision)) try: record["stage_budget"] = StageFailureBudget.from_decision(store, task, stage_decision).count() except Exception: record["stage_budget"] = 0 # Resolve the retry handoff identity that was assigned when the pending # quota refresh was created before the first durable locator write, so # the first record on disk already carries the stable handoff ID a # crash/restart can match against (the locator path changes on every # attempt). retry_handoff_id: str | None = None if isinstance(store, StateStore): retry_ctx = store.task_state(task).get("retry_quota_refresh_context") if isinstance(retry_ctx, dict): retry_handoff_id = retry_ctx.get("handoff_id") if retry_handoff_id: record["retry_handoff_id"] = retry_handoff_id write_json(locator_path, record) if isinstance(store, StateStore): if retry_handoff_id: # One-save transition: update active_locator, clear pending flag, # and clear context together. Restore pre-state on save failure. # A mismatch (False) or a save fault (raises) must stop before # the provider process seam so we never launch a duplicate # invocation against a retry intent we failed to commit. if not store.commit_retry_handoff_locator(task, retry_handoff_id, str(locator_path)): raise ExecutionDecisionError("retry handoff commit mismatch") else: store.update_task(task, active_locator=str(locator_path)) prefix = f"[{task.directory.name}][{role}][a{attempt:02d}]" def persist_locator_record() -> None: """Do not abort a live model solely because a locator refresh failed.""" try: write_json(locator_path, record) except OSError as exc: record["locator_write_error"] = str(exc) attempt_event( prefix, f"locator 기록 경고: locator={locator_path} error={exc}", ) attempt_event(prefix, f"locator={locator_path}") try: append_milestone_event( task, event="START", execution_id=identity, role=role, attempt=attempt, model=spec.display, result="running", locator=locator_path, ) except OSError as exc: line = f"milestone work log setup failed: {exc}" heartbeat_path.write_text(line + "\n", encoding="utf-8") record.update( status="failed", finished_at=now_iso(), exit_code=1, failure_class="work-log-setup", failure_source="work-log", provider_transport_failure_confirmed=False, work_log_error=str(exc), ) persist_locator_record() attempt_event(prefix, line) return 1, "work-log-setup", locator_path command = build_command( spec, prompt, workspace, session_id, attempt_dir, pi_resume_session=pi_resume_session, ) before_agy = agy_conversations() if spec.cli == "agy" else {} diagnostics: list[str] = [] diagnostic_origins: list[str] = [] control_violation: str | None = None try: process = await asyncio.create_subprocess_exec( *command, cwd=workspace, env={ **os.environ, AGENT_PROCESS_MARKER_ENV: process_marker, }, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, limit=10 * 1024 * 1024, start_new_session=True, ) # Keep the child PID in the locator before monitoring output. If this # dispatcher is interrupted, a later dispatcher can distinguish a # genuinely live, silent model from a stale locator and must not launch # a duplicate continuation. record["agent_pid"] = process.pid record["agent_process_start_token"] = process_start_token(process.pid) persist_locator_record() except FileNotFoundError: line = f"command not found: {command[0]}" heartbeat_path.write_text(line + "\n", encoding="utf-8") failure_class = "generic-error" try: append_milestone_event( task, event="FINISH", execution_id=identity, role=role, attempt=attempt, model=spec.display, result=f"failed:{failure_class}:127", locator=locator_path, ) except OSError as exc: record["work_log_runtime_error"] = str(exc) failure_class = "work-log-runtime-write" record.update( status="failed", finished_at=now_iso(), exit_code=127, failure_class=failure_class, failure_source=( "work-log" if failure_class == "work-log-runtime-write" else "cli-launch" ), provider_transport_failure_confirmed=False, ) persist_locator_record() attempt_event(prefix, line) return 127, failure_class, locator_path readers: list[asyncio.Task[None]] = [] try: assert process.stdout is not None and process.stderr is not None queue: asyncio.Queue[tuple[str, bytes | None]] = asyncio.Queue() async def pump(channel: str, stream: asyncio.StreamReader) -> None: try: while True: raw = await stream.readline() if not raw: break await queue.put((channel, raw)) finally: await queue.put((channel, None)) readers = [ asyncio.create_task(pump("stdout", process.stdout)), asyncio.create_task(pump("stderr", process.stderr)), ] finished_streams = 0 loop = asyncio.get_running_loop() last_native_mtime: int | None = None last_stream_mtime: int | None = None last_native_progress_at = loop.time() last_stream_progress_at = loop.time() with ( stream_path.open("w", encoding="utf-8") as stream_log, normalized_output_path.open("w", encoding="utf-8") as normalized_output_log, heartbeat_path.open("a", encoding="utf-8") as heartbeat_log, ): while finished_streams < len(readers): try: channel, raw = await asyncio.wait_for( queue.get(), timeout=STREAM_HEARTBEAT_SECONDS ) except asyncio.TimeoutError: try: stream_mtime = stream_path.stat().st_mtime_ns except OSError: stream_mtime = None if stream_mtime is not None: record["stream_log_mtime_ns"] = stream_mtime if stream_mtime != last_stream_mtime: last_stream_mtime = stream_mtime last_stream_progress_at = loop.time() record.pop("pi_silence_inspection", None) native_path = ( str(pi_resume_session) if pi_resume_session is not None else native_session_path( spec.cli, workspace, record.get("session_id"), attempt_dir, ) ) if native_path: record["native_session_path"] = native_path native_mtime = native_session_mtime_ns( record.get("native_session_path") ) if native_mtime is not None: record["native_session_mtime_ns"] = native_mtime if native_mtime != last_native_mtime: last_native_mtime = native_mtime last_native_progress_at = loop.time() # Native events and the separately flushed stream log # are peer progress signals. A trailing toolResult only # selects the timeout budget; it never overrides later # reasoning/text output. record["pi_activity_state"] = "working" pi_session_state = pi_native_session_state( record.get("native_session_path") ) pi_phase = pi_session_state.phase is_pi_tool_execution = pi_phase == "tool-running" # Outside a toolCall->toolResult interval, model stdout/stderr # is the liveness signal. A completed tool result changes phase # but must not reset the model-response silence clock. pi_inactive_seconds = loop.time() - ( max(last_native_progress_at, last_stream_progress_at) if is_pi_tool_execution else last_stream_progress_at ) if spec.local_pi: record["pi_session_phase"] = pi_phase record["pi_session_phase_reason"] = ( pi_session_state.reason ) record["pi_expected_tool_call_ids"] = list( pi_session_state.expected_tool_call_ids ) record["pi_completed_tool_call_ids"] = list( pi_session_state.completed_tool_call_ids ) record["pi_pending_tool_call_ids"] = list( pi_session_state.pending_tool_call_ids ) record["pi_stall_timeout_seconds"] = None record.setdefault("pi_activity_state", "starting") if ( spec.local_pi and not is_pi_tool_execution and pi_inactive_seconds >= PI_MODEL_RESPONSE_STALL_SECONDS and "pi_silence_inspection" not in record ): inspection = { "at": now_iso(), "silence_seconds": round(pi_inactive_seconds, 3), "stream_tail": log_tail_excerpt(stream_path), } record["pi_silence_inspection"] = inspection diagnostic = ( f"Pi {pi_phase} stream produced no update for " f"{pi_inactive_seconds:.1f}s; recorded stream tail for inspection " "without terminating the model process" ) heartbeat_log.write(f"[silence-inspection] {diagnostic}\n") heartbeat_log.flush() persist_locator_record() attempt_event(prefix, f"모델응답점검: {diagnostic}") non_pi_inactive_seconds = loop.time() - max( last_native_progress_at, last_stream_progress_at ) if ( not spec.local_pi and non_pi_inactive_seconds >= PI_MODEL_RESPONSE_STALL_SECONDS and "stream_silence_inspection" not in record ): inspection = { "at": now_iso(), "silence_seconds": round(non_pi_inactive_seconds, 3), "stream_tail": log_tail_excerpt(stream_path), } record["stream_silence_inspection"] = inspection diagnostic = ( f"{spec.cli} emitted no stream output or native-session event for " f"{non_pi_inactive_seconds:.1f}s; recorded stream tail for inspection " "without terminating the model process" ) heartbeat_log.write(f"[silence-inspection] {diagnostic}\n") heartbeat_log.flush() persist_locator_record() attempt_event(prefix, f"모델응답점검: {diagnostic}") heartbeat = ( f"작업중... locator={locator_path} " f"native_session={record.get('native_session_path') or 'none'} " f"native_mtime_ns={record.get('native_session_mtime_ns', 'none')}" ) if spec.local_pi: heartbeat += ( f" pi_activity={record.get('pi_activity_state')}" f" pi_phase={pi_phase}" ) heartbeat_log.write(f"[heartbeat] {heartbeat}\n") heartbeat_log.flush() persist_locator_record() # Heartbeat is recovery state, not a user-visible lifecycle # event. Keep it out of the caller-facing event stream. continue if raw is None: finished_streams += 1 continue record.pop("pi_silence_inspection", None) record.pop("stream_silence_inspection", None) if spec.local_pi and channel == "stdout": record["pi_activity_state"] = "streaming" line = raw.decode("utf-8", errors="replace").rstrip("\n") stream_log.write(f"[{channel}] {line}\n") stream_log.flush() diagnostic = terminal_diagnostic(spec.cli, channel, line) if diagnostic: diagnostics.append(diagnostic) diagnostic_origins.append(f"{spec.cli}:{channel}") if spec.cli == "codex" and role == "review" and channel == "stdout": collaboration_tool = codex_collaboration_tool(line) if collaboration_tool and control_violation is None: control_violation = collaboration_tool diagnostics.append( f"official review invoked forbidden collaboration tool: " f"{collaboration_tool}" ) diagnostic_origins.append("dispatcher:review-control") attempt_event( prefix, f"리뷰 제어 계약 위반: collaboration-tool=" f"{collaboration_tool}", ) await terminate_process_group(process) rendered, discovered = ( render_json_line(spec.cli, line) if channel == "stdout" else ([line], None) ) if discovered and record.get("session_id") != discovered: record["session_id"] = discovered if pi_resume_session is None: record["native_session_path"] = native_session_path( spec.cli, workspace, discovered, attempt_dir ) persist_locator_record() for display_line in rendered: if display_line: normalized_output_log.write(display_line + "\n") normalized_output_log.flush() # Child output is retained for recovery and review but is # not itself a dispatcher lifecycle event. await asyncio.gather(*readers) return_code = await process.wait() except asyncio.CancelledError: for reader in readers: reader.cancel() if readers: await asyncio.gather(*readers, return_exceptions=True) await terminate_process_group(process) runtime_error: OSError | None = None try: append_milestone_event( task, event="FINISH", execution_id=identity, role=role, attempt=attempt, model=spec.display, result="failed:cancelled", locator=locator_path, ) except OSError as exc: runtime_error = exc record.update( status="failed", finished_at=now_iso(), exit_code="cancelled", failure_class="cancelled", failure_source="caller-cancel", provider_transport_failure_confirmed=False, ) if runtime_error is not None: record["work_log_runtime_error"] = str(runtime_error) persist_locator_record() raise if spec.cli == "agy": after_agy = agy_conversations() changed = [ path for path, mtime in after_agy.items() if path not in before_agy or before_agy[path] != mtime ] if changed: selected = max(changed, key=lambda path: after_agy[path]) record["session_id"] = selected.stem record["native_session_path"] = str(selected) agy_diagnostics = agy_log_diagnostics(attempt_dir / "agy-cli.log") diagnostics.extend(agy_diagnostics) diagnostic_origins.extend("agy:cli-log" for _ in agy_diagnostics) native_path = ( str(pi_resume_session) if pi_resume_session is not None else native_session_path( spec.cli, workspace, record.get("session_id"), attempt_dir ) ) if native_path: record["native_session_path"] = native_path native_mtime = native_session_mtime_ns(record.get("native_session_path")) if native_mtime is not None: record["native_session_mtime_ns"] = native_mtime failure_source: str | None = None failure_evidence: str | None = None failure_evidence_source: str | None = None provider_transport_failure_confirmed = False termination = termination_signal(return_code) if termination is not None: record["termination_signal"] = termination[0] record["termination_signal_inferred"] = termination[1] if control_violation: failure_class = "review-control-violation" failure_source = "dispatcher-control" failure_evidence_source = "dispatcher:review-control" for index in range(len(diagnostics) - 1, -1, -1): if diagnostic_origins[index] == failure_evidence_source: failure_evidence = diagnostics[index] break elif return_code != 0 and termination is not None: failure_class = "process-terminated" failure_source = "process-termination" record["termination_initiator"] = "unknown" else: classified_failure, classified_evidence = classify_failure_with_evidence( "\n".join(diagnostics[-50:]) ) if return_code != 0 or classified_evidence is not None: failure_class = classified_failure failure_evidence = classified_evidence else: failure_class = None if failure_class is not None and failure_evidence is not None: for index in range(len(diagnostics) - 1, -1, -1): if diagnostics[index] == failure_evidence: failure_evidence_source = diagnostic_origins[index] break if failure_class in PROVIDER_TRANSPORT_FAILURES: failure_source = "provider-terminal-diagnostic" provider_transport_failure_confirmed = failure_evidence is not None elif failure_evidence is not None: failure_source = "cli-terminal-diagnostic" elif return_code != 0: failure_source = "cli-exit" try: append_milestone_event( task, event="FINISH", execution_id=identity, role=role, attempt=attempt, model=spec.display, result=( f"succeeded:0" if return_code == 0 and failure_class is None else f"failed:{failure_class or 'generic-error'}:{return_code}" ), locator=locator_path, ) except OSError as exc: if failure_class is not None: record["prior_failure_class"] = failure_class record["work_log_runtime_error"] = str(exc) failure_class = "work-log-runtime-write" failure_source = "work-log" failure_evidence = None failure_evidence_source = None provider_transport_failure_confirmed = False if failure_evidence is not None: record["failure_evidence_excerpt"] = failure_evidence[:FAILURE_EVIDENCE_LIMIT] record["failure_evidence_truncated"] = ( len(failure_evidence) > FAILURE_EVIDENCE_LIMIT ) if failure_evidence_source is not None: record["failure_evidence_source"] = failure_evidence_source record.update( status="succeeded" if return_code == 0 and failure_class is None else "failed", finished_at=now_iso(), exit_code=return_code, failure_class=failure_class, failure_source=failure_source, provider_transport_failure_confirmed=provider_transport_failure_confirmed, ) persist_locator_record() return return_code, failure_class, locator_path def base_prompt(task: Task, role: str, spec: AgentSpec) -> str: if role == "review": target = task.review or task.directory if task.review: return f"Read {target.resolve()} and start the review. Keep artifact content in English. Final in Korean." return f"Continue the review for {target.resolve()}. Keep artifact content in English. Final in Korean." if task.plan is None: raise RuntimeError("worker PLAN이 없다") target = task.plan.resolve() if role == "selfcheck": if task.review is None: raise RuntimeError("selfcheck CODE_REVIEW 파일이 없다") return ( f"Think in English. Keep artifact content in English. Final in Korean. Read {task.review.resolve()} and fill " "every missing implementation field. Do not finish until all implementation " "fields are complete. This is a self-check of completed work, not a review. " f"Read {target} and finish any missing work. Recheck and fix your work." ) if spec.local_pi: return f"Think in English. Keep artifact content in English. Final in Korean. Read {target} and complete the task." return f"Read {target} and complete the task. Keep artifact content in English. Final in Korean." def build_context_package( workspace: Path, task: Task, locator: Path, *, previous_spec: AgentSpec, next_spec: AgentSpec ) -> dict[str, Any]: """Build the fail-closed continuation context for a target transition.""" if not locator.is_file(): raise ExecutionDecisionError("logical context locator가 없다") try: record = json.loads(locator.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise ExecutionDecisionError("logical context locator를 읽을 수 없다") from exc if not isinstance(record, dict): raise ExecutionDecisionError("logical context locator 형식이 잘못됐다") workspace_value = record.get("workspace") if not isinstance(workspace_value, str) or not workspace_value: raise ExecutionDecisionError("logical context workspace가 없다") recorded_workspace = Path(workspace_value) if not recorded_workspace.is_absolute() or recorded_workspace.resolve() != workspace.resolve(): raise ExecutionDecisionError("logical context workspace가 일치하지 않는다") if record.get("task") != task.name: raise ExecutionDecisionError("logical context task가 일치하지 않는다") if task.plan is None or not task.plan.is_file(): raise ExecutionDecisionError("logical context PLAN이 없다") plan_value = record.get("plan_path") if not isinstance(plan_value, str) or not plan_value: raise ExecutionDecisionError("logical context PLAN 경로가 없다") plan_path = Path(plan_value) if not plan_path.is_absolute() or plan_path.resolve() != task.plan.resolve(): raise ExecutionDecisionError("logical context PLAN 경로가 일치하지 않는다") required = { "normalized_output": ("normalized_output_log", "normalized-output.log"), "raw_log": ("stream_log", "stream.log"), } paths: dict[str, str] = {} attempt_dir = locator.resolve().parent for field, (record_field, filename) in required.items(): value = record.get(record_field) if not isinstance(value, str) or not value: raise ExecutionDecisionError(f"logical context {field} artifact가 없다") path = Path(value) expected = attempt_dir / filename if not path.is_absolute() or path.resolve() != expected or not expected.is_file(): raise ExecutionDecisionError(f"logical context {field} artifact가 locator attempt와 일치하지 않는다") paths[field] = str(expected) same_pi = previous_spec.local_pi and next_spec.local_pi package = { "plan": str(task.plan.resolve()), "locator": str(locator.resolve()), "workspace": str(workspace.resolve()), **paths, "resume_mode": "native" if same_pi else "logical", } if same_pi: native = Path(str(record.get("native_session_path", ""))) if not native.is_file(): raise ExecutionDecisionError("same-Pi logical context native session이 없다") package["native_session_path"] = str(native.resolve()) return package def canonical_selector_failover_route(decision: dict[str, Any] | None) -> bool: if not isinstance(decision, dict): return False candidates = decision.get("candidates") return isinstance(candidates, list) and len(candidates) > 1 def logical_context_prompt(context: dict[str, Any]) -> str: plan = context["plan"] locator = context["locator"] workspace = context["workspace"] raw_log = context["raw_log"] normalized_output = context["normalized_output"] return ( f"Think in English. Keep artifact content in English. Final in Korean. " f"Read plan={plan}, locator={locator}, workspace={workspace}, " f"raw_log={raw_log}, normalized_output={normalized_output} and complete the task." ) def continuation_prompt_from_package( context_package: dict[str, Any], *, target: dict[str, Any] | None = None, native_resume: bool = False, ) -> str: if native_resume or context_package.get("resume_mode") == "native": return ( "Think in English. Keep artifact content in English. Final in Korean. Continue this session and complete " "the current task." ) plan = context_package["plan"] locator = context_package["locator"] workspace = context_package["workspace"] raw_log = context_package["raw_log"] normalized_output = context_package["normalized_output"] return ( f"Think in English. Keep artifact content in English. Final in Korean. " f"Read plan={plan}, locator={locator}, workspace={workspace}, " f"raw_log={raw_log}, normalized_output={normalized_output} and complete the task." ) def continuation_prompt( task: Task, role: str, locator: Path | None = None, *, local_pi: bool = False, resume_same_pi_session: bool = False, context: dict[str, Any] | None = None, ) -> str: if context is not None: return continuation_prompt_from_package( context, native_resume=resume_same_pi_session or context.get("resume_mode") == "native", ) if local_pi: if resume_same_pi_session: return ( "Think in English. Keep artifact content in English. Final in Korean. Continue this session and complete " "the current task." ) if role == "selfcheck" and task.plan and task.review: return ( f"Think in English. Keep artifact content in English. Final in Korean. Read {task.review.resolve()} and fill " "every missing implementation field. Do not finish until all implementation " "fields are complete. This is a self-check of completed work, not a review. " f"Read {task.plan.resolve()} and finish any missing work. Recheck and fix " "your work." ) target = task.plan or task.directory return f"Think in English. Keep artifact content in English. Final in Korean. Read {target.resolve()} and complete the task." if role == "review": return f"Continue the review for {task.directory.resolve()}. Keep artifact content in English. Final in Korean." return ( f"Continue from {locator.resolve() if locator else task.directory.resolve()}. Check the saved context and current " "workspace. Keep artifact content in English. Final in Korean." ) async def run_escalating( workspace: Path, store: StateStore, task: Task, role: str, initial: AgentSpec, initial_resume_locator: Path | None = None, ) -> tuple[bool, Path | None]: spec = initial previous_locator = initial_resume_locator codex_recovery_count = 0 codex_session_stall_retries = 0 review_control_retries = 0 pi_recovery_retries = 0 generic_retries = 0 terminal_recovery_retries = 0 pi_resume_locator = initial_resume_locator recovery_failures = 0 stage_budget: StageFailureBudget | None = None if isinstance(store, StateStore): state = store.task_state(task) persisted = state.get("recovery_failures", {}) if isinstance(persisted, dict): recovery_failures = int(persisted.get(role, 0)) decisions = state.get("execution_decisions", {}) decision = decisions.get(role) if isinstance(decisions, dict) and role in {"worker", "review"} else None if isinstance(decision, dict): stage_budget = StageFailureBudget.from_decision(store, task, decision) recovery_failures = stage_budget.count() legacy_recovery: LegacyPromotionRecovery | None = None if initial_resume_locator is not None and isinstance(store, StateStore): state = store.task_state(task) legacy_recovery = legacy_promotion_recovery( store.runs, task, state, ) if legacy_recovery is not None: recovery_failures = 1 persisted_failures = dict(state.get("recovery_failures", {})) persisted_failures[legacy_recovery.role] = recovery_failures store.update_task( task, blocked=None, recovery_failures=persisted_failures, legacy_terminal_reclassification={ "role": legacy_recovery.role, "failure_class": legacy_recovery.failure_class, "evidence_source": legacy_recovery.evidence_source, "prior_dispatcher_sha256": legacy_recovery.prior_dispatcher_sha256, "current_dispatcher_sha256": DISPATCHER_SOURCE_SHA256, "locator": str(legacy_recovery.locator), "failed_cli": legacy_recovery.failed_cli, "failed_model": legacy_recovery.failed_model, "failed_reasoning_effort": legacy_recovery.failed_reasoning_effort, }, ) else: legacy_recovery = persisted_legacy_promotion_recovery( task, state, initial_resume_locator, role, ) if legacy_recovery is not None and legacy_recovery.role == role: failed_spec = failed_spec_from_recovery(legacy_recovery) next_spec = promoted_spec(failed_spec, codex_recovery_count) if next_spec is not None: banner( "모델승격", task.name, [ f"from={failed_spec.display}", f"to={next_spec.display}", f"failure_class={legacy_recovery.failure_class}", "failure_source=legacy-terminal-reclassification", f"failure_evidence_source={legacy_recovery.evidence_source}", "dispatcher_source_sha256=" f"{legacy_recovery.prior_dispatcher_sha256}", f"dispatcher_source_current_sha256={DISPATCHER_SOURCE_SHA256}", f"locator={legacy_recovery.locator}", ], ) spec = next_spec if recovery_failures >= RECOVERY_FAILURE_LIMIT: locator = initial_resume_locator reason = ( f"{role} recovery failure limit already exhausted: " f"{recovery_failures}/{RECOVERY_FAILURE_LIMIT}" ) if isinstance(store, StateStore): decision = store.task_state(task).get("execution_decisions", {}).get(role, {}) store.update_task( task, blocked=f"{reason} locator={locator}", blocker_evidence={ "role": role, "failure_class": None, "locator": str(locator) if locator else None, "selected": decision.get("selected") if isinstance(decision, dict) else None, "work_unit_id": decision.get("work_unit_id") if isinstance(decision, dict) else None, }, ) banner( "작업차단", task.name, [ "reason=recovery-failure-limit", f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}", f"locator={locator}", ], ) return False, locator context: dict[str, Any] | None = None while True: prompt = ( base_prompt(task, role, spec) if previous_locator is None else continuation_prompt( task, role, previous_locator, local_pi=spec.local_pi, resume_same_pi_session=pi_resume_locator is not None, context=context, ) ) context = None rc, failure, locator = await invoke( workspace, store, task, role, spec, prompt, resume_locator=pi_resume_locator, ) pi_resume_locator = None if rc == 0 and failure is None: if isinstance(store, StateStore): state = store.task_state(task) persisted = dict(state.get("recovery_failures", {})) persisted.pop(role, None) store.update_task(task, recovery_failures=persisted) if stage_budget is not None: stage_budget.reset_on_success() return True, locator failure = failure or "generic-error" if failure in { "work-log-blocked", "work-log-incomplete", "work-log-setup", "work-log-runtime-write", }: banner("작업차단", task.name, failure_report_lines(failure, locator)) return False, locator recovery_failures += 1 if stage_budget is not None: selected = stage_budget.store.task_state(task)["execution_decisions"][role]["selected"] transition = stage_budget.store.task_state(task)["execution_decisions"][role]["transition"]["trigger"] recovery_failures = stage_budget.record_failure(target=selected, transition=transition) if isinstance(store, StateStore): state = store.task_state(task) persisted = dict(state.get("recovery_failures", {})) persisted[role] = recovery_failures store.update_task(task, recovery_failures=persisted) if recovery_failures >= RECOVERY_FAILURE_LIMIT: reason = ( f"{role} recovery failure limit exhausted: " f"{recovery_failures}/{RECOVERY_FAILURE_LIMIT}" ) if isinstance(store, StateStore): decision = store.task_state(task).get("execution_decisions", {}).get(role, {}) store.update_task( task, blocked=f"{reason} locator={locator}", blocker_evidence={ "role": role, "failure_class": failure, "locator": str(locator) if locator else None, "selected": decision.get("selected") if isinstance(decision, dict) else None, "work_unit_id": decision.get("work_unit_id") if isinstance(decision, dict) else None, }, ) banner( "작업차단", task.name, [ "reason=recovery-failure-limit", *failure_report_lines(failure, locator), f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}", ], ) return False, locator if role == "review" and failure == "review-control-violation": review_control_retries += 1 banner( "리뷰재시도", task.name, [ *failure_report_lines(failure, locator), f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}", ], ) previous_locator = None await asyncio.sleep(min(30, 2 ** min(review_control_retries, 5))) continue current_decision = None quota_snapshot = None if isinstance(store, StateStore): task_state = store.task_state(task) decisions = task_state.get("execution_decisions", {}) if isinstance(decisions, dict): current_decision = decisions.get(role) quota_snapshot = task_state.get("quota_snapshot") if canonical_selector_failover_route(current_decision) and failure in QUALIFIED_FAILOVER_FAILURES: try: if failure == "provider-quota": derived = derive_work_unit_quota_evidence( current_decision, status="exhausted", reason="confirmed_runtime_provider_quota", ) quota_snapshot = derived next_decision = select_execution_decision( task, stage=role, prior_decision=current_decision, quota_snapshot=quota_snapshot, transition="failover", failure_class=failure, ) next_spec = agent_spec_from_decision(next_decision) if next_spec != spec: if locator is None: raise ExecutionDecisionError("logical context locator가 없다") context = build_context_package( workspace, task, locator, previous_spec=spec, next_spec=next_spec ) commit_execution_decision(store, task, role, next_decision) banner( "모델승격" if role == "worker" else "리뷰승격", task.name, [ f"from={spec.display}", f"to={next_spec.display}", *failure_report_lines(failure, locator), ], ) spec = next_spec previous_locator = locator continue else: commit_execution_decision(store, task, role, next_decision) except (ExecutionDecisionError, OSError, ValueError) as exc: code = getattr(exc, "code", "") if not code: if "no_failover_candidate" in str(exc): code = "no_failover_candidate" else: code = exc.__class__.__name__ store.update_task( task, blocked=f"{role} selector decision 실패 [{code}]: {exc}" ) banner( "작업차단", task.name, [f"reason={code}", *failure_report_lines(failure, locator)], ) return False, locator if spec.local_pi: if ( spec.model.startswith("laguna-s") and failure in {"context-limit", "session-stall"} ): pi_recovery_retries += 1 banner( "Pi세션연속재시작", task.name, [ f"model={spec.display}", *failure_report_lines(failure, locator), f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}", ], ) previous_locator = locator pi_resume_locator = locator await asyncio.sleep(min(30, 2 ** min(pi_recovery_retries, 5))) continue pi_recovery_retries += 1 if failure == "session-stall": event = "세션응답복구재시도" elif failure in { "provider-connection", "provider-stream-disconnect", }: event = "세션연결재시도" else: event = "Pi복구재시도" banner( event, task.name, [ f"model={spec.display}", *failure_report_lines(failure, locator), f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}", ], ) previous_locator = locator await asyncio.sleep(min(30, 2 ** min(pi_recovery_retries, 5))) continue if spec.cli == "codex" and failure == "session-stall": codex_session_stall_retries += 1 banner( "세션응답복구재시도", task.name, [ f"model={spec.display}", *failure_report_lines(failure, locator), f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}", ], ) previous_locator = locator await asyncio.sleep(min(30, 2 ** min(codex_session_stall_retries, 5))) continue if failure == "generic-error": generic_retries += 1 banner( "작업복구재시도", task.name, [ f"model={spec.display}", *failure_report_lines(failure, locator), f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}", ], ) previous_locator = locator await asyncio.sleep(min(30, 2 ** min(generic_retries, 5))) continue if failure not in CLOUD_PROMOTION_FAILURES: terminal_recovery_retries += 1 banner( "모델복구재시도", task.name, [ f"model={spec.display}", *failure_report_lines(failure, locator), f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}", ], ) previous_locator = locator await asyncio.sleep( min(30, 2 ** min(terminal_recovery_retries, 5)) ) continue if role == "review": terminal_recovery_retries += 1 banner( "리뷰재시도", task.name, [ f"model={spec.display}", "reason=official-review-fixed-target", *failure_report_lines(failure, locator), f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}", ], ) previous_locator = locator await asyncio.sleep( min(30, 2 ** min(terminal_recovery_retries, 5)) ) continue if ( isinstance(store, StateStore) and role == "worker" and isinstance(current_decision, dict) ): try: next_decision = select_execution_decision( task, stage=role, prior_decision=current_decision, quota_snapshot=quota_snapshot, transition="promotion", failure_class=failure, ) except ExecutionDecisionError as exc: if "no_promotion_target" in str(exc): # canonical promotion 대상 없음: selector-backed worker는 # 현재 target recovery/budget exhaustion 또는 block으로만 종결. # legacy promoted_spec()으로의 fallthrough 금지. banner( "모델재시도", task.name, [ f"model={spec.display}", "reason=no-promotion-target", *failure_report_lines(failure, locator), f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}", ], ) previous_locator = locator terminal_recovery_retries += 1 await asyncio.sleep( min(30, 2 ** min(terminal_recovery_retries, 5)) ) continue store.update_task( task, blocked=f"{role} selector promotion 실패: {exc}", ) banner( "작업차단", task.name, [ "reason=selector-promotion", *failure_report_lines(failure, locator), ], ) return False, locator else: next_spec = agent_spec_from_decision(next_decision) if next_spec == spec: store.update_task( task, blocked="selector promotion이 현재 target을 다시 선택했다", ) return False, locator if locator is None: store.update_task( task, blocked="logical context locator가 없다" ) return False, locator try: context = build_context_package( workspace, task, locator, previous_spec=spec, next_spec=next_spec, ) except ExecutionDecisionError as exc: store.update_task(task, blocked=str(exc)) return False, locator commit_execution_decision(store, task, role, next_decision) banner( "모델승격", task.name, [ f"from={spec.display}", f"to={next_spec.display}", *failure_report_lines(failure, locator), ], ) spec = next_spec previous_locator = locator continue next_spec = promoted_spec(spec, codex_recovery_count) if next_spec is None: terminal_recovery_retries += 1 banner( "모델복구재시도", task.name, [ f"model={spec.display}", *failure_report_lines(failure, locator), f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}", ], ) previous_locator = locator await asyncio.sleep(min(30, 2 ** min(terminal_recovery_retries, 5))) continue if spec.cli == "codex": codex_recovery_count += 1 banner( "모델승격", task.name, [ f"from={spec.display}", f"to={next_spec.display}", *failure_report_lines(failure, locator), ], ) spec = next_spec previous_locator = locator def task_signature(workspace: Path, task: Task) -> str: digest = hashlib.sha256() if not task.directory.exists(): return "moved" for path in sorted(p for p in task.directory.iterdir() if p.is_file()): if ( PLAN_RE.match(path.name) or REVIEW_RE.match(path.name) or path.name.endswith(".log") ): digest.update(path.name.encode()) digest.update(sha256_file(path).encode()) for raw_path in sorted(task.write_set): path = Path(raw_path) path = path if path.is_absolute() else workspace / path digest.update(raw_path.encode()) if path.is_file(): digest.update(str(path.stat().st_mode).encode()) digest.update(sha256_file(path).encode()) elif path.exists(): digest.update(b"non-file") else: digest.update(b"missing") return digest.hexdigest() def read_verdict(path: Path) -> str | None: if not path.exists(): return None return verdict_from_text(path.read_text(encoding="utf-8", errors="replace")) def verdict_from_text(text: str) -> str | None: selected: tuple[re.Match[str], re.Pattern[str], re.Pattern[str]] | None = None for heading_re, line_re, block_re in VERDICT_SCHEMA_MATCHERS: headings = list(heading_re.finditer(text)) if not headings: continue # A duplicated heading, or headings from both schemas, is ambiguous. if len(headings) != 1 or selected is not None: return None selected = (headings[0], line_re, block_re) if selected is None: return None heading, line_re, block_re = selected next_heading = re.search(r"^##\s+", text[heading.end():], re.MULTILINE) end = heading.end() + next_heading.start() if next_heading else len(text) section = text[heading.end():end] inline_matches = list(line_re.finditer(section)) block_matches = list(block_re.finditer(section)) matches = inline_matches + block_matches return matches[0].group(1) if len(matches) == 1 else None def matching_archive_directories_by_name( workspace: Path, task_name: str, *, require_complete: bool = True, ) -> list[Path]: archive = workspace / "agent-task" / "archive" parts = task_name.split("/") if not archive.is_dir() or len(parts) not in {1, 2}: return [] group = parts[0] final_name = parts[-1] suffix_re = re.compile(rf"^{re.escape(final_name)}(?:_\d+)?$") matches: list[Path] = [] try: years = list(archive.iterdir()) except FileNotFoundError: return [] for year in years: if not year.is_dir(): continue try: months = list(year.iterdir()) except FileNotFoundError: continue for month in months: if not month.is_dir(): continue parent = month if len(parts) == 1 else month / group if not parent.is_dir(): continue try: candidates = list(parent.iterdir()) except FileNotFoundError: continue for candidate in candidates: if ( candidate.is_dir() and suffix_re.match(candidate.name) and ( not require_complete or (candidate / "complete.log").is_file() ) ): matches.append(candidate) return sorted(matches) def matching_archive_directories(workspace: Path, task: Task) -> list[Path]: return matching_archive_directories_by_name(workspace, task.name) def task_group_name(task_name: str) -> str: return task_name.split("/", 1)[0] def work_log_event_cells(line: str) -> list[str] | None: stripped = line.strip() if not stripped.startswith("|") or not stripped.endswith("|"): return None cells = [ cell.strip().replace(r"\|", "|") for cell in re.split(r"(? str: """Merge duplicate dispatcher timelines without losing conflicting rows.""" allowed_metadata = { "# Milestone Work Log", "## Dispatcher Timeline", "> Dispatcher-owned execution timeline. Workers and reviewers do not edit this file.", "> Dispatcher-owned. Workers and reviewers do not edit this section.", "| seq | time | event | task | role | attempt | model | result | locator |", "|---:|---|---|---|---|---:|---|---|---|", } unique_rows: dict[tuple[str, str, str, str, str], tuple[str, ...]] = {} ordered_rows: list[tuple[str, int, int, list[str]]] = [] for source_index, source in enumerate(sorted(sources)): try: lines = source.read_text( encoding="utf-8", errors="replace", ).splitlines() except OSError as exc: raise ValueError( f"WORK_LOG source를 읽을 수 없다: source={source} error={exc}" ) from exc for line_number, line in enumerate(lines, start=1): cells = work_log_event_cells(line) if cells is None: if not line.strip() or line.strip() in allowed_metadata: continue raise ValueError( "WORK_LOG 병합 대상에 안전하게 보존할 수 없는 내용이 있다: " f"source={source} line={line_number}" ) if cells[0] == "seq" or cells[0].startswith("---"): continue if cells[2] not in {"START", "FINISH"}: raise ValueError( "WORK_LOG 병합 대상에 지원하지 않는 event가 있다: " f"source={source} line={line_number} event={cells[2]}" ) try: sequence = int(cells[0]) int(cells[5]) except ValueError as exc: raise ValueError( "WORK_LOG 병합 대상의 sequence 또는 attempt가 유효하지 않다: " f"source={source} line={line_number}" ) from exc key = (cells[2], cells[3], cells[4], cells[5], cells[8]) fingerprint = tuple(cells[1:]) previous = unique_rows.get(key) if previous is not None: if previous != fingerprint: raise ValueError( "WORK_LOG 병합 충돌: " f"event={cells[2]} task={cells[3]} role={cells[4]} " f"attempt={cells[5]} locator={cells[8]}" ) continue unique_rows[key] = fingerprint ordered_rows.append((cells[1], source_index, sequence, cells)) if not ordered_rows: raise ValueError("WORK_LOG 병합 대상에 timeline row가 없다") header = ( "# Milestone Work Log\n\n" "> Dispatcher-owned execution timeline. Workers and reviewers do not edit this file.\n\n" "| seq | time | event | task | role | attempt | model | result | locator |\n" "|---:|---|---|---|---|---:|---|---|---|\n" ) rendered_rows: list[str] = [] for sequence, (_, _, _, cells) in enumerate(sorted(ordered_rows), start=1): escaped = [str(value).replace("|", r"\|").replace("\n", " ") for value in cells] escaped[0] = str(sequence) rendered_rows.append("| " + " | ".join(escaped) + " |\n") return header + "".join(rendered_rows) def unfinished_work_log_attempts(path: Path) -> list[dict[str, Any]]: """Return START rows that have no matching FINISH row.""" try: lines = path.read_text( encoding="utf-8", errors="replace", ).splitlines() except OSError: raise open_attempts: dict[str, dict[str, Any]] = {} for line in lines: cells = work_log_event_cells(line) if cells is None or cells[2] not in {"START", "FINISH"}: continue try: sequence = int(cells[0]) attempt = int(cells[5]) except ValueError: continue locator = cells[8] key = locator or "\0".join( (cells[3], cells[4], cells[5], cells[6]) ) if cells[2] == "START": open_attempts[key] = { "sequence": sequence, "task_name": cells[3], "role": cells[4], "attempt": attempt, "model": cells[6], "locator": locator, } else: open_attempts.pop(key, None) return sorted( open_attempts.values(), key=lambda record: int(record["sequence"]), ) def close_unfinished_work_log_attempts(path: Path) -> int: """Close orphaned START rows after verified group completion.""" unfinished = unfinished_work_log_attempts(path) for record in unfinished: locator = Path(str(record["locator"])) append_work_log_event( path, task_name=str(record["task_name"]), event="FINISH", execution_id=f"reconciled-{record['sequence']}", role=str(record["role"]), attempt=int(record["attempt"]), model=str(record["model"]), result="reconciled:verified-complete-archive", locator=locator, ) return len(unfinished) def archived_task_group_directories( workspace: Path, task_group: str, ) -> list[Path]: """Return month-local archive directories for one logical task group.""" archive_root = workspace / "agent-task" / "archive" if not archive_root.is_dir(): return [] suffix_re = re.compile(rf"^{re.escape(task_group)}(?:_\d+)?$") matches: list[Path] = [] try: years = list(archive_root.iterdir()) except FileNotFoundError: return [] for year in years: if not year.is_dir(): continue try: months = list(year.iterdir()) except FileNotFoundError: continue for month in months: if not month.is_dir(): continue try: candidates = list(month.iterdir()) except FileNotFoundError: continue matches.extend( candidate for candidate in candidates if candidate.is_dir() and suffix_re.fullmatch(candidate.name) ) return sorted(matches) def next_work_log_archive_number( workspace: Path, task_group: str, ) -> int: numbers = [ int(match.group(1)) for directory in archived_task_group_directories(workspace, task_group) for path in directory.glob("work_log_*.log") if (match := WORK_LOG_ARCHIVE_RE.fullmatch(path.name)) ] return max(numbers, default=-1) + 1 def completed_group_archive_directory( task_group: str, task_names: set[str], completed_tasks: dict[str, str], ) -> Path | None: candidates: list[tuple[int, str, Path]] = [] for task_name in task_names: archive_raw = completed_tasks.get(task_name) if not archive_raw: continue archive = Path(archive_raw) if not archive.is_dir(): continue complete_log = archive / "complete.log" target = archive if task_name == task_group else archive.parent try: completed_at = ( complete_log.stat().st_mtime_ns if complete_log.is_file() else archive.stat().st_mtime_ns ) except OSError: continue candidates.append((completed_at, str(target), target)) return ( max(candidates, key=lambda item: (item[0], item[1]))[2] if candidates else None ) def archive_completed_group_work_logs( workspace: Path, observed_tasks: set[str], completed_tasks: dict[str, str], active_or_running: set[str], ) -> tuple[dict[str, str], dict[str, str]]: """Archive each completed task-group timeline after its last writer exits.""" observed_by_group: dict[str, set[str]] = {} for task_name in observed_tasks: observed_by_group.setdefault(task_group_name(task_name), set()).add( task_name ) active_groups = { task_group_name(task_name) for task_name in active_or_running } archived: dict[str, str] = {} errors: dict[str, str] = {} for task_group, task_names in sorted(observed_by_group.items()): if task_group in active_groups or not task_names <= set(completed_tasks): continue active_source = ( workspace / "agent-task" / task_group / WORK_LOG_NAME ) legacy_sources = { Path(completed_tasks[task_name]) / WORK_LOG_NAME for task_name in task_names if (Path(completed_tasks[task_name]) / WORK_LOG_NAME).is_file() } sources = { *legacy_sources, *([active_source] if active_source.is_file() else []), } if not sources: continue target_directory = completed_group_archive_directory( task_group, task_names, completed_tasks, ) if target_directory is None: errors[task_group] = ( "검증된 task archive에서 WORK_LOG 대상 디렉터리를 정할 수 없다" ) continue archive_number = next_work_log_archive_number( workspace, task_group, ) destination = target_directory / f"work_log_{archive_number}.log" source = next(iter(sources)) if destination.exists(): errors[task_group] = ( "WORK_LOG archive destination이 이미 존재한다: " "sources=" + ",".join(str(path) for path in sorted(sources)) + f" destination={destination}" ) continue source = next(iter(sources)) temporary = destination.with_name(destination.name + ".tmp") try: if len(sources) == 1: close_unfinished_work_log_attempts(source) source.replace(destination) else: if temporary.exists(): raise OSError( "WORK_LOG archive temporary destination이 이미 존재한다: " f"temporary={temporary}" ) temporary.write_text( merge_work_log_sources(sources), encoding="utf-8", ) close_unfinished_work_log_attempts(temporary) temporary.replace(destination) for merged_source in sources: merged_source.unlink() except (OSError, ValueError) as exc: if temporary.exists(): try: temporary.unlink() except OSError: pass errors[task_group] = ( "WORK_LOG archive 실패: sources=" + ",".join(str(path) for path in sorted(sources)) + " " f"destination={destination} error={exc}" ) continue if active_source in sources: for task_name in sorted(task_names, reverse=True): if "/" not in task_name: continue try: (workspace / "agent-task" / task_name).rmdir() except OSError: pass try: active_source.parent.rmdir() except OSError: pass archived[task_group] = str(destination.resolve()) return archived, errors def task_attempt_log_directories(runs: Path, task_name: str) -> list[Path]: """Return dispatcher-owned attempt directories whose locator names the task.""" matches: list[Path] = [] if not runs.is_dir(): return matches for attempt_dir in runs.iterdir(): if not attempt_dir.is_dir(): continue locator = attempt_dir / "locator.json" try: record = json.loads(locator.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue if record.get("task") != task_name: continue matches.append(attempt_dir) return matches def cleanup_completed_task_attempt_logs(runs: Path, task_name: str) -> int: """Remove only dispatcher-owned logs for a task after its complete archive exists.""" removed = 0 for attempt_dir in task_attempt_log_directories(runs, task_name): try: shutil.rmtree(attempt_dir) except OSError as exc: attempt_event( "[attempt-log-cleanup-warning]", f"task={task_name} path={attempt_dir} error={exc}", ) continue removed += 1 return removed def review_fingerprints(workspace: Path, task: Task) -> set[tuple[str, str]]: directories = [task.directory] if task.directory.is_dir() else [] directories.extend(matching_archive_directories(workspace, task)) fingerprints: set[tuple[str, str]] = set() for directory in directories: for path in directory.iterdir(): if path.is_file() and (path.name.startswith("code_review_") or REVIEW_RE.match(path.name)): fingerprints.add((str(path.resolve()), sha256_file(path))) return fingerprints def review_outcome( workspace: Path, task: Task, prior_fingerprints: set[tuple[str, str]] ) -> dict[str, str]: if task.directory.is_dir(): directories = [task.directory] archives: list[Path] = [] else: archives = matching_archive_directories(workspace, task) directories = list(archives) newest_directory: Path | None = None newest_log: Path | None = None newest_mtime = -1 for directory in directories: logs = list(directory.glob("code_review_*.log")) logs.extend(path for path in directory.iterdir() if path.is_file() and REVIEW_RE.match(path.name)) for log in logs: mtime = log.stat().st_mtime_ns fingerprint = (str(log.resolve()), sha256_file(log)) if fingerprint not in prior_fingerprints and mtime > newest_mtime and read_verdict(log): newest_directory = directory newest_log = log newest_mtime = mtime verdict = read_verdict(newest_log) if newest_log else "UNKNOWN" if newest_directory in archives: state = "archived" elif newest_directory and (newest_directory / "USER_REVIEW.md").exists(): state = "user-review" elif newest_directory and (newest_directory / "complete.log").exists(): state = "complete-finalization" elif newest_log and REVIEW_RE.match(newest_log.name): state = "finalization-pending" elif newest_directory and any(REVIEW_RE.match(path.name) for path in newest_directory.iterdir() if path.is_file()): state = "follow-up" else: state = "changed" return { "verdict": verdict or "UNKNOWN", "state": state, "path": str(newest_directory or task.directory), "review_log": str(newest_log) if newest_log else "unknown", } async def run_worker( workspace: Path, store: StateStore, task: Task, resume_locator: Path | None = None, quota_snapshot: dict[str, Any] | None = None, ) -> None: retry_context = store.task_state(task).get("retry_quota_refresh_context") if resume_locator is None and isinstance(retry_context, dict): locator_value = retry_context.get("locator") if isinstance(locator_value, str) and locator_value: resume_locator = Path(locator_value) # If an active_locator already exists from a prior attempt that wrote its # locator but crashed before consuming the pending handoff, consume it now # to prevent a duplicate invocation. The locator write is the durable # commitment; the pending handoff is the logical intent. Consume the intent # when the commitment is already present. if isinstance(store, StateStore): prior_state = store.task_state(task) prior_active = prior_state.get("active_locator") prior_pending = prior_state.get("retry_quota_refresh_pending") if prior_active and prior_pending: consumed = False # Prefer handoff_id matching: read the stable identity from the # active locator file so we can match across crash boundaries # where the locator path changes. try: prior_locator_data = json.loads(Path(prior_active).read_text(encoding="utf-8")) if isinstance(prior_locator_data, dict): handoff_id = prior_locator_data.get("retry_handoff_id") if handoff_id: consumed = store.commit_retry_handoff_locator( task, handoff_id, prior_active, ) except (OSError, json.JSONDecodeError): pass if not consumed: # Fallback: match by locator path when handoff_id is # unavailable (e.g. crash between state update and locator # write, or pre-existing state from a prior dispatcher # version). consumed = store.consume_matching_retry_handoff(task, prior_active) try: decision, spec = persisted_execution_decision( store, task, stage="worker", quota_snapshot=quota_snapshot ) except ExecutionDecisionError as exc: store.update_task(task, blocked=str(exc)) banner("작업차단", task.name, [f"reason={exc}"]) return work_log = milestone_work_log_path(task) banner( "작업시작", task.name, [ f"model={spec.display}", f"plan={task.plan.resolve()}", f"work_log={work_log.resolve()}", ], ) success, locator = await run_escalating( workspace, store, task, "worker", spec, initial_resume_locator=resume_locator, ) if not success: current = store.task_state(task).get("blocked") store.update_task( task, blocked=current or f"worker failure locator={locator}" ) return completed_spec = agent_spec_from_locator(locator) or spec try: _mark_worker_done( store, task, initial_decision=decision, worker_cli=completed_spec.cli, worker_model=completed_spec.model, ) except ExecutionDecisionError as exc: store.update_task(task, worker_done=False, blocked=f"worker completion validation failed: {exc}") banner("작업차단", task.name, [f"reason={exc}"]) return def _require_same_runtime_identity( expected_spec: AgentSpec, worker_cli: str, worker_model: str, ) -> None: """Verify the worker CLI/model matches the validated completing decision spec. Ensures the actual worker that ran is the same runtime identity that the completing decision authorizes. Prevents a cloud-completed worker from being recorded as a Pi selfcheck target or vice versa. """ if expected_spec.cli != worker_cli: raise ExecutionDecisionError( f"worker runtime CLI 불일치: expected={expected_spec.cli} actual={worker_cli}" ) if expected_spec.model != worker_model: raise ExecutionDecisionError( f"worker runtime model 불일치: expected={expected_spec.model} actual={worker_model}" ) def _mark_worker_done( store: StateStore, task: Task, *, initial_decision: dict[str, Any], worker_cli: str, worker_model: str, ) -> None: """Persist worker completion with the authoritative completing decision. Uses the persisted execution_decisions worker entry as the sole authoritative source. Does not fall back to initial_decision even when the persisted decision is malformed—malformed persisted state blocks completion rather than silently reverting to a speculative initial decision. Validates the completing decision through the strict contract validator and verifies the worker CLI/model identity matches the normalized spec. On any validation failure, raises ExecutionDecisionError to prevent worker_done from being recorded. """ decisions = store.task_state(task).get("execution_decisions", {}) if not isinstance(decisions, dict) or "worker" not in decisions: raise ExecutionDecisionError( "persisted worker decision이 execution_decisions에 없다" ) decision = decisions["worker"] if not isinstance(decision, dict): raise ExecutionDecisionError( "persisted worker decision이 dict가 아니다" ) validated_decision, expected_spec = _validated_completing_decision( task, decision ) _require_same_runtime_identity(expected_spec, worker_cli, worker_model) execution_class = validated_decision["selected"]["execution_class"] store.update_task( task, worker_done=True, worker_cli=worker_cli, worker_model=worker_model, completing_decision=validated_decision, execution_class=execution_class, selfcheck_done=(execution_class == "cloud_model"), blocked=None, ) async def run_selfcheck( workspace: Path, store: StateStore, task: Task, resume_locator: Path | None = None, ) -> None: completing = store.task_state(task).get("completing_decision") if not isinstance(completing, dict): store.update_task( task, blocked="completing decision이 없어 selfcheck를 실행할 수 없다" ) banner( "작업차단", task.name, ["reason=missing-completing-decision"], ) return try: _completed_decision, spec = _validated_completing_decision(task, completing) except ExecutionDecisionError as exc: store.update_task(task, blocked=str(exc)) banner("작업차단", task.name, [f"reason={exc}"]) return if not spec.local_pi: raise RuntimeError("Pi가 아닌 route에 selfcheck stage가 배정됐다") work_log = milestone_work_log_path(task) banner( "자가검증시작", task.name, [ f"model={spec.display}", f"plan={task.plan.resolve()}", f"work_log={work_log.resolve()}", ], ) retry = 0 if isinstance(store, StateStore): retry = int(store.task_state(task).get("selfcheck_incomplete", 0)) if retry >= SELF_CHECK_INCOMPLETE_LIMIT: locator = resume_locator reason = ( "selfcheck implementation field limit already exhausted: " f"{retry}/{SELF_CHECK_INCOMPLETE_LIMIT}" ) store.update_task(task, blocked=f"{reason} locator={locator}") banner( "작업차단", task.name, [ "reason=selfcheck-incomplete-limit", f"retry={retry}/{SELF_CHECK_INCOMPLETE_LIMIT}", f"locator={locator}", ], ) return while True: success, locator = await run_escalating( workspace, store, task, "selfcheck", spec, initial_resume_locator=resume_locator, ) resume_locator = None if not success: current = store.task_state(task).get("blocked") store.update_task( task, blocked=current or f"selfcheck failure locator={locator}" ) return errors = implementation_review_errors(task) if not errors: break retry += 1 if retry >= SELF_CHECK_INCOMPLETE_LIMIT: reason = ( "selfcheck implementation fields remain incomplete: " f"{retry}/{SELF_CHECK_INCOMPLETE_LIMIT}" ) store.update_task( task, blocked=f"{reason} locator={locator}", selfcheck_incomplete=retry, ) banner( "작업차단", task.name, [ "reason=selfcheck-incomplete-limit", f"detail={'; '.join(errors)}", f"retry={retry}/{SELF_CHECK_INCOMPLETE_LIMIT}", f"locator={locator}", ], ) return store.update_task(task, selfcheck_incomplete=retry) banner( "자가검증재시도", task.name, [ f"reason={'; '.join(errors)}", f"retry={retry}/{SELF_CHECK_INCOMPLETE_LIMIT}", f"locator={locator}", ], ) store.update_task( task, selfcheck_done=True, selfcheck_incomplete=0, blocked=None, ) async def run_review( workspace: Path, store: StateStore, task: Task, resume_locator: Path | None = None, quota_snapshot: dict[str, Any] | None = None, ) -> str | None: try: _, spec = persisted_execution_decision( store, task, stage="review", quota_snapshot=quota_snapshot ) except ExecutionDecisionError as exc: store.update_task(task, blocked=str(exc)) banner("작업차단", task.name, [f"reason={exc}"]) return None state = store.task_state(task) prior_no_progress = int(state.get("review_no_progress", 0)) if prior_no_progress >= REVIEW_NO_PROGRESS_LIMIT: locator = state.get("active_locator") reason = ( "review no-progress limit already exhausted: " f"{prior_no_progress}/{REVIEW_NO_PROGRESS_LIMIT}" ) store.update_task(task, blocked=f"{reason} locator={locator}") banner( "작업차단", task.name, [ "reason=review-no-progress-limit", f"unchanged_review_attempts={prior_no_progress}/{REVIEW_NO_PROGRESS_LIMIT}", f"locator={locator}", ], ) return None before = task_signature(workspace, task) prior_review_fingerprints = review_fingerprints(workspace, task) target = task.review.resolve() if task.review else task.directory.resolve() banner("리뷰시작", task.name, [f"model={spec.display}", f"review={target}"]) success, locator = await run_escalating( workspace, store, task, "review", spec, initial_resume_locator=resume_locator, ) if not success: current = store.task_state(task).get("blocked") store.update_task( task, blocked=current or f"review failure locator={locator}" ) return None after = task_signature(workspace, task) if before == after: state = store.task_state(task) count = int(state.get("review_no_progress", 0)) + 1 if count >= REVIEW_NO_PROGRESS_LIMIT: reason = ( "review made no progress: " f"{count}/{REVIEW_NO_PROGRESS_LIMIT} locator={locator}" ) store.update_task( task, review_no_progress=count, blocked=reason, ) banner( "작업차단", task.name, [ "reason=review-no-progress-limit", f"unchanged_review_attempts={count}/{REVIEW_NO_PROGRESS_LIMIT}", f"locator={locator}", ], ) return None store.update_task(task, review_no_progress=count) banner( "루프정체경고", task.name, [ f"unchanged_review_attempts={count}/{REVIEW_NO_PROGRESS_LIMIT}", f"locator={locator}", ], ) await asyncio.sleep(min(30, count * 5)) else: store.update_task(task, review_no_progress=0, blocked=None) outcome = review_outcome(workspace, task, prior_review_fingerprints) banner( "리뷰결과", task.name, [ f"verdict={outcome['verdict']}", f"state={outcome['state']}", f"path={outcome['path']}", f"review_log={outcome['review_log']}", f"locator={locator}", ], ) if outcome["verdict"] == "PASS" and outcome["state"] == "archived": banner("작업완료", task.name, [f"archive={outcome['path']}", f"locator={locator}"]) return outcome["path"] if outcome["verdict"] == "UNKNOWN" or outcome["state"] == "changed": raise RuntimeError( "official review가 판정과 다음 파일 상태를 materialize하지 않았다; " f"locator={locator}" ) if outcome["state"] == "archived": raise RuntimeError( f"PASS가 아닌 review가 완료 archive로 이동했다: " f"verdict={outcome['verdict']} locator={locator}" ) return None def status_lines( task: Task, stage: str, dependency: str, decision: dict[str, Any] | None = None, ) -> list[str]: route = f"{task.lane}-G{task.grade:02d}" if task.lane and task.grade else "recovery" base = [f"stage={stage}", f"route={route}", f"dependency={dependency}"] if decision is not None: return base + selector_evidence_lines(decision) return base def select_dispatch_candidates( store: StateStore, ready: list[tuple[Task, str]], *, persist: bool, available_slots: int | None = None, ) -> tuple[ list[tuple[Task, str]], list[tuple[Task, str, str]], str, ]: ready_reviews = [(task, stage) for task, stage in ready if stage == "review"] ready_workers = [(task, stage) for task, stage in ready if stage in {"worker", "selfcheck"}] ordered = ready_reviews + ready_workers claims = store.write_claim_snapshot() selected: list[tuple[Task, str]] = [] deferred: list[tuple[Task, str, str]] = [] timestamp = now_iso() for task, stage in ordered: if not task.write_set_known or not task.write_set: deferred.append( ( task, stage, "valid non-empty Modified Files Summary write claim이 필요하다", ) ) continue requested = sorted(task.write_set) invalid_path: str | None = None for raw_path in requested: path = Path(raw_path) resolved = path.resolve() try: resolved.relative_to(store.workspace) except ValueError: invalid_path = raw_path break if ( not path.is_absolute() or str(resolved) != raw_path or resolved == store.workspace ): invalid_path = raw_path break if invalid_path is not None: deferred.append( ( task, stage, f"write claim 경로가 canonical workspace file이 아니다: {invalid_path}", ) ) continue conflict: tuple[str, str] | None = None requested_set = set(requested) for owner in sorted(claims): if owner == task.name: continue other = claims[owner] if other.get("exclusive"): conflict = (owner, "") break intersection = sorted(requested_set & set(other.get("paths", []))) if intersection: conflict = (owner, intersection[0]) break if conflict is not None: owner, path = conflict deferred.append( ( task, stage, f"write claim 충돌 대기: owner={owner}; path={path}", ) ) continue # Capacity-only admission: admit and acquire/replace a claim only # while a slot remains. A newly capacity-deferred task gets a stable # wait reason and no new claim; a task that already owns its lifecycle # claim keeps it unchanged while waiting. if available_slots is not None and len(selected) >= available_slots: if task.name in claims: deferred.append( ( task, stage, f"capacity waiting: limit reached (selected={len(selected)}/{available_slots})", ) ) else: deferred.append( ( task, stage, f"capacity waiting: limit reached (selected={len(selected)}/{available_slots})", ) ) continue previous = claims.get(task.name, {}) claims[task.name] = { "task": task.name, "plan_hash": task.plan_hash, "paths": requested, "exclusive": False, "workspace_id": store.workspace_id, "acquired_at": previous.get("acquired_at") or timestamp, "updated_at": timestamp, "source": "plan", } selected.append((task, stage)) if persist: store.replace_write_claims(claims, persist=True) return selected, deferred, "" def ensure_review_shared_state(workspace: Path) -> None: helper = workspace / "agent-ops" / "bin" / "ai-ignore.sh" if not helper.is_file(): raise RuntimeError(f"review shared-state helper가 없다: {helper}") command = [ "bash", "-c", 'source "$1" && agent_ops_ensure_gitignore_task_artifact_block "$2"', "agent-task-review-preflight", str(helper), str(workspace / ".gitignore"), ] completed = subprocess.run( command, cwd=workspace, capture_output=True, text=True, check=False, ) if completed.returncode != 0: diagnostic = (completed.stderr or completed.stdout).strip() raise RuntimeError( f"review shared-state preflight 실패: {diagnostic or completed.returncode}" ) async def dispatch(args: argparse.Namespace) -> int: workspace = Path(args.workspace).resolve() store = StateStore(workspace) try: try: return await dispatch_with_store(args, workspace, store) except Exception as exc: # A scheduler/control-plane exception must not make asyncio.run() # cancel already-running agent attempts. Keep this loop alive until # every owned background task finishes naturally; the next # dispatcher run reconciles their file/state results. current = asyncio.current_task() active = [ task for task in asyncio.all_tasks() if task is not current and not task.done() ] if active: banner( "디스패처복구대기", args.task_group or "agent-task", [ f"running_async_tasks={len(active)}", "scheduler 예외와 무관하게 실행 중 agent를 자연 종료까지 추적", ], ) await asyncio.gather(*active, return_exceptions=True) raise DispatcherInterruptedWithActiveWork( f"running agent가 있던 scheduler 예외: {exc}" ) from exc raise finally: store.close() async def dispatch_with_store( args: argparse.Namespace, workspace: Path, store: StateStore, ) -> int: orchestration_scope = args.task_group or "__all__" if args.retry_blocked and not args.dry_run: store.mark_retry_quota_refresh(args.task_group) running: dict[str, asyncio.Task[str | None]] = {} last_wait: dict[str, str] = {} completed_tasks: dict[str, str] = {} fatal_errors: dict[str, str] = {} control_plane_errors: dict[str, str] = {} work_log_archive_errors: dict[str, str] = {} review_shared_state_ready = False candidate_scope: set[str] | None = None task_cache: dict[str, Task] | None = None resume_locators: dict[str, Path] = {} legacy_recoveries: dict[str, LegacyPromotionRecovery] = {} live_external_processes: dict[str, str] = {} capacity_waiting: set[str] = set() max_parallel = validated_max_parallel(getattr(args, "max_parallel", 0)) while True: if task_cache is None: tasks = scan_tasks(workspace, args.task_group) task_cache = {task.name: task for task in tasks} else: tasks = sorted(task_cache.values(), key=lambda task: (task.index, task.name)) if args.dry_run: persistent_errors: dict[str, str] = {} observed_tasks: set[str] = set() live_external_processes = {} else: store.prepare_orchestration(orchestration_scope, tasks, workspace) live_external_processes = orchestration_live_agent_processes( store, orchestration_scope, ) active_or_running = ( {task.name for task in tasks} | set(running) | set(live_external_processes) ) reconciled_completed, persistent_errors = store.reconcile_orchestration( orchestration_scope, workspace, active_or_running, ) completed_tasks.update(reconciled_completed) for task_name in persistent_errors: completed_tasks.pop(task_name, None) observed_tasks = store.orchestration_tasks(orchestration_scope) work_log_archives, work_log_archive_errors = ( archive_completed_group_work_logs( workspace, observed_tasks, completed_tasks, active_or_running, ) ) for task_group, archive in sorted(work_log_archives.items()): banner( "작업로그아카이브", task_group, [f"archive={archive}"], ) if not tasks and not running: if live_external_processes: for task_name, detail in sorted( live_external_processes.items() ): banner( "작업수행중", task_name, [ "이전 dispatcher의 model process를 종료시키지 않고 추적", detail, ], ) await asyncio.sleep(STREAM_HEARTBEAT_SECONDS) continue if control_plane_errors: banner( "디스패치추적대기", args.task_group or "agent-task", [ "예상하지 못한 dispatcher 중단 결과를 재조정해야 함", *( f"interrupted[{name}]={reason}" for name, reason in sorted(control_plane_errors.items()) ), ], ) return 3 if work_log_archive_errors: banner( "디스패치추적대기", args.task_group or "agent-task", [ "완료 task group의 WORK_LOG archive를 재시도해야 함", *( f"work-log-archive[{group}]={reason}" for group, reason in sorted( work_log_archive_errors.items() ) ), ], ) return 3 if args.task_group and not observed_tasks and not completed_tasks: reason = ( "명시한 task group에서 관찰된 active task나 " "검증된 complete.log 이력이 없다" ) if not args.dry_run: store.mark_orchestration_blocked(orchestration_scope, {}) banner( "디스패치차단", args.task_group, [f"reason=unobserved-task-group", reason], ) return 2 pending_attempt_logs = { name: paths for name in completed_tasks if ( paths := task_attempt_log_directories( store.runs, name, ) ) } if pending_attempt_logs: banner( "디스패치추적대기", args.task_group or "agent-task", [ "완료 task의 attempt 로그 정리가 아직 끝나지 않음", *( f"attempt-log-cleanup-pending[{name}]=" + ",".join(str(path) for path in paths) for name, paths in sorted(pending_attempt_logs.items()) ), ], ) return 3 incomplete = sorted(observed_tasks - completed_tasks.keys()) if incomplete or fatal_errors or persistent_errors: details = [ *(f"incomplete={name}" for name in incomplete), *( f"persistent[{name}]={reason}" for name, reason in sorted(persistent_errors.items()) ), *(f"error[{name}]={reason}" for name, reason in sorted(fatal_errors.items())), ] if not args.dry_run: store.mark_orchestration_blocked( orchestration_scope, { name: ( "blocked", persistent_errors.get(name) or fatal_errors.get(name) or "관찰된 task가 완료되지 않았다", ) for name in ( set(incomplete) | set(fatal_errors) | set(persistent_errors) ) }, ) banner("디스패치차단", args.task_group or "agent-task", details) return 2 if not args.dry_run: store.mark_orchestration_complete(orchestration_scope) banner( "작업완료", args.task_group or "agent-task", [ "active task 없음", f"verified_complete_tasks={len(completed_tasks)}", *(f"complete[{name}]={path}" for name, path in sorted(completed_tasks.items())), ], ) return 0 task_by_name = {task.name: task for task in tasks} finished_names: set[str] = set() complete_log_created = False for name, future in list(running.items()): if not future.done(): continue finished_names.add(name) completed_archive: str | None = None try: completed_archive = future.result() if completed_archive: complete_log_created = True store.mark_orchestration_task_complete( orchestration_scope, name, completed_archive ) completed_tasks[name] = completed_archive except Exception as exc: # keep other independent tasks alive banner( "디스패치추적대기", name, [f"agent coroutine exception={exc}"], ) control_plane_errors[name] = str(exc) task = task_by_name.get(name) if task: store.clear_active(task) if not completed_archive: refreshed = read_task_directory(workspace, task.directory) if refreshed is None: task_cache.pop(name, None) else: task_cache[name] = refreshed if completed_archive: task_cache.pop(name, None) del running[name] if finished_names: if complete_log_created: # Running reviewers may be archiving their own active directory # while this completion-triggered scan runs. Preserve their # already-loaded Task snapshots and do not reread those mutable # directories until their futures finish. running_snapshots = { name: task_cache[name] for name in running if name in task_cache } tasks = scan_tasks( workspace, args.task_group, exclude_names=set(running), ) task_cache = { **{task.name: task for task in tasks}, **running_snapshots, } tasks = sorted( task_cache.values(), key=lambda task: (task.index, task.name), ) store.prepare_orchestration(orchestration_scope, tasks, workspace) live_external_processes = orchestration_live_agent_processes( store, orchestration_scope, ) active_or_running = ( {task.name for task in tasks} | set(running) | set(live_external_processes) ) reconciled_completed, persistent_errors = store.reconcile_orchestration( orchestration_scope, workspace, active_or_running, ) completed_tasks.update(reconciled_completed) for task_name in persistent_errors: completed_tasks.pop(task_name, None) observed_tasks = store.orchestration_tasks(orchestration_scope) work_log_archives, work_log_archive_errors = ( archive_completed_group_work_logs( workspace, observed_tasks, completed_tasks, active_or_running, ) ) for task_group, archive in sorted( work_log_archives.items() ): banner( "작업로그아카이브", task_group, [f"archive={archive}"], ) candidate_scope = None else: tasks = sorted(task_cache.values(), key=lambda task: (task.index, task.name)) candidate_scope = finished_names | capacity_waiting capacity_waiting = set() if not tasks and not running: # A completion-triggered full scan may have removed the last active task. continue # Derive workspace-global capacity. Count unique task names across # current running futures and same-workspace live/conservative evidence, # regardless of --task-group. Do not count pump/heartbeat/selector/ # quota-probe coroutines as extra slots. workspace_live = workspace_live_agent_processes(store) workspace_live = { name: detail for name, detail in workspace_live.items() if name not in finished_names } occupied_names = set(running) | set(workspace_live) available_slots: int | None = ( None if max_parallel == 0 else max(0, max_parallel - len(occupied_names)) ) ready: list[tuple[Task, str]] = [] waiting_tasks: list[str] = [] externally_active: list[tuple[Task, str]] = [] blocked_details: dict[str, tuple[str, str, str]] = {} for task in tasks: # A running future owns this task directory. Re-reading its review # or dependency files can race with review finalization/archive and # must never interrupt unrelated tasks. if task.name in running: continue if task.name in control_plane_errors: reason = ( "예상하지 못한 agent coroutine 중단 결과를 " "다음 dispatcher가 재조정해야 함: " f"{control_plane_errors[task.name]}" ) blocked_details[task.name] = ( "디스패치추적대기", "interrupted", reason, ) waiting_tasks.append(task.name) continue state = store.peek_task_state(task) if args.dry_run else store.task_state(task) legacy_recovery: LegacyPromotionRecovery | None = None legacy_blocker_reclassified = False if state.get("blocked"): legacy_recovery = legacy_promotion_recovery( store.runs, task, state, ) legacy_blocker_reclassified = legacy_recovery is not None else: legacy_recovery = ( pending_persisted_legacy_promotion_recovery(task, state) ) if legacy_recovery is not None: legacy_recoveries[task.name] = legacy_recovery resume_locators[task.name] = legacy_recovery.locator if legacy_blocker_reclassified: state = dict(state) state["blocked"] = None if not args.dry_run: recovery_failures = dict( state.get("recovery_failures", {}) ) # Ten identical generic retries represent one terminal # quota/context/model failure after reclassification. recovery_failures[legacy_recovery.role] = 1 store.update_task( task, blocked=None, recovery_failures=recovery_failures, legacy_terminal_reclassification={ "role": legacy_recovery.role, "failure_class": legacy_recovery.failure_class, "evidence_source": legacy_recovery.evidence_source, "prior_dispatcher_sha256": legacy_recovery.prior_dispatcher_sha256, "current_dispatcher_sha256": DISPATCHER_SOURCE_SHA256, "locator": str(legacy_recovery.locator), "failed_cli": legacy_recovery.failed_cli, "failed_model": legacy_recovery.failed_model, "failed_reasoning_effort": legacy_recovery.failed_reasoning_effort, }, ) state = store.task_state(task) active_predecessors = live_predecessors( task, set(running) | set(live_external_processes), ) if active_predecessors: dependency_ready = False dependency = ( "predecessor FINISH 대기: " + ",".join(active_predecessors) ) else: dependency_ready, dependency = dependency_state( workspace, task, ) stage = task_stage(task, state) if state.get("active_stage"): active_stage = str(state["active_stage"]) active_live, active_detail = external_active_is_live( state, expected_workspace=store.workspace, expected_workspace_id=store.workspace_id, expected_runs_root=store.runs, ) if active_live: reason = f"외부 실행중: stage={active_stage}; {active_detail}" active_key = ( f"active|{active_stage}|{state.get('active_locator') or 'unknown'}" ) externally_active.append((task, active_stage)) blocked_details[task.name] = ("작업중", stage, reason) if not args.dry_run: store.adopt_active_write_claim(task) if not args.dry_run and last_wait.get(task.name) != active_key: banner("작업중", task.name, status_lines(task, stage, reason)) last_wait[task.name] = active_key continue if not args.dry_run: resume_locator = laguna_resume_locator( state, expected_workspace=store.workspace, expected_workspace_id=store.workspace_id, expected_runs_root=store.runs, ) if resume_locator is not None: resume_locators[task.name] = resume_locator banner( "작업복구", task.name, status_lines(task, stage, f"stale active 제외: {active_detail}"), ) store.clear_active(task) state = store.task_state(task) reason = "" if task.errors: reason = "; ".join(task.errors) elif task.user_review: blocking, detail = user_review_blocker_state(task.user_review) if task.plan is not None or task.review is not None: blocking = False detail = "active PLAN/CODE_REVIEW와 공존한다" reason = ( f"USER_REVIEW 대기: {task.user_review}; {detail}" if blocking else ( "USER_REVIEW stop 계약 불충족: " f"{task.user_review}; {detail}" ) ) elif state.get("blocked"): reason = str(state["blocked"]) elif not dependency_ready: reason = dependency if reason: wait_key = f"{stage}|{reason}" event = ( "작업차단" if ( task.errors or stage in {"blocked", "user-review"} or state.get("blocked") ) else "작업대기" ) blocked_details[task.name] = (event, stage, reason) if not args.dry_run and last_wait.get(task.name) != wait_key: banner(event, task.name, status_lines(task, stage, reason)) last_wait[task.name] = wait_key waiting_tasks.append(task.name) continue if task.name not in running and ( candidate_scope is None or task.name in candidate_scope ): ready.append((task, stage)) admission_time = datetime.now(KST) if args.dry_run: candidates, deferred, _ = select_dispatch_candidates( store, ready, persist=False, available_slots=available_slots, ) for task, stage, reason in deferred: event = ( "작업차단" if reason.startswith( ( "valid non-empty", "write claim 경로가", ) ) else "작업대기" ) blocked_details[task.name] = (event, stage, reason) waiting_tasks.append(task.name) ready_by_name = {task.name: stage for task, stage in candidates} batch_snapshot = build_admission_batch_snapshot( store, candidates, admission_time, ) for task in tasks: if task.name in ready_by_name: stage = ready_by_name[task.name] selector_stage = "review" if stage == "review" else "worker" preview_state = store.peek_task_state(task) decisions = preview_state.get("execution_decisions", {}) prior_decision = ( decisions.get(selector_stage) if isinstance(decisions, dict) else None ) quota_snapshot = ( batch_snapshot if batch_snapshot is not None else ( preview_state.get("quota_snapshot") if isinstance(preview_state.get("quota_snapshot"), dict) else None ) ) decision = read_or_preview_stage_decision( task, preview_state, stage=selector_stage, dry_run=args.dry_run, quota_snapshot=quota_snapshot, ) spec = agent_spec_from_decision(decision) legacy_recovery = legacy_recoveries.get(task.name) if legacy_recovery is not None: failed_spec = failed_spec_from_recovery( legacy_recovery ) spec = promoted_spec(failed_spec, 0) or failed_spec lines = status_lines(task, stage, "ready", decision=decision) if legacy_recovery is not None: lines.extend( [ "recovery=legacy-terminal-reclassification", f"failure_class={legacy_recovery.failure_class}", f"locator={legacy_recovery.locator}", ] ) banner( "작업대기", task.name, lines + [f"model={spec.display}"], ) else: event, stage, reason = blocked_details[task.name] banner(event, task.name, status_lines(task, stage, reason)) return 2 if waiting_tasks and not candidates else 0 candidates, deferred, _ = select_dispatch_candidates( store, ready, persist=True, available_slots=available_slots, ) batch_snapshot = build_admission_batch_snapshot(store, candidates, admission_time) for task, stage, reason in deferred: event = ( "작업차단" if reason.startswith( ( "valid non-empty", "write claim 경로가", ) ) else "작업대기" ) blocked_details[task.name] = (event, stage, reason) waiting_tasks.append(task.name) wait_key = f"{stage}|{reason}" if last_wait.get(task.name) != wait_key: banner(event, task.name, status_lines(task, stage, reason)) last_wait[task.name] = wait_key # Rebuild capacity_waiting from current capacity-only deferrals. # Dependency, blocker, invalid-write-set, and claim-collision deferrals # are not capacity waiters and rely on their existing wake-up event. if available_slots is not None: capacity_waiting = { task.name for task, stage, reason in deferred if reason.startswith("capacity waiting:") } if ( not review_shared_state_ready and any(stage == "review" for _, stage in candidates) ): try: ensure_review_shared_state(workspace) except (OSError, RuntimeError) as exc: # Shared review setup is a blocker only for reviews. It must # not prevent dependency-independent workers/selfchecks from # starting and draining in the same scheduler pass. review_removed: list[tuple[Task, str]] = [] remaining_candidates: list[tuple[Task, str]] = [] for task, stage in candidates: if stage != "review": remaining_candidates.append((task, stage)) continue reason = f"review shared-state preflight failed: {exc}" store.update_task(task, blocked=reason) fatal_errors[task.name] = reason if task.name not in waiting_tasks: waiting_tasks.append(task.name) blocked_details[task.name] = ( "작업차단", stage, reason, ) banner( "작업차단", task.name, status_lines(task, stage, reason), ) review_removed.append((task, stage)) candidates = remaining_candidates # Block every ready review that was deferred (e.g. by capacity). for task, stage, _ in deferred: if stage == "review": reason = f"review shared-state preflight failed: {exc}" store.update_task(task, blocked=reason) fatal_errors[task.name] = reason if task.name not in waiting_tasks: waiting_tasks.append(task.name) blocked_details[task.name] = ( "작업차단", stage, reason, ) banner( "작업차단", task.name, status_lines(task, stage, reason), ) # Refill freed runtime slots from disjoint non-review capacity # waiters in stable deferred order using persistent claim admission. # Reviews that had received slots retain their existing claims; # reviews that never received slots do not synthesize claims. if available_slots is not None and review_removed: freed_slots = len(review_removed) refill_inputs = [ (task, stage) for task, stage, reason in deferred if stage in {"worker", "selfcheck"} and reason.startswith("capacity waiting:") ] if refill_inputs: refilled, refill_deferred, _ = select_dispatch_candidates( store, refill_inputs, persist=True, available_slots=freed_slots, ) candidates.extend(refilled) for task, stage, reason in refill_deferred: event = ( "작업차단" if reason.startswith( ( "valid non-empty", "write claim 경로가", ) ) else "작업대기" ) blocked_details[task.name] = (event, stage, reason) wait_key = f"{stage}|{reason}" if last_wait.get(task.name) != wait_key: banner(event, task.name, status_lines(task, stage, reason)) last_wait[task.name] = wait_key capacity_waiting = { task.name for task, stage, reason in refill_deferred if reason.startswith("capacity waiting:") } else: capacity_waiting = set() batch_snapshot = build_admission_batch_snapshot( store, candidates, admission_time, ) else: review_shared_state_ready = True scheduled = False for task, stage in candidates: store.mark_active(task, stage) resume_locator = resume_locators.pop(task.name, None) state = store.task_state(task) task_snapshot = batch_snapshot if stage == "worker" and has_persisted_worker_decision(state, task) and not retry_quota_refresh_pending(state): task_snapshot = None if stage == "review": future = asyncio.create_task( run_review( workspace, store, task, **( {"quota_snapshot": task_snapshot} if task_snapshot is not None else {} ), **( {"resume_locator": resume_locator} if resume_locator is not None else {} ), ) ) elif stage == "selfcheck": future = asyncio.create_task( run_selfcheck( workspace, store, task, **( {"resume_locator": resume_locator} if resume_locator is not None else {} ), ) ) else: future = asyncio.create_task( run_worker( workspace, store, task, **( {"quota_snapshot": task_snapshot} if task_snapshot is not None else {} ), **( {"resume_locator": resume_locator} if resume_locator is not None else {} ), ) ) running[task.name] = future last_wait.pop(task.name, None) scheduled = True if running: await asyncio.wait(running.values(), return_when=asyncio.FIRST_COMPLETED) continue if work_log_archive_errors: banner( "디스패치추적대기", args.task_group or "agent-task", [ "완료 task group의 WORK_LOG archive를 재시도해야 함", *( f"work-log-archive[{group}]={reason}" for group, reason in sorted( work_log_archive_errors.items() ) ), ], ) return 3 if externally_active: banner( "디스패치추적대기", args.task_group or "agent-task", ["새 실행 후보 없음", "active task는 caller가 계속 추적"], ) return 3 if control_plane_errors: banner( "디스패치추적대기", args.task_group or "agent-task", [ "실행 중이던 독립 작업을 모두 소진했고 재조정이 필요함", *( f"interrupted[{name}]={reason}" for name, reason in sorted(control_plane_errors.items()) ), ], ) return 3 # Capacity-only wait: external live attempts fill the cap, but we must # not mark the orchestration blocked. Report as non-terminal tracking # state and return 3 so a restart with a larger limit or after occupancy # drops can resume naturally. if capacity_waiting and not scheduled: external_fillers = set(occupied_names) - set(running) if external_fillers: banner( "디스패치추적대기", args.task_group or "agent-task", [ f"capacity_waiting={','.join(sorted(capacity_waiting))}", f"occupied_by_external={','.join(sorted(external_fillers))}", f"max_parallel={max_parallel}", "용량 대기: 외부 실행이 용량을 채워 다음 dispatcher가 재조정", ], ) return 3 if not scheduled: store.mark_orchestration_blocked( orchestration_scope, { name: ( "blocked" if event == "작업차단" else "waiting", detail, ) for name, (event, _, detail) in blocked_details.items() }, ) reason = f"waiting={','.join(sorted(waiting_tasks))}" banner( "디스패치차단", args.task_group or "agent-task", [ "실행 가능한 독립 작업을 모두 소진함", reason, f"verified_complete_tasks={len(completed_tasks)}", *( f"complete[{name}]={path}" for name, path in sorted(completed_tasks.items()) ), *( f"{name}: stage={stage}; reason={detail}" for name, (_, stage, detail) in sorted( blocked_details.items() ) ), ], ) return 2 def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--workspace", default=".", help="repository root (default: current directory)") parser.add_argument("--task-group", help="run only agent-task/") parser.add_argument("--dry-run", action="store_true", help="classify and print without launching CLIs") parser.add_argument("--retry-blocked", action="store_true", help="clear dispatcher-local blocked state") parser.add_argument( "--max-parallel", type=int, default=0, metavar="MAX_PARALLEL", help=( "physical-workspace global cap on unique active task-stage " "attempts; 0 is unlimited (default)" ), ) parser.add_argument( "--validate-plan", metavar="PATH", help="validate one PLAN Modified Files Summary without starting the dispatcher", ) return parser.parse_args() def main() -> int: args = parse_args() try: validated_max_parallel(getattr(args, "max_parallel", 0)) except ValueError as exc: print(f"dispatcher error: {exc}", file=sys.stderr) return 2 validate_plan = getattr(args, "validate_plan", None) if validate_plan: workspace = Path(args.workspace).resolve() candidate = Path(validate_plan) if not candidate.is_absolute(): candidate = (Path.cwd() / candidate).resolve() write_set, diagnostics = inspect_write_set(candidate, workspace) if diagnostics: for diagnostic in diagnostics: print(f"plan write-set error: {diagnostic}", file=sys.stderr) return 2 for path in sorted(write_set): validation_claim(path) return 0 try: return asyncio.run(dispatch(args)) except KeyboardInterrupt: print("\n중단됨", file=sys.stderr) return 130 except DispatcherAlreadyRunning as exc: print(f"dispatcher active: {exc}", file=sys.stderr) return 3 except DispatcherTerminalStateError as exc: print(f"dispatcher error: {exc}", file=sys.stderr) return 2 except Exception as exc: # An unexpected dispatcher failure is not proof that the task group is # drained or terminal. The caller must inspect active PIDs/locators and # recover instead of treating it like exit 2. print(f"dispatcher interrupted: {exc}", file=sys.stderr) return 3 if __name__ == "__main__": raise SystemExit(main())