toki
fa62ccc4cd
Canonical English 출력과 legacy Korean 읽기 경계를 일치시켜 리뷰 복구와 dispatcher 판정이 같은 계약을 따르도록 한다.\n\nVerdict schema와 completion scan 회귀를 deterministic 테스트로 고정한다.
5933 lines
230 KiB
Python
5933 lines
230 KiB
Python
#!/usr/bin/env python3
|
|
"""Dispatch every independently-ready agent-task pair until the task group completes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import fcntl
|
|
import importlib.util
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import signal
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import uuid
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
_OBSERVATION_MODULE_NAME = "agent_task_dispatcher_observation"
|
|
|
|
|
|
def load_sibling_observation_module():
|
|
loaded = sys.modules.get(_OBSERVATION_MODULE_NAME)
|
|
if loaded is not None:
|
|
return loaded
|
|
spec = importlib.util.spec_from_file_location(
|
|
_OBSERVATION_MODULE_NAME,
|
|
Path(__file__).with_name("dispatcher_observation.py"),
|
|
)
|
|
if spec is None or spec.loader is None:
|
|
raise RuntimeError("failed to load dispatcher observation module")
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[_OBSERVATION_MODULE_NAME] = module
|
|
try:
|
|
spec.loader.exec_module(module)
|
|
except BaseException:
|
|
sys.modules.pop(_OBSERVATION_MODULE_NAME, None)
|
|
raise
|
|
return module
|
|
|
|
|
|
observation = load_sibling_observation_module()
|
|
SEP = observation.SEP
|
|
banner = observation.banner
|
|
attempt_event = observation.attempt_event
|
|
|
|
PLAN_RE = re.compile(r"^PLAN-(local|cloud)-G(0[1-9]|10)\.md$")
|
|
REVIEW_RE = re.compile(r"^CODE_REVIEW-(local|cloud)-G(0[1-9]|10)\.md$")
|
|
PLAN_LOG_RE = re.compile(
|
|
r"^plan_(local|cloud)_G(0[1-9]|10)_(0|[1-9][0-9]*)\.log$"
|
|
)
|
|
REVIEW_LOG_RE = re.compile(
|
|
r"^code_review_(local|cloud)_G(0[1-9]|10)_(0|[1-9][0-9]*)\.log$"
|
|
)
|
|
SUBTASK_RE = re.compile(r"^(?P<index>\d{2})(?:\+(?P<deps>\d{2}(?:,\d{2})*))?_[a-z0-9_]+$")
|
|
MODIFIED_FILES_HEADINGS = ("Modified Files Summary", "수정 파일 요약")
|
|
IMPLEMENTATION_CHECKLIST_HEADINGS = ("Implementation Checklist", "구현 체크리스트")
|
|
# The canonical English and legacy Korean verdict contracts are paired: a
|
|
# heading only accepts the verdict label of its own schema. Mixed pairs are not
|
|
# a documented schema and must fail closed.
|
|
CODE_REVIEW_RESULT_SCHEMAS = (
|
|
("Code Review Result", "Overall Verdict"),
|
|
("코드리뷰 결과", "종합 판정"),
|
|
)
|
|
VERDICT_SCHEMA_MATCHERS = tuple(
|
|
(
|
|
re.compile(rf"^##\s*{re.escape(heading)}[ \t]*$", re.MULTILINE),
|
|
re.compile(
|
|
rf"^(?:-\s*)?(?:\*\*)?{re.escape(label)}(?:\*\*)?\s*:\s*(PASS|WARN|FAIL)[ \t]*$",
|
|
re.MULTILINE,
|
|
),
|
|
re.compile(
|
|
rf"^###\s+{re.escape(label)}[ \t]*$\s*^(?:\*\*)?(PASS|WARN|FAIL)(?:\*\*)?[ \t]*$",
|
|
re.MULTILINE,
|
|
),
|
|
)
|
|
for heading, label in CODE_REVIEW_RESULT_SCHEMAS
|
|
)
|
|
PLAN_IDENTITY_RE = re.compile(
|
|
r"<!--\s+task=(?P<task>\S+)\s+plan=(?P<plan>\d+)\s+tag=(?P<tag>\S+)\s+-->"
|
|
)
|
|
IMPLEMENTATION_CHECKBOX_RE = re.compile(
|
|
r"^-\s+\[([^\]\r\n]*)\]", re.MULTILINE
|
|
)
|
|
WORK_LOG_NAME = "WORK_LOG.md"
|
|
WORK_LOG_ARCHIVE_RE = re.compile(r"^work_log_(\d+)\.log$")
|
|
AGENT_PROCESS_MARKER_ENV = "IOP_AGENT_TASK_EXECUTION_ID"
|
|
KST = timezone(timedelta(hours=9), name="KST")
|
|
STREAM_HEARTBEAT_SECONDS = 30
|
|
PI_MODEL_RESPONSE_STALL_SECONDS = 3 * 60
|
|
PI_SESSION_SCHEMA_VERSION = 3
|
|
RECOVERY_FAILURE_LIMIT = 10
|
|
SELF_CHECK_INCOMPLETE_LIMIT = 10
|
|
REVIEW_NO_PROGRESS_LIMIT = 10
|
|
PROVIDER_TRANSPORT_FAILURES = frozenset(
|
|
{"provider-connection", "provider-stream-disconnect"}
|
|
)
|
|
FAILURE_EVIDENCE_LIMIT = 2000
|
|
# Used only to reject a stale locator whose dispatcher and agent PIDs are both
|
|
# gone. A live process is inspected after silence; it is never killed solely by
|
|
# this fallback clock.
|
|
CODEX_STREAM_STALL_SECONDS = 5 * 60
|
|
PROMOTABLE_PATTERNS = {
|
|
"context-limit": [
|
|
r"context (?:length|window)", r"maximum context", r"prompt is too long",
|
|
r"too many tokens", r"token limit", r"exceeded.{0,40}token",
|
|
r"output (?:token )?limit", r"maximum output", r"\bmax_tokens\b",
|
|
r"response (?:is )?too long",
|
|
],
|
|
"provider-quota": [
|
|
r"rate.?limit", r"\bquota\b", r"resource[_ ]?exhausted", r"\b429\b",
|
|
r"too many requests", r"usage limit", r"capacity limit",
|
|
r"\bsession limit\b",
|
|
],
|
|
"model-unavailable": [
|
|
r"model.{0,40}(?:not found|unavailable)", r"overloaded",
|
|
r"temporarily unavailable",
|
|
],
|
|
"provider-connection": [
|
|
r"\bprovider[_ -]?tunnel[_ -]?error\b",
|
|
(
|
|
r"(?:provider|backend|/v1/chat/completions|/v1/responses)"
|
|
r".{0,160}(?:connection refused|dial tcp)"
|
|
),
|
|
],
|
|
"provider-stream-disconnect": [
|
|
r"backend connection failed during streaming request",
|
|
r"sse stream before done",
|
|
r"llama-server was unresponsive",
|
|
r"backend watchdog",
|
|
r"model will be reloaded automatically on retry",
|
|
(
|
|
r"(?:provider|backend|sse).{0,160}"
|
|
r"curl error: failure when receiving data from the peer"
|
|
),
|
|
],
|
|
}
|
|
PROMOTABLE_FAILURES = frozenset(
|
|
{"context-limit", "provider-quota", "model-unavailable"}
|
|
)
|
|
CLOUD_PROMOTION_FAILURES = PROMOTABLE_FAILURES | PROVIDER_TRANSPORT_FAILURES
|
|
QUALIFIED_FAILOVER_FAILURES = frozenset(
|
|
{"provider-quota", "context-limit", "model-unavailable", "provider-stream-disconnect"}
|
|
)
|
|
|
|
|
|
class DispatcherAlreadyRunning(RuntimeError):
|
|
"""A live dispatcher owns the workspace; this is non-terminal tracking state."""
|
|
|
|
|
|
class DispatcherTerminalStateError(RuntimeError):
|
|
"""Persistent workspace state prevents safe dispatch before work can start."""
|
|
|
|
|
|
class DispatcherInterruptedWithActiveWork(RuntimeError):
|
|
"""A control-plane error occurred after one or more agent tasks had started."""
|
|
|
|
|
|
class ExecutionDecisionError(RuntimeError):
|
|
"""A selector decision is invalid for this task and must fail closed."""
|
|
|
|
|
|
def now_iso() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def work_log_now_kst() -> str:
|
|
return datetime.now(KST).strftime("%y-%m-%d %H:%M:%S")
|
|
|
|
|
|
def sha256_file(path: Path | None) -> str:
|
|
if path is None or not path.exists():
|
|
return "none"
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(65536), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
DISPATCHER_SOURCE_PATH = Path(__file__).resolve()
|
|
DISPATCHER_SOURCE_SHA256 = sha256_file(DISPATCHER_SOURCE_PATH)
|
|
DISPATCHER_PROCESS_STARTED_AT = now_iso()
|
|
|
|
|
|
def dispatcher_source_provenance() -> dict[str, Any]:
|
|
current_sha256 = sha256_file(DISPATCHER_SOURCE_PATH)
|
|
return {
|
|
"dispatcher_pid": os.getpid(),
|
|
"dispatcher_process_start_token": process_start_token(os.getpid()),
|
|
"dispatcher_process_started_at": DISPATCHER_PROCESS_STARTED_AT,
|
|
"dispatcher_source_path": str(DISPATCHER_SOURCE_PATH),
|
|
"dispatcher_source_sha256": DISPATCHER_SOURCE_SHA256,
|
|
"dispatcher_source_current_sha256": current_sha256,
|
|
"dispatcher_source_matches_loaded": current_sha256 == DISPATCHER_SOURCE_SHA256,
|
|
}
|
|
|
|
|
|
def plan_identity(path: Path | None) -> str:
|
|
if path is None or not path.exists():
|
|
return "none"
|
|
text = path.read_text(encoding="utf-8", errors="replace")[:1024]
|
|
match = PLAN_IDENTITY_RE.search(text)
|
|
if not match:
|
|
return sha256_file(path)
|
|
identity = "\0".join(match.group(name) for name in ("task", "plan", "tag"))
|
|
return "meta:" + hashlib.sha256(identity.encode()).hexdigest()
|
|
|
|
|
|
def write_json(path: Path, value: dict[str, Any]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = path.with_suffix(path.suffix + ".tmp")
|
|
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
temporary.replace(path)
|
|
|
|
|
|
def milestone_work_log_path(task: Task) -> Path:
|
|
return (
|
|
task.directory.parent / WORK_LOG_NAME
|
|
if "/" in task.name
|
|
else task.directory / WORK_LOG_NAME
|
|
)
|
|
|
|
|
|
def append_work_log_event(
|
|
path: Path,
|
|
*,
|
|
task_name: str,
|
|
event: str,
|
|
execution_id: str,
|
|
role: str,
|
|
attempt: int,
|
|
model: str,
|
|
result: str,
|
|
locator: Path,
|
|
) -> Path:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open("a+", encoding="utf-8") as stream:
|
|
fcntl.flock(stream.fileno(), fcntl.LOCK_EX)
|
|
try:
|
|
stream.seek(0)
|
|
text = stream.read()
|
|
if not text:
|
|
stream.write(
|
|
"# Milestone Work Log\n\n"
|
|
"> Dispatcher-owned execution timeline. Workers and reviewers do not edit this file.\n\n"
|
|
"| seq | time | event | task | role | attempt | model | result | locator |\n"
|
|
"|---:|---|---|---|---|---:|---|---|---|\n"
|
|
)
|
|
sequence = 1
|
|
else:
|
|
stream.seek(0, os.SEEK_END)
|
|
if "| seq | time | event | task | role | attempt | model | result | locator |" not in text:
|
|
if not text.endswith("\n"):
|
|
stream.write("\n")
|
|
stream.write(
|
|
"\n## Dispatcher Timeline\n\n"
|
|
"> Dispatcher-owned. Workers and reviewers do not edit this section.\n\n"
|
|
"| seq | time | event | task | role | attempt | model | result | locator |\n"
|
|
"|---:|---|---|---|---|---:|---|---|---|\n"
|
|
)
|
|
sequence = 1 + max(
|
|
(
|
|
int(match.group(1))
|
|
for match in re.finditer(r"^\|\s*(\d+)\s*\|", text, re.MULTILINE)
|
|
),
|
|
default=0,
|
|
)
|
|
if not text.endswith("\n"):
|
|
stream.write("\n")
|
|
|
|
def cell(value: Any) -> str:
|
|
return str(value).replace("|", r"\|").replace("\n", " ")
|
|
|
|
stream.write(
|
|
f"| {sequence} | {work_log_now_kst()} | {cell(event)} | "
|
|
f"{cell(task_name)} | "
|
|
f"{cell(role)} | {attempt} | {cell(model)} | {cell(result)} | "
|
|
f"{cell(locator.resolve())} |\n"
|
|
)
|
|
stream.flush()
|
|
finally:
|
|
fcntl.flock(stream.fileno(), fcntl.LOCK_UN)
|
|
return path
|
|
|
|
|
|
def append_milestone_event(
|
|
task: Task,
|
|
*,
|
|
event: str,
|
|
execution_id: str,
|
|
role: str,
|
|
attempt: int,
|
|
model: str,
|
|
result: str,
|
|
locator: Path,
|
|
) -> Path:
|
|
return append_work_log_event(
|
|
milestone_work_log_path(task),
|
|
task_name=task.name,
|
|
event=event,
|
|
execution_id=execution_id,
|
|
role=role,
|
|
attempt=attempt,
|
|
model=model,
|
|
result=result,
|
|
locator=locator,
|
|
)
|
|
|
|
|
|
def safe_name(value: str) -> str:
|
|
return re.sub(r"[^A-Za-z0-9_.-]+", "__", value).strip("_") or "task"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AgentSpec:
|
|
cli: str
|
|
model: str
|
|
display: str
|
|
local_pi: bool = False
|
|
reasoning_effort: str | None = None
|
|
|
|
|
|
def effective_reasoning_effort(spec: AgentSpec) -> str | None:
|
|
if spec.cli in {"codex", "claude"}:
|
|
return spec.reasoning_effort or "xhigh"
|
|
return None
|
|
|
|
|
|
def agent_spec_from_record(record: dict[str, Any]) -> AgentSpec | None:
|
|
cli = str(record.get("cli") or "")
|
|
model = str(record.get("model") or "")
|
|
if not cli or not model:
|
|
return None
|
|
reasoning_effort = record.get("reasoning_effort")
|
|
if reasoning_effort is not None:
|
|
reasoning_effort = str(reasoning_effort)
|
|
local_pi = cli == "pi"
|
|
if cli in {"codex", "claude"}:
|
|
effort = reasoning_effort or "xhigh"
|
|
display = f"{cli}/{model} {effort}"
|
|
elif cli == "pi":
|
|
display = f"pi/iop/{model}"
|
|
else:
|
|
display = f"{cli}/{model}"
|
|
return AgentSpec(
|
|
cli,
|
|
model,
|
|
display,
|
|
local_pi=local_pi,
|
|
reasoning_effort=reasoning_effort,
|
|
)
|
|
|
|
|
|
def agent_spec_from_locator(locator: Path | None) -> AgentSpec | None:
|
|
if locator is None:
|
|
return None
|
|
try:
|
|
record = json.loads(locator.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
return None
|
|
if not isinstance(record, dict):
|
|
return None
|
|
return agent_spec_from_record(record)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PiSessionState:
|
|
phase: str
|
|
expected_tool_call_ids: tuple[str, ...] = ()
|
|
completed_tool_call_ids: tuple[str, ...] = ()
|
|
pending_tool_call_ids: tuple[str, ...] = ()
|
|
reason: str = ""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class LegacyPromotionRecovery:
|
|
locator: Path
|
|
role: str
|
|
failure_class: str
|
|
evidence: str
|
|
evidence_source: str
|
|
prior_dispatcher_sha256: str
|
|
failed_cli: str
|
|
failed_model: str
|
|
failed_reasoning_effort: str | None
|
|
|
|
|
|
def failed_spec_from_recovery(
|
|
recovery: LegacyPromotionRecovery,
|
|
) -> AgentSpec:
|
|
record = {
|
|
"cli": recovery.failed_cli,
|
|
"model": recovery.failed_model,
|
|
"reasoning_effort": recovery.failed_reasoning_effort,
|
|
}
|
|
spec = agent_spec_from_record(record)
|
|
if spec is None:
|
|
raise ValueError("legacy promotion recovery에 failed agent identity가 없다")
|
|
return spec
|
|
|
|
|
|
@dataclass
|
|
class Task:
|
|
name: str
|
|
directory: Path
|
|
plan: Path | None
|
|
review: Path | None
|
|
user_review: Path | None
|
|
recovery: bool
|
|
errors: list[str] = field(default_factory=list)
|
|
index: int = 0
|
|
deps: tuple[str, ...] = ()
|
|
write_set: set[str] = field(default_factory=set)
|
|
write_set_known: bool = False
|
|
plan_hash: str = "none"
|
|
lane: str | None = None
|
|
grade: int | None = None
|
|
|
|
|
|
def next_execution_identity(
|
|
store: StateStore,
|
|
task: Task,
|
|
role: str,
|
|
) -> tuple[int, str]:
|
|
attempt = store.next_attempt(task, role)
|
|
identity = (
|
|
f"{safe_name(task.name)}__p{plan_number(task)}__{role}__a{attempt:02d}"
|
|
)
|
|
return attempt, identity
|
|
|
|
|
|
class StateStore:
|
|
def __init__(self, workspace: Path):
|
|
workspace_id = hashlib.sha256(str(workspace).encode()).hexdigest()[:16]
|
|
git_marker = workspace / ".git"
|
|
git_directory: Path | None = None
|
|
if git_marker.is_dir():
|
|
git_directory = git_marker
|
|
elif git_marker.is_file():
|
|
marker = git_marker.read_text(encoding="utf-8", errors="replace").strip()
|
|
if marker.startswith("gitdir:"):
|
|
candidate = Path(marker.split(":", 1)[1].strip())
|
|
git_directory = candidate if candidate.is_absolute() else (workspace / candidate).resolve()
|
|
candidates = []
|
|
if git_directory is not None:
|
|
candidates.append(git_directory / "agent-task-dispatcher")
|
|
state_base = Path(os.environ.get("XDG_STATE_HOME", str(Path.home() / ".local" / "state")))
|
|
candidates.append(state_base / "agent-task-dispatcher" / workspace_id)
|
|
self.root = candidates[-1]
|
|
last_error: OSError | None = None
|
|
for candidate in candidates:
|
|
try:
|
|
candidate.mkdir(parents=True, exist_ok=True)
|
|
self.root = candidate
|
|
last_error = None
|
|
break
|
|
except OSError as exc:
|
|
last_error = exc
|
|
if last_error is not None:
|
|
raise DispatcherTerminalStateError(
|
|
f"dispatcher state 디렉터리를 만들 수 없다: {candidates}"
|
|
) from last_error
|
|
self.path = self.root / "state.json"
|
|
self.runs = self.root / "runs"
|
|
self.runs.mkdir(exist_ok=True)
|
|
self.lock_stream = (self.root / "dispatcher.lock").open("a+", encoding="utf-8")
|
|
try:
|
|
fcntl.flock(self.lock_stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError as exc:
|
|
self.lock_stream.seek(0)
|
|
owner = self.lock_stream.read().strip() or "owner metadata unavailable"
|
|
self.lock_stream.close()
|
|
raise DispatcherAlreadyRunning(
|
|
f"같은 workspace의 dispatcher가 이미 실행 중이다: "
|
|
f"{self.root}; owner={owner}"
|
|
) from exc
|
|
try:
|
|
self.lock_stream.seek(0)
|
|
self.lock_stream.truncate()
|
|
self.lock_stream.write(
|
|
json.dumps(dispatcher_source_provenance(), ensure_ascii=False) + "\n"
|
|
)
|
|
self.lock_stream.flush()
|
|
except OSError as exc:
|
|
self.lock_stream.close()
|
|
raise DispatcherTerminalStateError(
|
|
f"dispatcher lock owner metadata를 기록할 수 없다: {self.root}"
|
|
) from exc
|
|
if self.path.exists():
|
|
try:
|
|
self.data = json.loads(self.path.read_text(encoding="utf-8"))
|
|
except (json.JSONDecodeError, OSError) as exc:
|
|
self.lock_stream.close()
|
|
raise DispatcherTerminalStateError(
|
|
f"dispatcher state를 읽을 수 없다: {self.path}"
|
|
) from exc
|
|
if not isinstance(self.data, dict):
|
|
self.lock_stream.close()
|
|
raise DispatcherTerminalStateError(
|
|
f"dispatcher state가 object가 아니다: {self.path}"
|
|
)
|
|
else:
|
|
self.data = {"tasks": {}, "attempt_counters": {}}
|
|
|
|
def save(self) -> None:
|
|
write_json(self.path, self.data)
|
|
|
|
def close(self) -> None:
|
|
if not self.lock_stream.closed:
|
|
self.lock_stream.close()
|
|
|
|
def task_state(self, task: Task) -> dict[str, Any]:
|
|
tasks = self.data.setdefault("tasks", {})
|
|
current = tasks.get(task.name)
|
|
if not current or current.get("plan_hash") != task.plan_hash:
|
|
current = {
|
|
"plan_hash": task.plan_hash,
|
|
"worker_done": False,
|
|
"worker_cli": None,
|
|
"worker_model": None,
|
|
"selfcheck_done": False,
|
|
"blocked": None,
|
|
"active_stage": None,
|
|
"active_locator": None,
|
|
"review_no_progress": 0,
|
|
"selfcheck_incomplete": 0,
|
|
"recovery_failures": {},
|
|
"execution_decisions": {},
|
|
"route_transition_history": [],
|
|
"stage_failure_budgets": {},
|
|
"retry_quota_refresh_pending": False,
|
|
"retry_quota_refresh_context": None,
|
|
"blocker_evidence": None,
|
|
}
|
|
tasks[task.name] = current
|
|
self.save()
|
|
return current
|
|
|
|
def peek_task_state(self, task: Task) -> dict[str, Any]:
|
|
current = self.data.get("tasks", {}).get(task.name)
|
|
if current and current.get("plan_hash") == task.plan_hash:
|
|
return dict(current)
|
|
return {
|
|
"plan_hash": task.plan_hash,
|
|
"worker_done": False,
|
|
"worker_cli": None,
|
|
"worker_model": None,
|
|
"selfcheck_done": False,
|
|
"blocked": None,
|
|
"active_stage": None,
|
|
"active_locator": None,
|
|
"review_no_progress": 0,
|
|
"selfcheck_incomplete": 0,
|
|
"recovery_failures": {},
|
|
"execution_decisions": {},
|
|
"route_transition_history": [],
|
|
"retry_quota_refresh_pending": False,
|
|
"retry_quota_refresh_context": None,
|
|
"blocker_evidence": None,
|
|
}
|
|
|
|
def update_task(self, task: Task, **values: Any) -> None:
|
|
state = self.task_state(task)
|
|
state.update(values)
|
|
self.save()
|
|
|
|
def mark_active(self, task: Task, stage: str, locator: Path | None = None) -> None:
|
|
self.update_task(
|
|
task,
|
|
active_stage=stage,
|
|
active_locator=str(locator) if locator else None,
|
|
active_started_at=now_iso(),
|
|
)
|
|
|
|
def clear_active(self, task: Task) -> None:
|
|
self.update_task(
|
|
task,
|
|
active_stage=None,
|
|
active_locator=None,
|
|
active_started_at=None,
|
|
)
|
|
|
|
def consume_matching_retry_handoff(self, task: Task, locator_path: str) -> bool:
|
|
"""Atomically consume a pending retry handoff when a matching locator exists.
|
|
|
|
When a worker writes its locator and sets active_locator, the pending
|
|
retry_quota_refresh state must be cleared in the same transaction.
|
|
This prevents a crash window where a restart sees the pending handoff
|
|
and creates a duplicate invocation.
|
|
|
|
Returns True if the pending handoff was consumed, False if no matching
|
|
locator was found (active_locator is None or differs from locator_path).
|
|
"""
|
|
state = self.task_state(task)
|
|
active = state.get("active_locator")
|
|
if active != locator_path:
|
|
return False
|
|
pending = state.get("retry_quota_refresh_pending")
|
|
if not pending:
|
|
return False
|
|
context = state.get("retry_quota_refresh_context")
|
|
if not isinstance(context, dict):
|
|
return False
|
|
context_locator = context.get("locator")
|
|
if context_locator != locator_path:
|
|
return False
|
|
# Snapshot current state to restore on save failure. This ensures the
|
|
# crash window is not widened by a partial consume: if the save fails,
|
|
# the pending handoff remains intact both in-memory and on-disk.
|
|
pre_state = dict(state)
|
|
pre_keys = set(state.keys())
|
|
pre_values = {k: state.get(k) for k in ["retry_quota_refresh_pending", "retry_quota_refresh_context"]}
|
|
try:
|
|
self.update_task(
|
|
task,
|
|
retry_quota_refresh_pending=False,
|
|
retry_quota_refresh_context=None,
|
|
)
|
|
except Exception:
|
|
# Restore the pre-consume state on any failure.
|
|
for k, v in pre_values.items():
|
|
state[k] = v
|
|
# Restore key existence: if a key existed before, restore its value;
|
|
# if a key did not exist before, ensure it is not present.
|
|
for k in list(state.keys()):
|
|
if k not in pre_keys:
|
|
del state[k]
|
|
for k, v in pre_values.items():
|
|
if k not in state:
|
|
state[k] = v
|
|
raise
|
|
return True
|
|
|
|
def commit_retry_handoff_locator(
|
|
self, task: Task, handoff_id: str, locator_path: str,
|
|
) -> bool:
|
|
"""Atomically commit a new locator and consume a matching pending retry handoff.
|
|
|
|
This is the durable one-save transition for retry handoff. It matches
|
|
the pending handoff by stable handoff_id (not by locator path, which
|
|
changes on each attempt) and atomically updates active_locator, clears
|
|
the pending flag, and clears the context in a single save.
|
|
|
|
On save failure the pre-state is fully restored both in-memory and on
|
|
disk so the crash window is not widened.
|
|
|
|
Returns True if a matching pending handoff was consumed, False if no
|
|
pending handoff with the given handoff_id was found.
|
|
"""
|
|
state = self.task_state(task)
|
|
pending = state.get("retry_quota_refresh_pending")
|
|
if not pending:
|
|
return False
|
|
context = state.get("retry_quota_refresh_context")
|
|
if not isinstance(context, dict):
|
|
return False
|
|
if context.get("handoff_id") != handoff_id:
|
|
return False
|
|
# Snapshot current state to restore on save failure.
|
|
pre_state = dict(state)
|
|
pre_keys = set(state.keys())
|
|
pre_values = {
|
|
k: state.get(k)
|
|
for k in [
|
|
"retry_quota_refresh_pending",
|
|
"retry_quota_refresh_context",
|
|
"active_locator",
|
|
]
|
|
}
|
|
try:
|
|
self.update_task(
|
|
task,
|
|
active_locator=locator_path,
|
|
retry_quota_refresh_pending=False,
|
|
retry_quota_refresh_context=None,
|
|
)
|
|
except Exception:
|
|
for k, v in pre_values.items():
|
|
state[k] = v
|
|
for k in list(state.keys()):
|
|
if k not in pre_keys:
|
|
del state[k]
|
|
for k, v in pre_values.items():
|
|
if k not in state:
|
|
state[k] = v
|
|
raise
|
|
return True
|
|
|
|
def next_attempt(self, task: Task, role: str) -> int:
|
|
key = f"{task.name}|{task.plan_hash}|{role}"
|
|
counters = self.data.setdefault("attempt_counters", {})
|
|
number = int(counters.get(key, 0))
|
|
counters[key] = number + 1
|
|
self.save()
|
|
return number
|
|
|
|
def clear_blocked(self, task_group: str | None = None) -> None:
|
|
prefix = f"{task_group}/" if task_group else None
|
|
for task_name, value in self.data.get("tasks", {}).items():
|
|
if (
|
|
task_group is not None
|
|
and task_name != task_group
|
|
and not task_name.startswith(prefix)
|
|
):
|
|
continue
|
|
value["blocked"] = None
|
|
value["review_no_progress"] = 0
|
|
value["selfcheck_incomplete"] = 0
|
|
value["recovery_failures"] = {}
|
|
value["stage_failure_budgets"] = {}
|
|
value["retry_quota_refresh_pending"] = False
|
|
self.save()
|
|
|
|
def mark_retry_quota_refresh(self, task_group: str | None = None, workspace: Path | None = None) -> None:
|
|
prefix = f"{task_group}/" if task_group else None
|
|
for task_name, value in self.data.get("tasks", {}).items():
|
|
if (
|
|
task_group is not None
|
|
and task_name != task_group
|
|
and not task_name.startswith(prefix)
|
|
):
|
|
continue
|
|
if not value.get("blocked"):
|
|
continue
|
|
blocker_evidence = value.get("blocker_evidence") if isinstance(value.get("blocker_evidence"), dict) else {}
|
|
decisions = value.get("execution_decisions", {})
|
|
worker_decision = decisions.get("worker") if isinstance(decisions, dict) else None
|
|
role = blocker_evidence.get("role")
|
|
failure_class = blocker_evidence.get("failure_class")
|
|
locator = blocker_evidence.get("locator")
|
|
selected = blocker_evidence.get("selected")
|
|
work_unit_id = blocker_evidence.get("work_unit_id")
|
|
qualified = (
|
|
role == "worker"
|
|
and failure_class in QUALIFIED_FAILOVER_FAILURES
|
|
and isinstance(locator, str)
|
|
and locator.strip()
|
|
and isinstance(selected, dict)
|
|
and isinstance(work_unit_id, str)
|
|
and isinstance(worker_decision, dict)
|
|
and worker_decision.get("work_unit_id") == work_unit_id
|
|
)
|
|
handoff_id = str(uuid.uuid4())
|
|
retry_context = ({
|
|
"role": role,
|
|
"failure_class": failure_class,
|
|
"locator": locator,
|
|
"selected": selected,
|
|
"work_unit_id": work_unit_id,
|
|
"handoff_id": handoff_id,
|
|
} if qualified else None)
|
|
|
|
value["blocked"] = None
|
|
value["review_no_progress"] = 0
|
|
value["selfcheck_incomplete"] = 0
|
|
value["recovery_failures"] = {}
|
|
value["stage_failure_budgets"] = {}
|
|
value["retry_quota_refresh_pending"] = qualified
|
|
value["retry_quota_refresh_context"] = retry_context
|
|
value["blocker_evidence"] = None
|
|
self.save()
|
|
|
|
|
|
def prepare_orchestration(
|
|
self,
|
|
scope: str,
|
|
tasks: list[Task],
|
|
workspace: Path,
|
|
) -> None:
|
|
orchestrations = self.data.setdefault("orchestrations", {})
|
|
current = orchestrations.get(scope)
|
|
if current is None or (current.get("status") == "complete" and tasks):
|
|
current = {"status": "running", "tasks": {}}
|
|
orchestrations[scope] = current
|
|
changed = False
|
|
tracked = current.setdefault("tasks", {})
|
|
for task in tasks:
|
|
record = tracked.get(task.name)
|
|
if record is None:
|
|
tracked[task.name] = {
|
|
"status": "active",
|
|
"archive": None,
|
|
"archive_baseline": [
|
|
str(path.resolve())
|
|
for path in matching_archive_directories_by_name(
|
|
workspace,
|
|
task.name,
|
|
require_complete=False,
|
|
)
|
|
],
|
|
}
|
|
changed = True
|
|
continue
|
|
if record.get("status") != "complete" and (
|
|
record.get("status") != "active" or "reason" in record
|
|
):
|
|
record["status"] = "active"
|
|
record.pop("reason", None)
|
|
changed = True
|
|
if changed or current.get("status") != "running":
|
|
current["status"] = "running"
|
|
self.save()
|
|
|
|
def mark_orchestration_task_complete(
|
|
self,
|
|
scope: str,
|
|
task_name: str,
|
|
archive: str | Path,
|
|
) -> None:
|
|
archive_path = Path(archive).resolve()
|
|
if not archive_path.is_dir() or not (archive_path / "complete.log").is_file():
|
|
raise RuntimeError(
|
|
f"완료 archive에 complete.log가 없다: task={task_name} archive={archive_path}"
|
|
)
|
|
current = self.data.setdefault("orchestrations", {}).setdefault(
|
|
scope, {"status": "running", "tasks": {}}
|
|
)
|
|
tracked = current.setdefault("tasks", {})
|
|
record = tracked.setdefault(
|
|
task_name,
|
|
{"status": "active", "archive": None, "archive_baseline": []},
|
|
)
|
|
record.update(status="complete", archive=str(archive_path))
|
|
record.pop("reason", None)
|
|
self.save()
|
|
cleanup_completed_task_attempt_logs(self.runs, task_name)
|
|
|
|
def mark_orchestration_blocked(
|
|
self,
|
|
scope: str,
|
|
outcomes: dict[str, tuple[str, str]],
|
|
) -> None:
|
|
current = self.data.setdefault("orchestrations", {}).setdefault(
|
|
scope, {"status": "running", "tasks": {}}
|
|
)
|
|
current["status"] = "blocked"
|
|
tracked = current.setdefault("tasks", {})
|
|
for task_name, (status, reason) in outcomes.items():
|
|
record = tracked.setdefault(
|
|
task_name,
|
|
{
|
|
"status": "active",
|
|
"archive": None,
|
|
"archive_baseline": [],
|
|
},
|
|
)
|
|
if record.get("status") == "complete":
|
|
continue
|
|
record.update(status=status, reason=reason)
|
|
self.save()
|
|
|
|
def reconcile_orchestration(
|
|
self,
|
|
scope: str,
|
|
workspace: Path,
|
|
active_or_running: set[str],
|
|
) -> tuple[dict[str, str], dict[str, str]]:
|
|
current = self.data.setdefault("orchestrations", {}).setdefault(
|
|
scope, {"status": "running", "tasks": {}}
|
|
)
|
|
completed: dict[str, str] = {}
|
|
errors: dict[str, str] = {}
|
|
changed = False
|
|
for task_name, record in current.setdefault("tasks", {}).items():
|
|
if record.get("status") == "complete":
|
|
archive = str(record.get("archive") or "")
|
|
if archive and (Path(archive) / "complete.log").is_file():
|
|
completed[task_name] = archive
|
|
else:
|
|
errors[task_name] = "persisted complete archive가 유효하지 않다"
|
|
continue
|
|
if task_name in active_or_running:
|
|
continue
|
|
baseline = set(str(path) for path in record.get("archive_baseline", []))
|
|
candidates = [
|
|
path
|
|
for path in matching_archive_directories_by_name(workspace, task_name)
|
|
if str(path.resolve()) not in baseline
|
|
]
|
|
if len(candidates) == 1:
|
|
archive = str(candidates[0].resolve())
|
|
record.update(status="complete", archive=archive)
|
|
completed[task_name] = archive
|
|
changed = True
|
|
elif not candidates:
|
|
errors[task_name] = (
|
|
"관찰된 task가 active와 새 complete.log archive 모두에서 사라졌다"
|
|
)
|
|
else:
|
|
errors[task_name] = (
|
|
"새 complete.log archive가 여러 개라 완료 경로를 확정할 수 없다: "
|
|
+ ",".join(str(path) for path in candidates)
|
|
)
|
|
if changed:
|
|
self.save()
|
|
for task_name in completed:
|
|
if task_name not in active_or_running:
|
|
cleanup_completed_task_attempt_logs(self.runs, task_name)
|
|
return completed, errors
|
|
|
|
def orchestration_tasks(self, scope: str) -> set[str]:
|
|
current = self.data.get("orchestrations", {}).get(scope, {})
|
|
return set(current.get("tasks", {}))
|
|
|
|
def mark_orchestration_complete(self, scope: str) -> None:
|
|
current = self.data.setdefault("orchestrations", {}).setdefault(
|
|
scope, {"status": "running", "tasks": {}}
|
|
)
|
|
current["status"] = "complete"
|
|
self.save()
|
|
|
|
|
|
def orchestration_live_agent_processes(
|
|
store: StateStore,
|
|
scope: str,
|
|
) -> dict[str, str]:
|
|
"""Return observed tasks with live or conservatively active evidence."""
|
|
task_states = store.data.get("tasks", {})
|
|
live: dict[str, str] = {}
|
|
for task_name in store.orchestration_tasks(scope):
|
|
state = task_states.get(task_name)
|
|
if not isinstance(state, dict):
|
|
continue
|
|
is_live, detail = external_active_is_live(state)
|
|
if is_live:
|
|
live[task_name] = detail
|
|
return live
|
|
|
|
|
|
def parse_route(plan: Path | None) -> tuple[str | None, int | None]:
|
|
if plan is None:
|
|
return None, None
|
|
match = PLAN_RE.match(plan.name)
|
|
if not match:
|
|
return None, None
|
|
return match.group(1), int(match.group(2))
|
|
|
|
|
|
def parse_task_name(task_root: Path, directory: Path) -> str:
|
|
return directory.relative_to(task_root).as_posix()
|
|
|
|
|
|
def extract_write_set(plan: Path | None, workspace: Path) -> tuple[set[str], bool]:
|
|
if plan is None or not plan.exists():
|
|
return set(), False
|
|
text = plan.read_text(encoding="utf-8", errors="replace")
|
|
matches = []
|
|
for heading in MODIFIED_FILES_HEADINGS:
|
|
pattern = rf"^##\s*{re.escape(heading)}[ \t]*$([\s\S]*?)(?=^##\s|\Z)"
|
|
for m in re.finditer(pattern, text, re.MULTILINE):
|
|
matches.append(m)
|
|
if len(matches) != 1:
|
|
return set(), False
|
|
match = matches[0]
|
|
result: set[str] = set()
|
|
invalid = False
|
|
for line in match.group(1).splitlines():
|
|
if not line.lstrip().startswith("|"):
|
|
continue
|
|
cells = [cell.strip() for cell in line.strip().strip("|").split("|")]
|
|
if not cells:
|
|
continue
|
|
for value in re.findall(r"`([^`]+)`", cells[0]):
|
|
normalized = re.sub(r":\d+(?::\d+)?$", "", value.strip())
|
|
if normalized and not normalized.startswith(("http://", "https://")):
|
|
if any(character in normalized for character in "*?[]"):
|
|
invalid = True
|
|
continue
|
|
candidate = Path(normalized)
|
|
resolved = (
|
|
candidate.resolve()
|
|
if candidate.is_absolute()
|
|
else (workspace / candidate).resolve()
|
|
)
|
|
try:
|
|
resolved.relative_to(workspace)
|
|
except ValueError:
|
|
invalid = True
|
|
continue
|
|
if resolved == workspace or resolved.is_dir():
|
|
invalid = True
|
|
continue
|
|
result.add(str(resolved))
|
|
return result, bool(result) and not invalid
|
|
|
|
|
|
def latest_verdict_log(directory: Path) -> Path | None:
|
|
candidates: list[tuple[int, Path]] = []
|
|
for path in directory.glob("code_review_*.log"):
|
|
match = REVIEW_LOG_RE.fullmatch(path.name)
|
|
if match is None or not path.is_file() or read_verdict(path) is None:
|
|
continue
|
|
candidates.append((int(match.group(3)), path))
|
|
if not candidates:
|
|
return None
|
|
return max(
|
|
candidates,
|
|
key=lambda candidate: (candidate[0], candidate[1].name),
|
|
)[1]
|
|
|
|
|
|
def matching_plan_log(directory: Path, review_log: Path | None) -> Path | None:
|
|
if review_log is None:
|
|
return None
|
|
review_identity = plan_identity(review_log)
|
|
matches = [
|
|
path
|
|
for path in directory.glob("plan_*.log")
|
|
if PLAN_LOG_RE.fullmatch(path.name) is not None
|
|
and path.is_file()
|
|
and plan_identity(path) == review_identity
|
|
]
|
|
return max(matches, key=lambda path: path.stat().st_mtime_ns) if matches else None
|
|
|
|
|
|
def read_task_directory(workspace: Path, directory: Path) -> Task | None:
|
|
"""Read one already-known task directory without scanning the task group."""
|
|
task_root = workspace / "agent-task"
|
|
if not directory.is_dir():
|
|
return None
|
|
plans = sorted(p for p in directory.iterdir() if p.is_file() and PLAN_RE.match(p.name))
|
|
reviews = sorted(p for p in directory.iterdir() if p.is_file() and REVIEW_RE.match(p.name))
|
|
users = sorted(directory.glob("USER_REVIEW.md"))
|
|
complete = directory / "complete.log"
|
|
recovery_log = latest_verdict_log(directory)
|
|
if not plans and not reviews and not users and not complete.exists() and recovery_log is None:
|
|
return None
|
|
name = parse_task_name(task_root, directory)
|
|
errors: list[str] = []
|
|
if len(plans) > 1:
|
|
errors.append(f"active PLAN이 {len(plans)}개다")
|
|
if len(reviews) > 1:
|
|
errors.append(f"active CODE_REVIEW가 {len(reviews)}개다")
|
|
if len(users) > 1:
|
|
errors.append(f"USER_REVIEW가 {len(users)}개다")
|
|
if users and (plans or reviews):
|
|
errors.append("USER_REVIEW stop state와 active PLAN/CODE_REVIEW가 공존한다")
|
|
plan = plans[0] if len(plans) == 1 else None
|
|
review = reviews[0] if len(reviews) == 1 else None
|
|
recovery = complete.exists() or recovery_log is not None
|
|
if bool(plan) != bool(review) and not recovery:
|
|
errors.append("active PLAN/CODE_REVIEW pair가 불완전하다")
|
|
relative = directory.relative_to(task_root)
|
|
subtask = relative.parts[1] if len(relative.parts) == 2 else None
|
|
index = 0
|
|
deps: tuple[str, ...] = ()
|
|
if subtask:
|
|
match = SUBTASK_RE.match(subtask)
|
|
if match:
|
|
index = int(match.group("index"))
|
|
deps = tuple((match.group("deps") or "").split(",")) if match.group("deps") else ()
|
|
else:
|
|
errors.append(f"split subtask 이름이 계약과 다르다: {subtask}")
|
|
lane, grade = parse_route(plan)
|
|
recovery_plan = matching_plan_log(directory, recovery_log)
|
|
write_set_source = plan or recovery_plan
|
|
write_set, write_set_known = extract_write_set(write_set_source, workspace)
|
|
if plan is not None:
|
|
metadata = PLAN_IDENTITY_RE.search(
|
|
plan.read_text(encoding="utf-8", errors="replace")[:1024]
|
|
)
|
|
if metadata is None:
|
|
errors.append("PLAN task/plan/tag metadata를 판별할 수 없다")
|
|
elif metadata.group("task") != name:
|
|
errors.append(
|
|
f"PLAN task metadata가 디렉터리와 다르다: {metadata.group('task')}"
|
|
)
|
|
return Task(
|
|
name=name,
|
|
directory=directory,
|
|
plan=plan,
|
|
review=review,
|
|
user_review=users[0] if len(users) == 1 else None,
|
|
recovery=recovery,
|
|
errors=errors,
|
|
index=index,
|
|
deps=deps,
|
|
write_set=write_set,
|
|
write_set_known=write_set_known,
|
|
plan_hash=(
|
|
plan_identity(plan)
|
|
if plan
|
|
else sha256_file(
|
|
recovery_log
|
|
or (users[0] if len(users) == 1 else complete)
|
|
)
|
|
),
|
|
lane=lane,
|
|
grade=grade,
|
|
)
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
class StageFailureBudget:
|
|
"""Persistent failure counter shared by one work unit and stage."""
|
|
|
|
store: StateStore
|
|
task: Task
|
|
work_unit_id: str
|
|
stage: str
|
|
|
|
|
|
@classmethod
|
|
def from_decision(cls, store: StateStore, task: Task, decision: dict[str, Any]) -> "StageFailureBudget":
|
|
work_unit_id = decision.get("work_unit_id")
|
|
stage = decision.get("stage")
|
|
if not isinstance(work_unit_id, str) or not work_unit_id or not isinstance(stage, str) or not stage:
|
|
raise ExecutionDecisionError("stage failure budget identity가 유효하지 않다")
|
|
return cls(store, task, work_unit_id, stage)
|
|
|
|
|
|
@property
|
|
def key(self) -> str:
|
|
return f"{self.work_unit_id}|{self.stage}"
|
|
|
|
def _budgets(self) -> dict[str, Any]:
|
|
state = self.store.task_state(self.task)
|
|
budgets = state.get("stage_failure_budgets", {})
|
|
if not isinstance(budgets, dict):
|
|
raise ExecutionDecisionError("persisted stage failure budgets schema가 유효하지 않다")
|
|
return dict(budgets)
|
|
|
|
def count(self) -> int:
|
|
entry = self._budgets().get(self.key, {})
|
|
if not isinstance(entry, dict):
|
|
raise ExecutionDecisionError("persisted stage failure budget entry가 유효하지 않다")
|
|
return int(entry.get("count", 0))
|
|
|
|
def record_failure(self, *, target: dict[str, Any], transition: str) -> int:
|
|
budgets = self._budgets()
|
|
entry = dict(budgets.get(self.key, {}))
|
|
count = int(entry.get("count", 0)) + 1
|
|
entry.update(
|
|
work_unit_id=self.work_unit_id, stage=self.stage, count=count,
|
|
last_target={"adapter": target.get("adapter"), "target": target.get("target")},
|
|
last_transition=transition,
|
|
)
|
|
budgets[self.key] = entry
|
|
self.store.update_task(self.task, stage_failure_budgets=budgets)
|
|
return count
|
|
|
|
def reset_on_success(self) -> None:
|
|
budgets = self._budgets()
|
|
budgets.pop(self.key, None)
|
|
self.store.update_task(self.task, stage_failure_budgets=budgets)
|
|
|
|
|
|
def scan_tasks(
|
|
workspace: Path,
|
|
task_group: str | None,
|
|
*,
|
|
exclude_names: set[str] | None = None,
|
|
) -> list[Task]:
|
|
task_root = workspace / "agent-task"
|
|
if not task_root.is_dir():
|
|
raise DispatcherTerminalStateError(
|
|
f"agent-task 디렉터리가 없다: {task_root}"
|
|
)
|
|
directories: list[Path] = []
|
|
try:
|
|
groups = [task_root / task_group] if task_group else sorted(
|
|
p for p in task_root.iterdir() if p.is_dir() and p.name != "archive"
|
|
)
|
|
except FileNotFoundError:
|
|
return []
|
|
for group in groups:
|
|
if not group.is_dir():
|
|
continue
|
|
directories.append(group)
|
|
try:
|
|
directories.extend(sorted(p for p in group.iterdir() if p.is_dir()))
|
|
except FileNotFoundError:
|
|
continue
|
|
tasks = [
|
|
task
|
|
for directory in directories
|
|
if (
|
|
exclude_names is None
|
|
or parse_task_name(task_root, directory) not in exclude_names
|
|
)
|
|
if (task := read_task_directory(workspace, directory)) is not None
|
|
]
|
|
return sorted(tasks, key=lambda task: (task.index, task.name))
|
|
|
|
|
|
def dependency_candidates(workspace: Path, task: Task, predecessor: str) -> list[Path]:
|
|
parts = task.name.split("/")
|
|
if len(parts) != 2:
|
|
return []
|
|
group = parts[0]
|
|
task_root = workspace / "agent-task"
|
|
found: list[Path] = []
|
|
active_group = task_root / group
|
|
for pattern in (f"{predecessor}_*/complete.log", f"{predecessor}+*/complete.log"):
|
|
found.extend(active_group.glob(pattern))
|
|
archive = task_root / "archive"
|
|
if archive.is_dir():
|
|
try:
|
|
years = list(archive.iterdir())
|
|
except FileNotFoundError:
|
|
years = []
|
|
for year in years:
|
|
if not year.is_dir():
|
|
continue
|
|
try:
|
|
months = list(year.iterdir())
|
|
except FileNotFoundError:
|
|
continue
|
|
for month in months:
|
|
archived_group = month / group
|
|
if not archived_group.is_dir():
|
|
continue
|
|
for pattern in (f"{predecessor}_*/complete.log", f"{predecessor}+*/complete.log"):
|
|
found.extend(archived_group.glob(pattern))
|
|
return sorted(set(path.resolve() for path in found))
|
|
|
|
|
|
def dependency_state(workspace: Path, task: Task) -> tuple[bool, str]:
|
|
missing: list[str] = []
|
|
ambiguous: list[str] = []
|
|
for predecessor in task.deps:
|
|
candidates = dependency_candidates(workspace, task, predecessor)
|
|
if not candidates:
|
|
missing.append(predecessor)
|
|
elif len(candidates) > 1:
|
|
ambiguous.append(f"{predecessor}={','.join(str(p) for p in candidates)}")
|
|
if ambiguous:
|
|
return False, "dependency ambiguity: " + "; ".join(ambiguous)
|
|
if missing:
|
|
return False, "predecessor complete.log 대기: " + ",".join(missing)
|
|
return True, "ready"
|
|
|
|
|
|
def live_predecessors(
|
|
task: Task,
|
|
active_task_names: set[str],
|
|
) -> list[str]:
|
|
parts = task.name.split("/")
|
|
if len(parts) != 2 or not task.deps:
|
|
return []
|
|
group = parts[0]
|
|
live: list[str] = []
|
|
for predecessor in task.deps:
|
|
prefix = re.compile(rf"^{re.escape(predecessor)}(?:[+_])")
|
|
if any(
|
|
name.startswith(f"{group}/")
|
|
and prefix.match(name.split("/", 1)[1])
|
|
for name in active_task_names
|
|
):
|
|
live.append(predecessor)
|
|
return live
|
|
|
|
|
|
def _selector_module():
|
|
if "agent_task_execution_target_selector" in sys.modules:
|
|
return sys.modules["agent_task_execution_target_selector"]
|
|
path = Path(__file__).resolve().parent / "select_execution_target.py"
|
|
spec = importlib.util.spec_from_file_location("agent_task_execution_target_selector", path)
|
|
if spec is None or spec.loader is None:
|
|
raise ExecutionDecisionError(f"selector load 실패: {path}")
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _decision_file(task: Task, stage: str) -> Path:
|
|
path = task.plan if stage == "worker" else task.review
|
|
if path is None or not path.is_file():
|
|
raise ExecutionDecisionError(f"{stage} selector 입력 파일이 없다")
|
|
return path
|
|
|
|
|
|
def agent_spec_from_decision(decision: dict[str, Any]) -> AgentSpec:
|
|
selected = decision.get("selected")
|
|
if not isinstance(selected, dict):
|
|
raise ExecutionDecisionError("selector selected가 object가 아니다")
|
|
adapter, target = selected.get("adapter"), selected.get("target")
|
|
local_pi = selected.get("selfcheck_required")
|
|
execution_class = selected.get("execution_class")
|
|
if (not isinstance(adapter, str) or not isinstance(target, str) or not target
|
|
or not isinstance(local_pi, bool)
|
|
or execution_class not in {"local_model", "cloud_model"}):
|
|
raise ExecutionDecisionError("selector selected schema가 유효하지 않다")
|
|
try:
|
|
selector = _selector_module()
|
|
selector._validate_prior_decision(decision)
|
|
decision_info = decision["decision"]
|
|
evaluated_at = selector.datetime.fromisoformat(decision_info["evaluated_at"])
|
|
policy_targets = selector.policy.select_policy(
|
|
stage=decision["stage"], lane=decision["lane"], grade=decision["grade"],
|
|
evaluated_at=evaluated_at,
|
|
).candidates
|
|
selector._validate_prior_candidate_identity(
|
|
decision,
|
|
stage=decision["stage"],
|
|
lane=decision["lane"],
|
|
grade=decision["grade"],
|
|
)
|
|
canonical = selector.policy.canonical_target(adapter, target)
|
|
except Exception as exc:
|
|
raise ExecutionDecisionError(f"selector policy validation 실패: {exc}") from exc
|
|
if canonical is None or (
|
|
canonical.execution_class != execution_class
|
|
or canonical.selfcheck_required != local_pi
|
|
):
|
|
raise ExecutionDecisionError("selector selected가 canonical policy target이 아니다")
|
|
initial_keys = {(item.adapter, item.target) for item in policy_targets}
|
|
if (adapter, target) not in initial_keys:
|
|
promotion_path = decision.get("promotion_path")
|
|
if not isinstance(promotion_path, list) or len(promotion_path) < 2:
|
|
raise ExecutionDecisionError("selector promotion path가 없다")
|
|
resolved_path = []
|
|
for index, entry in enumerate(promotion_path):
|
|
if not isinstance(entry, dict):
|
|
raise ExecutionDecisionError(
|
|
f"selector promotion path[{index}]가 object가 아니다"
|
|
)
|
|
resolved = selector.policy.canonical_target(
|
|
entry.get("adapter"), entry.get("target")
|
|
)
|
|
if resolved is None:
|
|
raise ExecutionDecisionError(
|
|
f"selector promotion path[{index}] target이 canonical이 아니다"
|
|
)
|
|
resolved_path.append(resolved)
|
|
if (resolved_path[0].adapter, resolved_path[0].target) not in initial_keys:
|
|
raise ExecutionDecisionError("selector promotion path 시작 target이 잘못됐다")
|
|
if any(
|
|
selector.policy.promotion_target(previous) != current
|
|
for previous, current in zip(resolved_path, resolved_path[1:])
|
|
):
|
|
raise ExecutionDecisionError("selector promotion path 순서가 잘못됐다")
|
|
if resolved_path[-1] != canonical:
|
|
raise ExecutionDecisionError("selector promotion path tail이 selected와 다르다")
|
|
if adapter == "pi":
|
|
if not target.startswith("iop/") or not local_pi:
|
|
raise ExecutionDecisionError("Pi selector target/schema가 유효하지 않다")
|
|
return AgentSpec("pi", target.removeprefix("iop/"), f"pi/{target}", local_pi=True)
|
|
if adapter not in {"agy", "claude", "codex"} or local_pi:
|
|
raise ExecutionDecisionError(f"selector adapter/schema가 유효하지 않다: {adapter!r}")
|
|
reasoning_effort = "high" if canonical == selector.policy.CODEX_TERRA_HIGH else None
|
|
suffix = " high" if reasoning_effort == "high" else (
|
|
" xhigh" if adapter in {"claude", "codex"} else ""
|
|
)
|
|
display = f"{adapter}/{target}{suffix}"
|
|
if reasoning_effort is not None:
|
|
return AgentSpec(
|
|
adapter,
|
|
target,
|
|
display,
|
|
reasoning_effort=reasoning_effort,
|
|
)
|
|
return AgentSpec(adapter, target, display)
|
|
|
|
|
|
def _spec_from_completing_decision(decision: dict[str, Any]) -> AgentSpec:
|
|
"""Lightweight AgentSpec extraction from a persisted completing decision.
|
|
|
|
Unlike `agent_spec_from_decision`, this does not re-validate against the
|
|
selector policy. The completing decision is already authoritative evidence
|
|
of the target that succeeded, so re-running policy is unnecessary and would
|
|
defeat the purpose of pinning the selfcheck target.
|
|
"""
|
|
selected = decision.get("selected")
|
|
if not isinstance(selected, dict):
|
|
raise ExecutionDecisionError(
|
|
"completing decision selected schema가 유효하지 않다"
|
|
)
|
|
adapter = selected.get("adapter")
|
|
target = selected.get("target")
|
|
execution_class = selected.get("execution_class")
|
|
selfcheck_required = selected.get("selfcheck_required")
|
|
if not all(isinstance(value, str) and value for value in (adapter, target, execution_class)):
|
|
raise ExecutionDecisionError(
|
|
"completing decision selected의 adapter/target/execution_class는 빈 문자열이 아닌 string이어야 한다"
|
|
)
|
|
if execution_class not in {"local_model", "cloud_model"}:
|
|
raise ExecutionDecisionError(
|
|
f"completing decision execution_class이 유효하지 않다: {execution_class}"
|
|
)
|
|
if not isinstance(selfcheck_required, bool):
|
|
raise ExecutionDecisionError(
|
|
"completing decision selected.selfcheck_required must be a boolean"
|
|
)
|
|
if adapter == "pi":
|
|
if not target.startswith("iop/"):
|
|
raise ExecutionDecisionError(
|
|
f"Pi completing decision target이 iop/ prefix가 아니다: {target}"
|
|
)
|
|
if execution_class != "local_model":
|
|
raise ExecutionDecisionError(
|
|
f"Pi completing decision execution_class이 local_model이 아니다: {execution_class}"
|
|
)
|
|
if not selfcheck_required:
|
|
raise ExecutionDecisionError(
|
|
"Pi completing decision selfcheck_required가 False이다"
|
|
)
|
|
model = target.removeprefix("iop/")
|
|
display = f"pi/{target}"
|
|
return AgentSpec(adapter, model, display, local_pi=True)
|
|
if adapter not in {"agy", "claude", "codex"}:
|
|
raise ExecutionDecisionError(
|
|
f"completing decision adapter가 유효하지 않다: {adapter!r}"
|
|
)
|
|
if execution_class != "cloud_model":
|
|
raise ExecutionDecisionError(
|
|
f"cloud completing decision execution_class이 cloud_model이 아니다: {adapter}/{execution_class}"
|
|
)
|
|
if selfcheck_required:
|
|
raise ExecutionDecisionError(
|
|
f"cloud completing decision selfcheck_required가 True이다: {adapter}/{target}"
|
|
)
|
|
display = f"{adapter}/{target}"
|
|
return AgentSpec(adapter, target, display, local_pi=False)
|
|
|
|
|
|
def select_execution_decision(
|
|
task: Task, *, stage: str, prior_decision: dict[str, Any] | None = None,
|
|
quota_snapshot: dict[str, Any] | None = None,
|
|
evaluated_at: datetime | None = None,
|
|
transition: str | None = None,
|
|
failure_class: str | None = None,
|
|
) -> dict[str, Any]:
|
|
try:
|
|
selector = _selector_module()
|
|
except Exception as exc:
|
|
code = getattr(exc, "code", exc.__class__.__name__)
|
|
raise ExecutionDecisionError(
|
|
f"{stage} selector load 실패 [{code}]: {exc}"
|
|
) from exc
|
|
try:
|
|
if transition is None:
|
|
if prior_decision is not None and stage == "worker" and task.plan and task.plan.is_file():
|
|
current_id = work_unit_id_from_file(task.plan)
|
|
prior_id = prior_decision.get("work_unit_id") if isinstance(prior_decision, dict) else None
|
|
if current_id and isinstance(prior_id, str) and prior_id and prior_id != current_id:
|
|
prior_decision = None
|
|
transition = "resume" if prior_decision is not None else "initial"
|
|
return selector.select_execution_target(
|
|
_decision_file(task, stage), stage=stage,
|
|
evaluated_at=evaluated_at or datetime.now(KST),
|
|
transition=transition,
|
|
prior_decision=prior_decision,
|
|
quota_snapshot=quota_snapshot,
|
|
failure_class=failure_class,
|
|
)
|
|
except (OSError, ValueError, selector.SelectorInputError) as exc:
|
|
code = getattr(exc, "code", exc.__class__.__name__)
|
|
raise ExecutionDecisionError(
|
|
f"{stage} selector decision 실패 [{code}]: {exc}"
|
|
) from exc
|
|
|
|
|
|
def work_unit_id_from_file(path: Path) -> str | None:
|
|
if not path.is_file():
|
|
return None
|
|
try:
|
|
head = path.read_text(encoding="utf-8", errors="replace")[:1024]
|
|
match = PLAN_IDENTITY_RE.search(head)
|
|
if match:
|
|
return f"{match.group('task')}::plan-{match.group('plan')}::tag-{match.group('tag')}"
|
|
except Exception:
|
|
pass
|
|
return None
|
|
|
|
|
|
def official_review_plan_source(task: Task) -> Path:
|
|
"""Resolve the authoritative PLAN generation for an official review."""
|
|
if task.plan is not None or task.review is not None:
|
|
if task.plan is None or task.review is None:
|
|
raise ExecutionDecisionError(
|
|
"official review active PLAN/CODE_REVIEW pair가 불완전하다"
|
|
)
|
|
return task.plan
|
|
recovery_log = latest_verdict_log(task.directory)
|
|
recovery_plan = matching_plan_log(task.directory, recovery_log)
|
|
if recovery_log is None or recovery_plan is None:
|
|
raise ExecutionDecisionError(
|
|
"official review recovery의 matching archived PLAN identity를 복구할 수 없다"
|
|
)
|
|
return recovery_plan
|
|
|
|
|
|
def official_review_source_identity(task: Task) -> tuple[str, int, str]:
|
|
source = official_review_plan_source(task)
|
|
route_match = PLAN_RE.match(source.name) or PLAN_LOG_RE.match(source.name)
|
|
if route_match is None:
|
|
raise ExecutionDecisionError(
|
|
f"official review PLAN route를 복구할 수 없다: {source.name}"
|
|
)
|
|
try:
|
|
head = source.read_text(encoding="utf-8", errors="replace")[:1024]
|
|
except OSError as exc:
|
|
raise ExecutionDecisionError(
|
|
f"official review PLAN source를 읽을 수 없다: {source}"
|
|
) from exc
|
|
metadata = PLAN_IDENTITY_RE.search(head)
|
|
if metadata is None or metadata.group("task") != task.name:
|
|
raise ExecutionDecisionError(
|
|
f"official review PLAN work-unit identity를 복구할 수 없다: {source}"
|
|
)
|
|
work_unit_id = (
|
|
f"{metadata.group('task')}::plan-{metadata.group('plan')}::"
|
|
f"tag-{metadata.group('tag')}"
|
|
)
|
|
return route_match.group(1), int(route_match.group(2)), work_unit_id
|
|
|
|
|
|
def synthesized_official_review_decision(
|
|
task: Task, *, evaluated_at: datetime | None = None
|
|
) -> dict[str, Any]:
|
|
lane, grade, work_unit_id = official_review_source_identity(task)
|
|
evaluated = evaluated_at or datetime.now(KST)
|
|
if evaluated.tzinfo is None or evaluated.utcoffset() is None:
|
|
raise ExecutionDecisionError(
|
|
"official review evaluated_at이 timezone-aware가 아니다"
|
|
)
|
|
recovery_from_archive = task.plan is None and task.review is None
|
|
selector = _selector_module()
|
|
policy_decision = selector.policy.select_policy(
|
|
stage="review", lane=lane, grade=grade, evaluated_at=evaluated
|
|
)
|
|
selected_target = policy_decision.candidates[0]
|
|
target_ref = {
|
|
"adapter": selected_target.adapter,
|
|
"target": selected_target.target,
|
|
}
|
|
candidate = {
|
|
"candidate_rank": 1,
|
|
"adapter": selected_target.adapter,
|
|
"target": selected_target.target,
|
|
"execution_class": selected_target.execution_class,
|
|
"selfcheck_required": selected_target.selfcheck_required,
|
|
"quota_mode": "bounded",
|
|
"quota_status": "unknown",
|
|
"eligibility": "eligible",
|
|
"rejection_reason": None,
|
|
}
|
|
return {
|
|
"schema_version": selector.SCHEMA_VERSION,
|
|
"work_unit_id": work_unit_id,
|
|
"stage": "review",
|
|
"lane": lane,
|
|
"grade": grade,
|
|
"selected": {
|
|
"adapter": selected_target.adapter,
|
|
"target": selected_target.target,
|
|
"execution_class": selected_target.execution_class,
|
|
"selfcheck_required": selected_target.selfcheck_required,
|
|
},
|
|
"candidates": [candidate],
|
|
"decision": {
|
|
"rule_id": policy_decision.rule_id,
|
|
"policy_priority": policy_decision.policy_priority,
|
|
"reason_codes": list(policy_decision.reason_codes),
|
|
"evaluated_at": evaluated.astimezone(KST).isoformat(),
|
|
"timezone": selector.TIMEZONE_NAME,
|
|
"time_window": policy_decision.time_window,
|
|
"pinned": recovery_from_archive,
|
|
},
|
|
"quota": {
|
|
"snapshot_id": None,
|
|
"mode": "bounded",
|
|
"status": "unknown",
|
|
"source": "official_review_fixed_policy",
|
|
"checked_at": None,
|
|
"targets": [],
|
|
},
|
|
"transition": {
|
|
"previous_target": dict(target_ref) if recovery_from_archive else None,
|
|
"next_target": dict(target_ref) if recovery_from_archive else None,
|
|
"trigger": "resume" if recovery_from_archive else "initial",
|
|
"context_transfer": "none",
|
|
},
|
|
}
|
|
|
|
|
|
def read_or_preview_stage_decision(
|
|
task: Task,
|
|
state: dict[str, Any],
|
|
*,
|
|
stage: str,
|
|
dry_run: bool = False,
|
|
evaluated_at: datetime | None = None,
|
|
quota_snapshot: dict[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
decisions = state.get("execution_decisions", {}) if isinstance(state, dict) else {}
|
|
prior = decisions.get(stage) if isinstance(decisions, dict) else None
|
|
|
|
if stage == "review":
|
|
lane, grade, work_unit_id = official_review_source_identity(task)
|
|
if (
|
|
isinstance(prior, dict)
|
|
and isinstance(prior.get("decision"), dict)
|
|
and isinstance(prior.get("quota"), dict)
|
|
):
|
|
if (
|
|
prior.get("work_unit_id") != work_unit_id
|
|
or prior.get("stage") != "review"
|
|
or prior.get("lane") != lane
|
|
or prior.get("grade") != grade
|
|
):
|
|
raise ExecutionDecisionError(
|
|
"persisted official review decision이 recovery source identity/route와 다르다"
|
|
)
|
|
agent_spec_from_decision(prior)
|
|
return prior
|
|
return synthesized_official_review_decision(
|
|
task, evaluated_at=evaluated_at
|
|
)
|
|
|
|
if isinstance(prior, dict) and task.plan and task.plan.is_file():
|
|
work_unit_id = work_unit_id_from_file(task.plan)
|
|
if work_unit_id and prior.get("work_unit_id") == work_unit_id:
|
|
return prior
|
|
|
|
if quota_snapshot is None:
|
|
quota_snapshot = state.get("quota_snapshot") if isinstance(state, dict) else None
|
|
return select_execution_decision(
|
|
task,
|
|
stage=stage,
|
|
prior_decision=prior,
|
|
quota_snapshot=quota_snapshot,
|
|
evaluated_at=evaluated_at,
|
|
)
|
|
|
|
|
|
def selector_evidence_lines(decision: dict[str, Any] | None) -> list[str]:
|
|
if not isinstance(decision, dict):
|
|
return []
|
|
selected = decision.get("selected", {})
|
|
if not isinstance(selected, dict):
|
|
return []
|
|
work_unit = decision.get("work_unit_id", "none")
|
|
decision_info = decision.get("decision", {})
|
|
if not isinstance(decision_info, dict):
|
|
decision_info = {}
|
|
rule_id = decision_info.get("rule_id", decision.get("rule_id", "none"))
|
|
priority = decision_info.get(
|
|
"policy_priority", decision.get("priority", "none")
|
|
)
|
|
transition = decision.get("transition", {})
|
|
trigger = transition.get("trigger", "none") if isinstance(transition, dict) else "none"
|
|
quota = decision.get("quota", decision.get("quota_snapshot", {}))
|
|
quota_status = quota.get("status", "none") if isinstance(quota, dict) else "none"
|
|
|
|
candidates = decision.get("candidates", [])
|
|
cand_strs = []
|
|
if isinstance(candidates, list):
|
|
for c in candidates:
|
|
if isinstance(c, dict):
|
|
rank = c.get("candidate_rank", "?")
|
|
adapter = c.get("adapter", "?")
|
|
target = c.get("target", "?")
|
|
elig = c.get("eligibility", "?")
|
|
cand_strs.append(f"#{rank}:{adapter}/{target}({elig})")
|
|
|
|
reasons = decision_info.get(
|
|
"reason_codes", selected.get("reason_codes", [])
|
|
)
|
|
reason_str = ",".join(reasons) if isinstance(reasons, list) else str(reasons)
|
|
|
|
lines = [
|
|
f"work_unit_id={work_unit}",
|
|
f"rule_id={rule_id}",
|
|
f"priority={priority}",
|
|
f"transition={trigger}",
|
|
f"quota_status={quota_status}",
|
|
]
|
|
if cand_strs:
|
|
lines.append(f"candidates={';'.join(cand_strs)}")
|
|
if reason_str:
|
|
lines.append(f"reason_codes={reason_str}")
|
|
return lines
|
|
|
|
|
|
def selector_runtime_evidence(decision: dict[str, Any]) -> dict[str, Any]:
|
|
"""Return canonical selector fields persisted in runtime audit records."""
|
|
return {
|
|
"work_unit_id": decision.get("work_unit_id"),
|
|
"candidates": decision.get("candidates"),
|
|
"selected": decision.get("selected"),
|
|
"decision": decision.get("decision"),
|
|
"quota": decision.get("quota"),
|
|
"transition": decision.get("transition"),
|
|
}
|
|
|
|
|
|
def commit_execution_decision(
|
|
store: StateStore, task: Task, stage: str, decision: dict[str, Any],
|
|
quota_snapshot: dict[str, Any] | None = None,
|
|
) -> None:
|
|
state = store.task_state(task)
|
|
decisions, history = state.get("execution_decisions", {}), state.get("route_transition_history", [])
|
|
if not isinstance(decisions, dict) or not isinstance(history, list):
|
|
raise ExecutionDecisionError("persisted selector state schema가 유효하지 않다")
|
|
decisions = dict(decisions)
|
|
decisions[stage] = decision
|
|
|
|
stage_budget_count = 0
|
|
try:
|
|
stage_budget_count = StageFailureBudget.from_decision(store, task, decision).count()
|
|
except Exception:
|
|
pass
|
|
|
|
selected = decision.get("selected", {})
|
|
history_entry = {
|
|
"stage": stage,
|
|
"transition": decision.get("transition", {}).get("trigger") if isinstance(decision.get("transition"), dict) else None,
|
|
"work_unit_id": decision.get("work_unit_id"),
|
|
"candidates": decision.get("candidates"),
|
|
"selected": selected,
|
|
"decision": decision.get("decision"),
|
|
"reason_codes": decision.get("decision", {}).get("reason_codes", [])
|
|
if isinstance(decision.get("decision"), dict)
|
|
else [],
|
|
"quota": decision.get("quota"),
|
|
"stage_budget": stage_budget_count,
|
|
}
|
|
history = [*history, history_entry]
|
|
# Preserve retry handoff state whenever a pending retry is in flight.
|
|
# The retry handoff (stable handoff_id, pending flag, context) must survive
|
|
# the decision commit so that the subsequent production invoke() can read
|
|
# it, embed the handoff_id in the new locator record, and atomically
|
|
# consume the pending handoff via commit_retry_handoff_locator().
|
|
# Clearing it here would force invoke() to fall back to a generic
|
|
# active-locator update and lose the crash-safe handoff identity.
|
|
# invoke() handles consumption regardless of failover or resume transition.
|
|
is_retry_in_flight = bool(state.get("retry_quota_refresh_pending"))
|
|
update_kwargs = {
|
|
"execution_decisions": decisions,
|
|
"route_transition_history": history,
|
|
"blocked": None,
|
|
"blocker_evidence": None,
|
|
}
|
|
if not is_retry_in_flight:
|
|
update_kwargs["retry_quota_refresh_pending"] = False
|
|
update_kwargs["retry_quota_refresh_context"] = None
|
|
if quota_snapshot is not None:
|
|
update_kwargs["quota_snapshot"] = quota_snapshot
|
|
store.update_task(task, **update_kwargs)
|
|
|
|
|
|
def persisted_execution_decision(
|
|
store: StateStore, task: Task, *, stage: str,
|
|
transition: str | None = None,
|
|
failure_class: str | None = None,
|
|
evaluated_at: datetime | None = None,
|
|
quota_snapshot: dict[str, Any] | None = None,
|
|
) -> tuple[dict[str, Any], AgentSpec]:
|
|
state = store.task_state(task)
|
|
decisions = state.get("execution_decisions", {})
|
|
if not isinstance(decisions, dict):
|
|
raise ExecutionDecisionError("persisted selector state schema가 유효하지 않다")
|
|
is_retry = retry_quota_refresh_pending(state) and stage == "worker"
|
|
if quota_snapshot is None and not is_retry:
|
|
quota_snapshot = state.get("quota_snapshot")
|
|
if quota_snapshot is not None and not isinstance(quota_snapshot, dict):
|
|
raise ExecutionDecisionError("persisted quota snapshot schema가 유효하지 않다")
|
|
prior_decision = decisions.get(stage)
|
|
|
|
retry_ctx = state.get("retry_quota_refresh_context") if isinstance(state.get("retry_quota_refresh_context"), dict) else {}
|
|
|
|
if stage == "review":
|
|
decision = read_or_preview_stage_decision(
|
|
task, state, stage=stage, evaluated_at=evaluated_at, quota_snapshot=quota_snapshot
|
|
)
|
|
else:
|
|
if transition is None:
|
|
if is_retry:
|
|
transition = "failover"
|
|
failure_class = failure_class or retry_ctx.get("failure_class") or "provider-quota"
|
|
elif prior_decision is not None and stage == "worker" and task.plan and task.plan.is_file():
|
|
current_id = work_unit_id_from_file(task.plan)
|
|
prior_id = prior_decision.get("work_unit_id") if isinstance(prior_decision, dict) else None
|
|
if current_id and isinstance(prior_id, str) and prior_id and prior_id != current_id:
|
|
prior_decision = None
|
|
transition = "resume" if prior_decision is not None else "initial"
|
|
else:
|
|
transition = "resume" if prior_decision is not None else "initial"
|
|
|
|
try:
|
|
decision = select_execution_decision(
|
|
task, stage=stage, prior_decision=prior_decision,
|
|
quota_snapshot=quota_snapshot,
|
|
transition=transition,
|
|
failure_class=failure_class,
|
|
evaluated_at=evaluated_at,
|
|
)
|
|
except ExecutionDecisionError as exc:
|
|
# No persisted unused quota target is an explicit resume case. A
|
|
# failed qualified failover with a fresh snapshot must not consume
|
|
# the retry intent before its decision can commit successfully.
|
|
if is_retry and transition == "failover" and quota_snapshot is None:
|
|
decision = select_execution_decision(
|
|
task, stage=stage, prior_decision=prior_decision,
|
|
quota_snapshot=quota_snapshot,
|
|
transition="resume",
|
|
evaluated_at=evaluated_at,
|
|
)
|
|
else:
|
|
raise
|
|
|
|
spec = agent_spec_from_decision(decision)
|
|
commit_execution_decision(store, task, stage, decision, quota_snapshot=quota_snapshot)
|
|
return decision, spec
|
|
|
|
|
|
def has_persisted_worker_decision(state: dict[str, Any], task: Task | None = None) -> bool:
|
|
decisions = state.get("execution_decisions", {})
|
|
if not isinstance(decisions, dict):
|
|
return False
|
|
prior = decisions.get("worker")
|
|
if prior is None:
|
|
return False
|
|
if task is not None and task.plan and task.plan.is_file():
|
|
current_id = work_unit_id_from_file(task.plan)
|
|
if current_id and prior.get("work_unit_id") != current_id:
|
|
return False
|
|
return True
|
|
|
|
|
|
def retry_quota_refresh_pending(state: dict[str, Any]) -> bool:
|
|
return bool(state.get("retry_quota_refresh_pending"))
|
|
|
|
|
|
def derive_work_unit_quota_evidence(
|
|
decision: dict[str, Any] | None,
|
|
status: str = "exhausted",
|
|
reason: str = "confirmed_runtime_provider_quota",
|
|
) -> dict[str, Any]:
|
|
selector = _selector_module()
|
|
func = getattr(selector, "derive_work_unit_quota_evidence", None)
|
|
if func is not None:
|
|
return func(decision, status=status, reason=reason)
|
|
return {
|
|
"schema_version": "1.0",
|
|
"snapshot_id": None,
|
|
"source": "derived_work_unit_quota",
|
|
"checked_at": datetime.now(KST).isoformat(),
|
|
"targets": [],
|
|
"required_caps": [],
|
|
"reason_codes": [reason],
|
|
}
|
|
|
|
|
|
def build_admission_batch_snapshot(
|
|
store: StateStore,
|
|
ready_items: list[tuple[Task, str]],
|
|
admission_time: datetime,
|
|
quota_probe_command: str = "iop-node quota-probe",
|
|
) -> dict[str, Any] | None:
|
|
selector = _selector_module()
|
|
policy_mod = selector.policy
|
|
|
|
unique_keys = []
|
|
seen_keys = set()
|
|
|
|
for task, stage in ready_items:
|
|
if stage != "worker":
|
|
continue
|
|
state = store.peek_task_state(task)
|
|
if has_persisted_worker_decision(state, task) and not retry_quota_refresh_pending(state):
|
|
continue
|
|
|
|
is_retry = retry_quota_refresh_pending(state)
|
|
if is_retry:
|
|
decisions = state.get("execution_decisions", {})
|
|
prior = decisions.get("worker") if isinstance(decisions, dict) else None
|
|
candidates = prior.get("candidates", []) if isinstance(prior, dict) else []
|
|
used = prior.get("used_candidates", []) if isinstance(prior, dict) else []
|
|
selected = prior.get("selected") if isinstance(prior, dict) else None
|
|
used_keys = {
|
|
(entry.get("adapter"), entry.get("target"))
|
|
for entry in used
|
|
if isinstance(entry, dict)
|
|
}
|
|
if isinstance(selected, dict):
|
|
used_keys.add((selected.get("adapter"), selected.get("target")))
|
|
candidates_to_probe = [
|
|
type("Target", (), candidate)()
|
|
for candidate in candidates
|
|
if isinstance(candidate, dict)
|
|
and candidate.get("execution_class") != "local_model"
|
|
and (candidate.get("adapter"), candidate.get("target")) not in used_keys
|
|
]
|
|
else:
|
|
lane, grade = task.lane, task.grade
|
|
if not lane or not grade:
|
|
continue
|
|
try:
|
|
pol_dec = policy_mod.select_policy(
|
|
stage="worker", lane=lane, grade=grade, evaluated_at=admission_time
|
|
)
|
|
except ValueError:
|
|
continue
|
|
candidates_to_probe = pol_dec.candidates
|
|
|
|
for cand in candidates_to_probe:
|
|
if cand.execution_class == "local_model" and not is_retry:
|
|
break
|
|
spec = policy_mod.quota_probe_spec(cand)
|
|
if spec is not None:
|
|
key = (cand.adapter, cand.target, spec.command, tuple(spec.required_caps))
|
|
if key not in seen_keys:
|
|
seen_keys.add(key)
|
|
unique_keys.append(key)
|
|
|
|
if not unique_keys:
|
|
return None
|
|
|
|
batch_id = f"batch-quota-{uuid.uuid4().hex[:12]}"
|
|
batch_provider_cls = getattr(selector, "QuotaBatchProvider", None)
|
|
if batch_provider_cls is None:
|
|
return None
|
|
batch_provider = batch_provider_cls(quota_probe_command=quota_probe_command)
|
|
return batch_provider.aggregate(
|
|
snapshot_id=batch_id,
|
|
checked_at=admission_time,
|
|
keys=unique_keys,
|
|
)
|
|
|
|
|
|
def plan_number(task: Task) -> int:
|
|
if task.plan and task.plan.exists():
|
|
match = PLAN_IDENTITY_RE.search(
|
|
task.plan.read_text(encoding="utf-8", errors="replace")[:1024]
|
|
)
|
|
if match:
|
|
return int(match.group("plan"))
|
|
return 0
|
|
|
|
|
|
def completing_decision_requires_selfcheck(state: dict[str, Any]) -> bool:
|
|
completing = state.get("completing_decision")
|
|
if not isinstance(completing, dict):
|
|
return False
|
|
selected = completing.get("selected")
|
|
if not isinstance(selected, dict):
|
|
return False
|
|
return selected.get("execution_class") == "local_model"
|
|
|
|
|
|
def _validated_completing_decision(
|
|
task: Task, decision: dict[str, Any]
|
|
) -> tuple[dict[str, Any], AgentSpec]:
|
|
"""Strictly validate a completing decision against the task contract.
|
|
|
|
Enforces that the decision's stage is "worker", its work_unit_id matches
|
|
the task's PLAN identity, and its selected fields pass the canonical
|
|
adapter/class/selfcheck normalization through `_spec_from_completing_decision`.
|
|
|
|
Returns the validated decision and its normalized AgentSpec.
|
|
Raises ExecutionDecisionError on any contract violation so that callers
|
|
can fail closed rather than advancing to an inconsistent stage.
|
|
"""
|
|
if not isinstance(decision, dict):
|
|
raise ExecutionDecisionError(
|
|
"completing decision이 dict가 아니다"
|
|
)
|
|
if decision.get("stage") != "worker":
|
|
raise ExecutionDecisionError(
|
|
f"completing decision stage가 worker가 아니다: {decision.get('stage')!r}"
|
|
)
|
|
expected_work_unit_id = work_unit_id_from_file(task.plan)
|
|
if decision.get("work_unit_id") != expected_work_unit_id:
|
|
raise ExecutionDecisionError(
|
|
f"completing decision work_unit_id 불일치: "
|
|
f"persisted={decision.get('work_unit_id')!r} "
|
|
f"plan={expected_work_unit_id!r}"
|
|
)
|
|
spec = _spec_from_completing_decision(decision)
|
|
return decision, spec
|
|
|
|
|
|
def _completing_decision_is_valid(
|
|
task: Task, state: dict[str, Any]
|
|
) -> bool:
|
|
"""Check whether the persisted completing decision satisfies the task contract.
|
|
|
|
Validates stage, work_unit_id, and selected adapter/class/selfcheck
|
|
combination. Used by task_stage to prevent a worker_done state with no
|
|
authoritative completing decision from advancing to review.
|
|
"""
|
|
completing = state.get("completing_decision")
|
|
if not isinstance(completing, dict):
|
|
return False
|
|
try:
|
|
_validated_completing_decision(task, completing)
|
|
except ExecutionDecisionError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def concrete_user_review_value(value: str) -> bool:
|
|
normalized = value.strip().strip("`").strip()
|
|
normalized = re.sub(r"^-\s*", "", normalized).strip()
|
|
if not normalized or re.search(r"\{[^}]+\}|<[^>]+>", normalized):
|
|
return False
|
|
return normalized.casefold() not in {
|
|
"-",
|
|
"n/a",
|
|
"na",
|
|
"none",
|
|
"unknown",
|
|
"미정",
|
|
"없음",
|
|
"해당 없음",
|
|
}
|
|
|
|
|
|
def user_review_blocker_state(path: Path) -> tuple[bool, str]:
|
|
if not path.is_file():
|
|
return False, "파일이 없다"
|
|
try:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError as exc:
|
|
return False, f"파일을 읽을 수 없다: {exc}"
|
|
status = markdown_section(text, "상태").strip().strip("`")
|
|
if status != "USER_REVIEW":
|
|
return False, "상태가 USER_REVIEW가 아니다"
|
|
reason = markdown_section(text, "사유")
|
|
if not re.search(r"(?m)^-\s*유형:\s*milestone-lock\s*$", reason):
|
|
return False, "milestone-lock 유형이 아니다"
|
|
target = re.search(r"(?m)^-\s*연결 대상:\s*(.+?)\s*$", reason)
|
|
target_value = target.group(1) if target else ""
|
|
if (
|
|
not concrete_user_review_value(target_value)
|
|
or "agent-roadmap/" not in target_value
|
|
or "/milestones/" not in target_value
|
|
or ".md" not in target_value
|
|
):
|
|
return False, "구체적인 Milestone 연결 대상이 없다"
|
|
evidence = markdown_section(text, "차단 근거")
|
|
evidence_line = re.search(r"(?m)^-\s*차단 판단 근거:\s*(.+?)\s*$", evidence)
|
|
if evidence_line is None or not concrete_user_review_value(
|
|
evidence_line.group(1)
|
|
):
|
|
return False, "구체적인 차단 판단 근거가 없다"
|
|
decision = markdown_section(text, "연결 결정 필요")
|
|
unresolved = [
|
|
value
|
|
for value in re.findall(r"(?m)^-\s*\[\s\]\s+(.+?)\s*$", decision)
|
|
if concrete_user_review_value(value)
|
|
]
|
|
if not unresolved:
|
|
return False, "미해결 연결 결정 항목이 없다"
|
|
resume = markdown_section(text, "재개 조건")
|
|
resume_conditions = [
|
|
line
|
|
for line in resume.splitlines()
|
|
if concrete_user_review_value(line)
|
|
]
|
|
if not resume_conditions:
|
|
return False, "구체적인 재개 조건이 없다"
|
|
return True, "unresolved milestone-lock decision"
|
|
|
|
|
|
def task_stage(task: Task, state: dict[str, Any]) -> str:
|
|
if task.errors:
|
|
return "blocked"
|
|
if task.user_review:
|
|
if task.plan is not None or task.review is not None:
|
|
return "blocked"
|
|
blocking, _ = user_review_blocker_state(task.user_review)
|
|
return "user-review" if blocking else "blocked"
|
|
if task.recovery and (task.plan is None or task.review is None):
|
|
return "review"
|
|
if task.review and task.review.exists():
|
|
text = task.review.read_text(encoding="utf-8", errors="replace")
|
|
if verdict_from_text(text):
|
|
return "review"
|
|
if state.get("worker_done"):
|
|
if not _completing_decision_is_valid(task, state):
|
|
return "blocked"
|
|
if completing_decision_requires_selfcheck(state) and not state.get("selfcheck_done"):
|
|
return "selfcheck"
|
|
return "review"
|
|
return "worker"
|
|
|
|
|
|
def markdown_section(text: str, heading: str | tuple[str, ...]) -> str:
|
|
headings = (heading,) if isinstance(heading, str) else heading
|
|
matches = []
|
|
for h in headings:
|
|
for m in re.finditer(rf"^##\s*{re.escape(h)}[ \t]*$", text, re.MULTILINE):
|
|
matches.append(m)
|
|
if len(matches) != 1:
|
|
return ""
|
|
match = matches[0]
|
|
next_heading = re.search(r"^##\s+", text[match.end():], re.MULTILINE)
|
|
end = match.end() + next_heading.start() if next_heading else len(text)
|
|
return text[match.end():end].strip()
|
|
|
|
|
|
def implementation_review_errors(task: Task) -> list[str]:
|
|
if task.review is None or not task.review.is_file():
|
|
return ["CODE_REVIEW 파일 없음"]
|
|
text = task.review.read_text(encoding="utf-8", errors="replace")
|
|
checklist = markdown_section(text, IMPLEMENTATION_CHECKLIST_HEADINGS)
|
|
checkbox_values = IMPLEMENTATION_CHECKBOX_RE.findall(checklist)
|
|
if not checkbox_values or any(not value.strip() for value in checkbox_values):
|
|
return ["구현 체크리스트 미완료"]
|
|
return []
|
|
|
|
|
|
def classify_failure_with_evidence(output: str) -> tuple[str, str | None]:
|
|
lines = output.splitlines()
|
|
for category, patterns in PROMOTABLE_PATTERNS.items():
|
|
for line in reversed(lines):
|
|
lowered = line.lower()
|
|
if any(re.search(pattern, lowered, re.DOTALL) for pattern in patterns):
|
|
return category, line
|
|
return "generic-error", None
|
|
|
|
|
|
def classify_failure(output: str) -> str:
|
|
return classify_failure_with_evidence(output)[0]
|
|
|
|
|
|
def termination_signal(return_code: int) -> tuple[str, bool] | None:
|
|
signal_number: int | None = None
|
|
inferred = False
|
|
if return_code < 0:
|
|
signal_number = -return_code
|
|
elif return_code > 128:
|
|
signal_number = return_code - 128
|
|
inferred = True
|
|
if signal_number is None:
|
|
return None
|
|
try:
|
|
return signal.Signals(signal_number).name, inferred
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def failure_report_lines(failure: str, locator: Path) -> list[str]:
|
|
record: dict[str, Any] = {}
|
|
try:
|
|
record = json.loads(locator.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
pass
|
|
failure_class = str(record.get("failure_class") or failure)
|
|
source = str(record.get("failure_source") or "unverified")
|
|
provider_confirmed = bool(
|
|
record.get("provider_transport_failure_confirmed", False)
|
|
)
|
|
lines = [
|
|
f"failure_class={failure_class}",
|
|
f"failure_source={source}",
|
|
"provider_transport_failure_confirmed="
|
|
f"{str(provider_confirmed).lower()}",
|
|
]
|
|
if record.get("dispatcher_pid") is not None:
|
|
lines.append(f"dispatcher_pid={record['dispatcher_pid']}")
|
|
if record.get("agent_pid") is not None:
|
|
lines.append(f"agent_pid={record['agent_pid']}")
|
|
if record.get("dispatcher_source_sha256"):
|
|
lines.append(
|
|
f"dispatcher_source_sha256={record['dispatcher_source_sha256']}"
|
|
)
|
|
source_matches_loaded = record.get("dispatcher_source_matches_loaded")
|
|
if source_matches_loaded is not None:
|
|
lines.append(
|
|
"dispatcher_source_matches_loaded="
|
|
f"{str(bool(source_matches_loaded)).lower()}"
|
|
)
|
|
if (
|
|
source_matches_loaded is False
|
|
and record.get("dispatcher_source_current_sha256")
|
|
):
|
|
lines.append(
|
|
"dispatcher_source_current_sha256="
|
|
f"{record['dispatcher_source_current_sha256']}"
|
|
)
|
|
if provider_confirmed:
|
|
evidence_source = record.get("failure_evidence_source")
|
|
evidence = record.get("failure_evidence_excerpt")
|
|
if evidence_source:
|
|
lines.append(f"provider_evidence_source={evidence_source}")
|
|
if evidence:
|
|
rendered = str(evidence).replace("\r", r"\r").replace("\n", r"\n")
|
|
lines.append(f"provider_evidence={rendered}")
|
|
if failure_class == "session-stall":
|
|
lines.extend(
|
|
[
|
|
f"timeout_phase={record.get('pi_session_phase') or 'unknown'}",
|
|
f"timeout_seconds={record.get('session_stall_seconds') or 'unknown'}",
|
|
"termination_initiator="
|
|
f"{record.get('termination_initiator') or 'dispatcher'}",
|
|
]
|
|
)
|
|
elif failure_class == "process-terminated":
|
|
lines.extend(
|
|
[
|
|
f"termination_signal={record.get('termination_signal') or 'unknown'}",
|
|
"termination_initiator="
|
|
f"{record.get('termination_initiator') or 'unknown'}",
|
|
]
|
|
)
|
|
lines.append(f"locator={locator}")
|
|
return lines
|
|
|
|
|
|
def terminal_diagnostic(cli: str, channel: str, line: str) -> str | None:
|
|
if channel == "stderr":
|
|
return line
|
|
try:
|
|
value = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
if cli == "agy" and re.match(
|
|
r"^\s*(?:error|fatal|provider error|model error)\b", line, re.IGNORECASE
|
|
):
|
|
return line
|
|
return None
|
|
event_type = str(value.get("type", ""))
|
|
if cli == "codex" and event_type in {"turn.failed", "error"}:
|
|
return json.dumps(value.get("error", value), ensure_ascii=False)
|
|
if cli == "agy":
|
|
severity = str(value.get("severity") or value.get("level") or "")
|
|
status = str(value.get("status") or "")
|
|
non_terminal_event = event_type.lower() in {
|
|
"assistant",
|
|
"message",
|
|
"tool",
|
|
"tool.result",
|
|
"tool_result",
|
|
}
|
|
if (
|
|
not non_terminal_event
|
|
and (
|
|
event_type.lower() in {
|
|
"error",
|
|
"fatal",
|
|
"request.failed",
|
|
"turn.failed",
|
|
}
|
|
or severity.lower() in {"error", "fatal"}
|
|
or (
|
|
status.lower() in {"failed", "rejected"}
|
|
and any(
|
|
field in value
|
|
for field in (
|
|
"code",
|
|
"error",
|
|
"error_code",
|
|
"status_code",
|
|
)
|
|
)
|
|
)
|
|
)
|
|
):
|
|
return json.dumps(value, ensure_ascii=False)
|
|
if cli == "claude":
|
|
subtype = str(value.get("subtype", ""))
|
|
if event_type == "rate_limit_event":
|
|
rate_limit_info = value.get("rate_limit_info")
|
|
if isinstance(rate_limit_info, dict) and str(
|
|
rate_limit_info.get("status", "")
|
|
).lower() == "rejected":
|
|
return json.dumps(value, ensure_ascii=False)
|
|
if event_type == "result" and (
|
|
value.get("is_error") or subtype.startswith("error")
|
|
):
|
|
# Preserve typed terminal fields such as api_error_status=429 and
|
|
# error=rate_limit. The human-readable result alone is not the
|
|
# failure contract and may change between Claude CLI releases.
|
|
return json.dumps(value, ensure_ascii=False)
|
|
if event_type == "system" and subtype.startswith("error"):
|
|
return json.dumps(value, ensure_ascii=False)
|
|
return None
|
|
|
|
|
|
def legacy_promotion_recovery(
|
|
runs_root: Path,
|
|
task: Task,
|
|
state: dict[str, Any],
|
|
) -> LegacyPromotionRecovery | None:
|
|
"""Reclassify only an older dispatcher's exhausted generic terminal failure."""
|
|
blocked = str(state.get("blocked") or "")
|
|
recovery_failures = state.get("recovery_failures")
|
|
if not blocked or not isinstance(recovery_failures, dict):
|
|
return None
|
|
locator_match = re.search(r"(?:^|\s)locator=(.+?)\s*$", blocked)
|
|
if locator_match is None:
|
|
return None
|
|
locator = Path(locator_match.group(1))
|
|
try:
|
|
locator = locator.resolve(strict=True)
|
|
runs_root = runs_root.resolve(strict=True)
|
|
if not locator.is_relative_to(runs_root) or locator.name != "locator.json":
|
|
return None
|
|
latest_record = json.loads(locator.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
return None
|
|
role = str(latest_record.get("role") or "")
|
|
try:
|
|
failure_count = int(recovery_failures.get(role, 0))
|
|
except (TypeError, ValueError):
|
|
return None
|
|
prior_sha256 = str(latest_record.get("dispatcher_source_sha256") or "")
|
|
failed_spec = agent_spec_from_record(latest_record)
|
|
if (
|
|
latest_record.get("task") != task.name
|
|
or latest_record.get("status") != "failed"
|
|
or latest_record.get("failure_class") != "generic-error"
|
|
or role not in {"worker", "selfcheck", "review"}
|
|
or failure_count < RECOVERY_FAILURE_LIMIT
|
|
or not prior_sha256
|
|
or prior_sha256 == DISPATCHER_SOURCE_SHA256
|
|
or failed_spec is None
|
|
or promoted_spec(failed_spec, 0) is None
|
|
):
|
|
return None
|
|
|
|
plan = latest_record.get("plan_number")
|
|
latest_attempt = latest_record.get("attempt")
|
|
if not isinstance(latest_attempt, int):
|
|
return None
|
|
classified_attempts: list[
|
|
tuple[int, Path, str, str, str]
|
|
] = []
|
|
for attempt_directory in runs_root.iterdir():
|
|
candidate = attempt_directory / "locator.json"
|
|
if not candidate.is_file():
|
|
continue
|
|
try:
|
|
record = json.loads(candidate.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
continue
|
|
if (
|
|
record.get("task") != task.name
|
|
or record.get("role") != role
|
|
or record.get("plan_number") != plan
|
|
or record.get("status") != "failed"
|
|
or record.get("failure_class") != "generic-error"
|
|
or record.get("dispatcher_source_sha256") != prior_sha256
|
|
or record.get("cli") != failed_spec.cli
|
|
or record.get("model") != failed_spec.model
|
|
or effective_reasoning_effort(
|
|
agent_spec_from_record(record) or failed_spec
|
|
)
|
|
!= effective_reasoning_effort(failed_spec)
|
|
or not isinstance(record.get("attempt"), int)
|
|
):
|
|
continue
|
|
diagnostics = attempt_terminal_diagnostics(
|
|
attempt_directory,
|
|
record,
|
|
)
|
|
if not diagnostics:
|
|
continue
|
|
failure_class, evidence = classify_failure_with_evidence(
|
|
"\n".join(diagnostic for _, diagnostic in diagnostics[-50:])
|
|
)
|
|
if failure_class not in CLOUD_PROMOTION_FAILURES or evidence is None:
|
|
continue
|
|
evidence_source = next(
|
|
(
|
|
source
|
|
for source, diagnostic in reversed(diagnostics)
|
|
if diagnostic == evidence
|
|
),
|
|
f"{failed_spec.cli}:terminal",
|
|
)
|
|
classified_attempts.append(
|
|
(
|
|
int(record["attempt"]),
|
|
candidate.resolve(),
|
|
failure_class,
|
|
evidence,
|
|
evidence_source,
|
|
)
|
|
)
|
|
classified_attempts.sort(key=lambda item: item[0])
|
|
expected_attempts = list(
|
|
range(latest_attempt - failure_count + 1, latest_attempt + 1)
|
|
)
|
|
matching_attempts = [
|
|
item
|
|
for item in classified_attempts
|
|
if item[0] in expected_attempts
|
|
]
|
|
if (
|
|
[item[0] for item in matching_attempts] != expected_attempts
|
|
or matching_attempts[-1][1] != locator
|
|
or len({item[2] for item in matching_attempts}) != 1
|
|
):
|
|
# Do not collapse a mixed or incomplete failure history to one retry.
|
|
return None
|
|
_, _, failure_class, evidence, evidence_source = matching_attempts[-1]
|
|
return LegacyPromotionRecovery(
|
|
locator=locator,
|
|
role=role,
|
|
failure_class=failure_class,
|
|
evidence=evidence,
|
|
evidence_source=evidence_source,
|
|
prior_dispatcher_sha256=prior_sha256,
|
|
failed_cli=failed_spec.cli,
|
|
failed_model=failed_spec.model,
|
|
failed_reasoning_effort=failed_spec.reasoning_effort,
|
|
)
|
|
|
|
|
|
def persisted_legacy_promotion_recovery(
|
|
task: Task,
|
|
state: dict[str, Any],
|
|
locator: Path,
|
|
role: str,
|
|
) -> LegacyPromotionRecovery | None:
|
|
metadata = state.get("legacy_terminal_reclassification")
|
|
if not isinstance(metadata, dict):
|
|
return None
|
|
try:
|
|
recorded_locator = Path(str(metadata["locator"])).resolve(strict=True)
|
|
record = json.loads(recorded_locator.read_text(encoding="utf-8"))
|
|
except (KeyError, OSError):
|
|
return None
|
|
except json.JSONDecodeError:
|
|
return None
|
|
if not isinstance(record, dict):
|
|
return None
|
|
failure_class = str(metadata.get("failure_class") or "")
|
|
failed_spec = agent_spec_from_record(record)
|
|
recorded_cli = str(metadata.get("failed_cli") or "")
|
|
recorded_model = str(metadata.get("failed_model") or "")
|
|
if (
|
|
recorded_locator != locator.resolve()
|
|
or record.get("task") != task.name
|
|
or record.get("role") != role
|
|
or record.get("status") != "failed"
|
|
or record.get("failure_class") != "generic-error"
|
|
or failure_class not in CLOUD_PROMOTION_FAILURES
|
|
or str(metadata.get("current_dispatcher_sha256") or "")
|
|
!= DISPATCHER_SOURCE_SHA256
|
|
or failed_spec is None
|
|
or promoted_spec(failed_spec, 0) is None
|
|
or (recorded_cli and recorded_cli != failed_spec.cli)
|
|
or (recorded_model and recorded_model != failed_spec.model)
|
|
):
|
|
return None
|
|
return LegacyPromotionRecovery(
|
|
locator=recorded_locator,
|
|
role=role,
|
|
failure_class=failure_class,
|
|
evidence=failure_class,
|
|
evidence_source=str(metadata.get("evidence_source") or "terminal"),
|
|
prior_dispatcher_sha256=str(
|
|
metadata.get("prior_dispatcher_sha256") or "unknown"
|
|
),
|
|
failed_cli=failed_spec.cli,
|
|
failed_model=failed_spec.model,
|
|
failed_reasoning_effort=(
|
|
str(metadata["failed_reasoning_effort"])
|
|
if metadata.get("failed_reasoning_effort") is not None
|
|
else failed_spec.reasoning_effort
|
|
),
|
|
)
|
|
|
|
|
|
def pending_persisted_legacy_promotion_recovery(
|
|
task: Task,
|
|
state: dict[str, Any],
|
|
) -> LegacyPromotionRecovery | None:
|
|
metadata = state.get("legacy_terminal_reclassification")
|
|
recovery_failures = state.get("recovery_failures")
|
|
if not isinstance(metadata, dict) or not isinstance(
|
|
recovery_failures, dict
|
|
):
|
|
return None
|
|
role = str(metadata.get("role") or "")
|
|
if not role:
|
|
pending_roles = [
|
|
str(candidate)
|
|
for candidate, count in recovery_failures.items()
|
|
if count
|
|
]
|
|
if len(pending_roles) != 1:
|
|
return None
|
|
role = pending_roles[0]
|
|
try:
|
|
failure_count = int(recovery_failures.get(role, 0))
|
|
locator = Path(str(metadata["locator"]))
|
|
except (KeyError, TypeError, ValueError):
|
|
return None
|
|
if not 0 < failure_count < RECOVERY_FAILURE_LIMIT:
|
|
return None
|
|
return persisted_legacy_promotion_recovery(
|
|
task,
|
|
state,
|
|
locator,
|
|
role,
|
|
)
|
|
|
|
|
|
def codex_collaboration_tool(line: str) -> str | None:
|
|
try:
|
|
value = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
return None
|
|
item = value.get("item") or {}
|
|
if (
|
|
value.get("type") == "item.started"
|
|
and item.get("type") == "collab_tool_call"
|
|
and item.get("tool")
|
|
):
|
|
return str(item["tool"])
|
|
return None
|
|
|
|
|
|
async def terminate_process_group(
|
|
process: asyncio.subprocess.Process,
|
|
grace_seconds: float = 5,
|
|
) -> None:
|
|
"""Terminate the exact subprocess group and escalate if descendants remain."""
|
|
try:
|
|
os.killpg(process.pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
if process.returncode is None:
|
|
await process.wait()
|
|
return
|
|
|
|
if process.returncode is None:
|
|
try:
|
|
await asyncio.wait_for(process.wait(), timeout=grace_seconds)
|
|
except TimeoutError:
|
|
try:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
await process.wait()
|
|
return
|
|
|
|
try:
|
|
os.killpg(process.pid, 0)
|
|
except ProcessLookupError:
|
|
return
|
|
try:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
|
|
|
|
def agy_log_diagnostics(path: Path) -> list[str]:
|
|
if not path.exists():
|
|
return []
|
|
diagnostics: list[str] = []
|
|
for line in path.read_text(encoding="utf-8", errors="replace").splitlines()[-200:]:
|
|
failure_class, evidence = classify_failure_with_evidence(line)
|
|
if (
|
|
failure_class not in CLOUD_PROMOTION_FAILURES
|
|
or evidence is None
|
|
):
|
|
continue
|
|
if failure_class == "provider-quota" and not re.search(
|
|
(
|
|
r"RESOURCE[_ ]?EXHAUSTED"
|
|
r"|\b(?:HTTP|status(?: code)?)\s*[:=]?\s*429\b"
|
|
r"|\btoo many requests\b"
|
|
r"|(?:rate.?limit|quota|capacity).{0,40}"
|
|
r"(?:exceed|exhaust|reached|reject)"
|
|
r"|(?:exceed|exhaust|reached|reject).{0,40}"
|
|
r"(?:rate.?limit|quota|capacity)"
|
|
r"|(?:rate.?limit|quota).{0,40}retry after"
|
|
),
|
|
line,
|
|
re.IGNORECASE,
|
|
):
|
|
continue
|
|
diagnostics.append(line)
|
|
return diagnostics
|
|
|
|
|
|
def attempt_terminal_diagnostics(
|
|
attempt_directory: Path,
|
|
record: dict[str, Any],
|
|
) -> list[tuple[str, str]]:
|
|
spec = agent_spec_from_record(record)
|
|
if spec is None:
|
|
return []
|
|
try:
|
|
stream_lines = (attempt_directory / "stream.log").read_text(
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
).splitlines()
|
|
except OSError:
|
|
stream_lines = []
|
|
diagnostics: list[tuple[str, str]] = []
|
|
for stream_line in stream_lines:
|
|
match = re.match(r"^\[(stdout|stderr)\]\s?(.*)$", stream_line)
|
|
if match is None:
|
|
continue
|
|
channel, payload = match.groups()
|
|
diagnostic = terminal_diagnostic(spec.cli, channel, payload)
|
|
if diagnostic:
|
|
diagnostics.append((f"{spec.cli}:{channel}", diagnostic))
|
|
if spec.cli == "agy":
|
|
diagnostics.extend(
|
|
("agy:cli-log", diagnostic)
|
|
for diagnostic in agy_log_diagnostics(
|
|
attempt_directory / "agy-cli.log"
|
|
)
|
|
)
|
|
return diagnostics
|
|
|
|
|
|
def promoted_spec(spec: AgentSpec, recovery_count: int) -> AgentSpec | None:
|
|
if spec.cli == "agy":
|
|
return AgentSpec("claude", "claude-opus-4-8", "claude/claude-opus-4-8 xhigh")
|
|
if spec.cli == "claude":
|
|
return AgentSpec(
|
|
"codex",
|
|
"gpt-5.6-terra",
|
|
"codex/gpt-5.6-terra high",
|
|
reasoning_effort="high",
|
|
)
|
|
if spec.cli == "codex" and recovery_count < 1:
|
|
return spec
|
|
return None
|
|
|
|
|
|
def render_json_line(cli: str, line: str) -> tuple[list[str], str | None]:
|
|
try:
|
|
value = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
return [line.rstrip()], None
|
|
session_id = value.get("thread_id") or value.get("session_id")
|
|
rendered: list[str] = []
|
|
if cli == "codex":
|
|
if value.get("type") == "thread.started" and session_id:
|
|
rendered.append(f"session={session_id}")
|
|
item = value.get("item") or {}
|
|
item_type = item.get("type")
|
|
if item_type == "agent_message" and item.get("text"):
|
|
rendered.extend(str(item["text"]).splitlines())
|
|
elif item_type == "command_execution":
|
|
rendered.append(f"$ {item.get('command', '')} (exit={item.get('exit_code', '?')})")
|
|
elif item_type in {"mcp_tool_call", "web_search"}:
|
|
rendered.append(f"{item_type}: {item.get('server', '')} {item.get('tool', item.get('query', ''))}")
|
|
elif value.get("type") == "turn.failed":
|
|
rendered.append(str(value.get("error", value)))
|
|
elif cli == "claude":
|
|
message = value.get("message") or {}
|
|
for block in message.get("content") or []:
|
|
if block.get("type") == "text":
|
|
rendered.extend(str(block.get("text", "")).splitlines())
|
|
elif block.get("type") == "tool_use":
|
|
rendered.append(f"tool={block.get('name', '')}")
|
|
if value.get("type") == "result" and value.get("result"):
|
|
rendered.extend(str(value["result"]).splitlines())
|
|
session_id = session_id or value.get("session_id")
|
|
return rendered, str(session_id) if session_id else None
|
|
|
|
|
|
def native_session_path(cli: str, workspace: Path, session_id: str | None, attempt_dir: Path) -> str | None:
|
|
if cli == "claude" and session_id:
|
|
encoded = str(workspace).replace("/", "-")
|
|
return str(Path.home() / ".claude" / "projects" / encoded / f"{session_id}.jsonl")
|
|
if cli == "pi" and session_id:
|
|
matches = list((attempt_dir / "pi-sessions").glob(f"*{session_id}*.jsonl"))
|
|
return str(matches[0]) if matches else str(attempt_dir / "pi-sessions")
|
|
if cli == "codex" and session_id:
|
|
matches = list((Path.home() / ".codex" / "sessions").glob(f"**/*{session_id}*.jsonl"))
|
|
return str(matches[0]) if matches else str(Path.home() / ".codex" / "sessions")
|
|
return None
|
|
|
|
|
|
def native_session_mtime_ns(path: str | None) -> int | None:
|
|
if not path:
|
|
return None
|
|
candidate = Path(path)
|
|
return candidate.stat().st_mtime_ns if candidate.is_file() else None
|
|
|
|
|
|
def reverse_jsonl_lines(path: Path):
|
|
with path.open("rb") as stream:
|
|
stream.seek(0, os.SEEK_END)
|
|
position = stream.tell()
|
|
buffer = b""
|
|
while position > 0:
|
|
read_size = min(8192, position)
|
|
position -= read_size
|
|
stream.seek(position)
|
|
buffer = stream.read(read_size) + buffer
|
|
lines = buffer.split(b"\n")
|
|
buffer = lines[0]
|
|
for line in reversed(lines[1:]):
|
|
if line.strip():
|
|
yield line
|
|
if buffer.strip():
|
|
yield buffer
|
|
|
|
|
|
def pi_session_header_version(path: Path) -> int | None:
|
|
with path.open("rb") as stream:
|
|
first_line = stream.readline()
|
|
if not first_line.strip():
|
|
return None
|
|
header = json.loads(first_line)
|
|
if not isinstance(header, dict) or header.get("type") != "session":
|
|
return None
|
|
version = header.get("version")
|
|
return version if isinstance(version, int) else None
|
|
|
|
|
|
def pi_native_session_state(path: str | None) -> PiSessionState:
|
|
if not path:
|
|
return PiSessionState("starting", reason="native-session-path-missing")
|
|
candidate = Path(path)
|
|
if not candidate.is_file():
|
|
return PiSessionState("starting", reason="native-session-file-missing")
|
|
completed_ids: list[str] = []
|
|
expected_entry_id: str | None = None
|
|
active_leaf_found = False
|
|
try:
|
|
version = pi_session_header_version(candidate)
|
|
if version != PI_SESSION_SCHEMA_VERSION:
|
|
return PiSessionState(
|
|
"unknown",
|
|
reason=(
|
|
f"unsupported-session-version:{version}"
|
|
if version is not None
|
|
else "session-header-invalid"
|
|
),
|
|
)
|
|
for raw_line in reverse_jsonl_lines(candidate):
|
|
value = json.loads(raw_line)
|
|
if not isinstance(value, dict):
|
|
return PiSessionState("unknown", reason="invalid-entry-schema")
|
|
if value.get("type") == "session":
|
|
break
|
|
entry_id = value.get("id")
|
|
parent_id = value.get("parentId")
|
|
if (
|
|
not isinstance(entry_id, str)
|
|
or not entry_id
|
|
or "parentId" not in value
|
|
or (parent_id is not None and not isinstance(parent_id, str))
|
|
):
|
|
return PiSessionState("unknown", reason="invalid-entry-identity")
|
|
if active_leaf_found and entry_id != expected_entry_id:
|
|
continue
|
|
active_leaf_found = True
|
|
expected_entry_id = parent_id
|
|
if value.get("type") != "message":
|
|
continue
|
|
message = value.get("message")
|
|
if not isinstance(message, dict):
|
|
return PiSessionState("unknown", reason="invalid-message-schema")
|
|
role = message.get("role")
|
|
if role == "toolResult":
|
|
tool_call_id = message.get("toolCallId")
|
|
if not isinstance(tool_call_id, str) or not tool_call_id:
|
|
return PiSessionState(
|
|
"unknown", reason="tool-result-id-missing"
|
|
)
|
|
if tool_call_id in completed_ids:
|
|
return PiSessionState(
|
|
"unknown", reason="duplicate-tool-result-id"
|
|
)
|
|
completed_ids.append(tool_call_id)
|
|
continue
|
|
if role == "user":
|
|
if completed_ids:
|
|
return PiSessionState(
|
|
"unknown", reason="tool-results-without-assistant"
|
|
)
|
|
return PiSessionState("awaiting-model", reason="user-message")
|
|
if role != "assistant":
|
|
return PiSessionState(
|
|
"unknown", reason=f"unsupported-message-role:{role}"
|
|
)
|
|
|
|
content = message.get("content")
|
|
if not isinstance(content, list):
|
|
return PiSessionState(
|
|
"unknown", reason="assistant-content-not-list"
|
|
)
|
|
if any(
|
|
not isinstance(block, dict)
|
|
or block.get("type") not in {"text", "thinking", "toolCall"}
|
|
for block in content
|
|
):
|
|
return PiSessionState(
|
|
"unknown", reason="unsupported-assistant-content"
|
|
)
|
|
tool_calls = [
|
|
block
|
|
for block in content
|
|
if isinstance(block, dict) and block.get("type") == "toolCall"
|
|
]
|
|
if not tool_calls:
|
|
if completed_ids:
|
|
return PiSessionState(
|
|
"unknown", reason="tool-results-without-tool-calls"
|
|
)
|
|
return PiSessionState("finishing", reason="assistant-final")
|
|
|
|
expected_ids: list[str] = []
|
|
for tool_call in tool_calls:
|
|
tool_call_id = tool_call.get("id")
|
|
if not isinstance(tool_call_id, str) or not tool_call_id:
|
|
return PiSessionState(
|
|
"unknown", reason="tool-call-id-missing"
|
|
)
|
|
if tool_call_id in expected_ids:
|
|
return PiSessionState(
|
|
"unknown", reason="duplicate-tool-call-id"
|
|
)
|
|
expected_ids.append(tool_call_id)
|
|
|
|
unexpected_ids = [
|
|
tool_call_id
|
|
for tool_call_id in completed_ids
|
|
if tool_call_id not in expected_ids
|
|
]
|
|
if unexpected_ids:
|
|
return PiSessionState(
|
|
"unknown", reason="tool-result-id-not-in-latest-batch"
|
|
)
|
|
completed_set = set(completed_ids)
|
|
completed = tuple(
|
|
tool_call_id
|
|
for tool_call_id in expected_ids
|
|
if tool_call_id in completed_set
|
|
)
|
|
pending = tuple(
|
|
tool_call_id
|
|
for tool_call_id in expected_ids
|
|
if tool_call_id not in completed_set
|
|
)
|
|
return PiSessionState(
|
|
"tool-running" if pending else "awaiting-model",
|
|
expected_tool_call_ids=tuple(expected_ids),
|
|
completed_tool_call_ids=completed,
|
|
pending_tool_call_ids=pending,
|
|
reason=(
|
|
"pending-tool-results"
|
|
if pending
|
|
else "all-tool-results-recorded"
|
|
),
|
|
)
|
|
if completed_ids:
|
|
return PiSessionState(
|
|
"unknown", reason="tool-results-without-assistant"
|
|
)
|
|
if active_leaf_found and expected_entry_id is not None:
|
|
return PiSessionState("unknown", reason="active-branch-parent-missing")
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
|
|
return PiSessionState("unknown", reason="unreadable-jsonl")
|
|
return PiSessionState("starting", reason="no-message-events")
|
|
|
|
|
|
def pi_native_session_phase(path: str | None) -> str:
|
|
return pi_native_session_state(path).phase
|
|
|
|
|
|
def log_tail_excerpt(path: Path, *, byte_limit: int = 8192, char_limit: int = 2000) -> str:
|
|
"""Return a bounded recent log excerpt without loading a long reasoning stream."""
|
|
try:
|
|
with path.open("rb") as stream:
|
|
stream.seek(max(0, path.stat().st_size - byte_limit))
|
|
text = stream.read().decode("utf-8", errors="replace")
|
|
except OSError as exc:
|
|
return f"<stream log unavailable: {exc}>"
|
|
return text[-char_limit:]
|
|
|
|
|
|
def process_start_token(value: Any) -> str | None:
|
|
"""Read Linux process start ticks so PID reuse is not treated as liveness."""
|
|
try:
|
|
pid = int(value)
|
|
text = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8")
|
|
close = text.rfind(")")
|
|
fields = text[close + 2 :].split()
|
|
return fields[19] if close >= 0 and len(fields) > 19 else None
|
|
except (TypeError, ValueError, OSError):
|
|
return None
|
|
|
|
|
|
def process_is_alive(value: Any, expected_start_token: Any = None) -> bool:
|
|
"""Return whether the same attempt/dispatcher process still exists."""
|
|
try:
|
|
pid = int(value)
|
|
if pid <= 0:
|
|
return False
|
|
os.kill(pid, 0)
|
|
except (TypeError, ValueError, OSError):
|
|
return False
|
|
current_token = process_start_token(pid)
|
|
if (
|
|
expected_start_token is not None
|
|
and current_token is not None
|
|
and str(expected_start_token) != current_token
|
|
):
|
|
return False
|
|
return True
|
|
|
|
|
|
def marked_agent_process_pids(marker: str) -> list[int]:
|
|
"""Find live processes carrying the per-attempt environment marker."""
|
|
expected = f"{AGENT_PROCESS_MARKER_ENV}={marker}".encode()
|
|
matches: list[int] = []
|
|
for environ in Path("/proc").glob("[0-9]*/environ"):
|
|
try:
|
|
values = environ.read_bytes().split(b"\0")
|
|
pid = int(environ.parent.name)
|
|
except (OSError, ValueError):
|
|
continue
|
|
if expected in values:
|
|
matches.append(pid)
|
|
return sorted(matches)
|
|
|
|
|
|
def external_active_is_live(state: dict[str, Any]) -> tuple[bool, str]:
|
|
raw_locator = state.get("active_locator")
|
|
if not raw_locator:
|
|
return False, "active locator 없음"
|
|
target = Path(str(raw_locator))
|
|
locator_path = target if target.name == "locator.json" else target / "locator.json"
|
|
locator: dict[str, Any] = {}
|
|
if locator_path.is_file():
|
|
try:
|
|
locator = json.loads(locator_path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
return False, f"locator 판독 실패: {locator_path}"
|
|
status = str(locator.get("status") or "")
|
|
if status and status != "running":
|
|
return False, f"locator status={status}"
|
|
|
|
# The stream may legitimately remain quiet during long reasoning. A live
|
|
# process is stronger evidence than a locator or dispatcher heartbeat, and
|
|
# prevents a second dispatcher from duplicating an active attempt.
|
|
agent_pid_recorded = locator.get("agent_pid") not in (None, "")
|
|
for field, token_field in (
|
|
("agent_pid", "agent_process_start_token"),
|
|
("dispatcher_pid", "dispatcher_process_start_token"),
|
|
):
|
|
if process_is_alive(locator.get(field), locator.get(token_field)):
|
|
return True, f"{field}={locator[field]} alive; output stream is monitored"
|
|
process_marker = str(locator.get("agent_process_marker") or "")
|
|
if process_marker:
|
|
marker_pids = marked_agent_process_pids(process_marker)
|
|
if marker_pids:
|
|
return (
|
|
True,
|
|
"agent process marker alive: "
|
|
+ ",".join(str(pid) for pid in marker_pids),
|
|
)
|
|
return (
|
|
False,
|
|
"agent process marker is absent from the process table",
|
|
)
|
|
if agent_pid_recorded:
|
|
return (
|
|
False,
|
|
"recorded agent process identity is no longer alive",
|
|
)
|
|
|
|
native_raw = locator.get("native_session_path")
|
|
native = Path(str(native_raw)) if native_raw else None
|
|
if native and native.is_dir():
|
|
sessions = list(native.glob("*.jsonl"))
|
|
native = max(sessions, key=lambda path: path.stat().st_mtime_ns) if sessions else None
|
|
if native is None or not native.is_file():
|
|
roots = [target] if target.is_dir() else [target.parent]
|
|
sessions = [
|
|
path
|
|
for root in roots
|
|
for path in (*root.glob("*.jsonl"), *root.glob("pi-sessions/*.jsonl"))
|
|
]
|
|
native = max(sessions, key=lambda path: path.stat().st_mtime_ns) if sessions else None
|
|
now = datetime.now(timezone.utc).timestamp()
|
|
cli = str(locator.get("cli") or "")
|
|
stream_progress_at: float | None = None
|
|
stream_raw = locator.get("stream_log")
|
|
stream = Path(str(stream_raw)) if stream_raw else None
|
|
if stream and stream.is_file():
|
|
stream_progress_at = stream.stat().st_mtime
|
|
if native and native.is_file():
|
|
native_progress_at = native.stat().st_mtime
|
|
progress_at = max(native_progress_at, stream_progress_at or 0.0)
|
|
inactive = max(0.0, now - progress_at)
|
|
if cli == "pi":
|
|
phase = pi_native_session_phase(str(native))
|
|
# Only an exact incomplete toolCall -> toolResult batch is a tool
|
|
# execution interval. Unknown/starting/model-reasoning states
|
|
# must never be treated as a stalled tool merely because their
|
|
# native event file is quiet.
|
|
if phase == "tool-running":
|
|
return (
|
|
True,
|
|
"phase=tool-running with no agent PID evidence; "
|
|
"time-based duplicate recovery is disabled",
|
|
)
|
|
return (
|
|
True,
|
|
f"phase={phase} native+stream inactive={inactive:.1f}s "
|
|
"with no agent PID evidence; time-based duplicate recovery is disabled",
|
|
)
|
|
return (
|
|
True,
|
|
"native+stream inactive="
|
|
f"{inactive:.1f}s with no agent PID evidence; "
|
|
"time-based duplicate recovery is disabled",
|
|
)
|
|
|
|
if stream_progress_at is not None:
|
|
inactive = max(0.0, now - stream_progress_at)
|
|
return (
|
|
True,
|
|
f"stream inactive={inactive:.1f}s with no agent PID evidence; "
|
|
"time-based duplicate recovery is disabled",
|
|
)
|
|
|
|
return False, f"active 증거 없음: {raw_locator}"
|
|
|
|
|
|
def laguna_resume_locator(state: dict[str, Any]) -> Path | None:
|
|
raw_locator = state.get("active_locator")
|
|
if not raw_locator:
|
|
return None
|
|
locator = Path(str(raw_locator))
|
|
try:
|
|
record = json.loads(locator.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
return None
|
|
if (
|
|
record.get("cli") != "pi"
|
|
or not str(record.get("model", "")).startswith("laguna-s")
|
|
or record.get("failure_class") not in {"context-limit", "session-stall"}
|
|
or record.get("status") != "failed"
|
|
):
|
|
return None
|
|
native_raw = record.get("native_session_path")
|
|
native = Path(str(native_raw)) if native_raw else None
|
|
if native is None or not native.exists():
|
|
return None
|
|
return locator
|
|
|
|
|
|
def agy_conversations() -> dict[Path, int]:
|
|
root = Path.home() / ".gemini" / "antigravity-cli" / "conversations"
|
|
if not root.is_dir():
|
|
return {}
|
|
return {path: path.stat().st_mtime_ns for path in root.glob("*.db")}
|
|
|
|
|
|
def build_command(
|
|
spec: AgentSpec,
|
|
prompt: str,
|
|
workspace: Path,
|
|
session_id: str,
|
|
attempt_dir: Path,
|
|
pi_resume_session: Path | None = None,
|
|
) -> list[str]:
|
|
if spec.cli == "codex":
|
|
return [
|
|
"codex", "exec", "--json", "-C", str(workspace), "-m", spec.model,
|
|
"-c", f'model_reasoning_effort="{effective_reasoning_effort(spec)}"',
|
|
"--dangerously-bypass-approvals-and-sandbox", prompt,
|
|
]
|
|
if spec.cli == "claude":
|
|
return [
|
|
"claude", "-p", "--output-format", "stream-json", "--verbose",
|
|
"--session-id", session_id, "--model", spec.model,
|
|
"--effort", str(effective_reasoning_effort(spec)),
|
|
"--dangerously-skip-permissions", prompt,
|
|
]
|
|
if spec.cli == "agy":
|
|
return [
|
|
# `--print` consumes its immediately following argument as the prompt.
|
|
# Keeping the timeout first makes Gemini answer the literal flag instead.
|
|
"agy", "--print", prompt, "--print-timeout", "8h", "--model", spec.model,
|
|
"--dangerously-skip-permissions", "--log-file", str(attempt_dir / "agy-cli.log"),
|
|
]
|
|
if spec.cli == "pi":
|
|
command = [
|
|
"pi", "-p", "--mode", "json", "--approve", "--provider", "iop", "--model", spec.model,
|
|
"--thinking", "high",
|
|
]
|
|
if pi_resume_session is not None:
|
|
command.extend(
|
|
[
|
|
"--session", str(pi_resume_session),
|
|
"--session-dir", str(pi_resume_session.parent),
|
|
]
|
|
)
|
|
else:
|
|
command.extend(
|
|
[
|
|
"--session-id", session_id,
|
|
"--session-dir", str(attempt_dir / "pi-sessions"),
|
|
]
|
|
)
|
|
command.append(prompt)
|
|
return command
|
|
raise RuntimeError(f"지원하지 않는 CLI: {spec.cli}")
|
|
|
|
|
|
async def invoke(
|
|
workspace: Path,
|
|
store: StateStore,
|
|
task: Task,
|
|
role: str,
|
|
spec: AgentSpec,
|
|
prompt: str,
|
|
resume_locator: Path | None = None,
|
|
) -> tuple[int, str | None, Path]:
|
|
attempt, identity = next_execution_identity(store, task, role)
|
|
attempt_dir = store.runs / f"{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}__{identity}"
|
|
attempt_dir.mkdir(parents=True, exist_ok=False)
|
|
locator_path = attempt_dir / "locator.json"
|
|
stream_path = attempt_dir / "stream.log"
|
|
normalized_output_path = attempt_dir / "normalized-output.log"
|
|
heartbeat_path = attempt_dir / "heartbeat.log"
|
|
stream_path.touch()
|
|
normalized_output_path.touch()
|
|
heartbeat_path.touch()
|
|
session_id = str(uuid.uuid4())
|
|
process_marker = f"{identity}__{uuid.uuid4()}"
|
|
pi_resume_session: Path | None = None
|
|
if spec.local_pi and resume_locator and resume_locator.is_file():
|
|
try:
|
|
prior = json.loads(resume_locator.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
prior = {}
|
|
prior_native = prior.get("native_session_path")
|
|
candidate = Path(str(prior_native)) if prior_native else None
|
|
if candidate and candidate.is_dir():
|
|
sessions = list(candidate.glob("*.jsonl"))
|
|
candidate = max(sessions, key=lambda path: path.stat().st_mtime_ns) if sessions else None
|
|
if candidate and candidate.is_file():
|
|
pi_resume_session = candidate
|
|
session_id = str(prior.get("session_id") or candidate.stem)
|
|
started_at = now_iso()
|
|
work_log_path = milestone_work_log_path(task)
|
|
record: dict[str, Any] = {
|
|
"execution_id": identity,
|
|
"task": task.name,
|
|
"plan_number": plan_number(task),
|
|
"role": role,
|
|
"attempt": attempt,
|
|
"workspace": str(workspace),
|
|
**dispatcher_source_provenance(),
|
|
"cli": spec.cli,
|
|
"model": spec.model,
|
|
"reasoning_effort": effective_reasoning_effort(spec),
|
|
"agent_process_marker": process_marker,
|
|
"plan_path": str(task.plan) if task.plan else None,
|
|
"review_path": str(task.review) if task.review else None,
|
|
"session_id": session_id if spec.cli in {"claude", "pi"} else None,
|
|
"native_session_path": (
|
|
str(pi_resume_session)
|
|
if pi_resume_session is not None
|
|
else native_session_path(spec.cli, workspace, session_id, attempt_dir)
|
|
),
|
|
"output_log": str(stream_path),
|
|
"stream_log": str(stream_path),
|
|
"normalized_output_log": str(normalized_output_path),
|
|
"heartbeat_log": str(heartbeat_path),
|
|
"cli_log": str(attempt_dir / "agy-cli.log") if spec.cli == "agy" else None,
|
|
"work_log": str(work_log_path.resolve()),
|
|
"started_at": started_at,
|
|
"status": "running",
|
|
"resumed_from_locator": str(resume_locator) if pi_resume_session else None,
|
|
}
|
|
stage_decision = None
|
|
if isinstance(store, StateStore):
|
|
decisions = store.task_state(task).get("execution_decisions", {})
|
|
if isinstance(decisions, dict):
|
|
stage_decision = decisions.get(role)
|
|
if isinstance(stage_decision, dict):
|
|
record.update(selector_runtime_evidence(stage_decision))
|
|
try:
|
|
record["stage_budget"] = StageFailureBudget.from_decision(store, task, stage_decision).count()
|
|
except Exception:
|
|
record["stage_budget"] = 0
|
|
# Resolve the retry handoff identity that was assigned when the pending
|
|
# quota refresh was created before the first durable locator write, so
|
|
# the first record on disk already carries the stable handoff ID a
|
|
# crash/restart can match against (the locator path changes on every
|
|
# attempt).
|
|
retry_handoff_id: str | None = None
|
|
if isinstance(store, StateStore):
|
|
retry_ctx = store.task_state(task).get("retry_quota_refresh_context")
|
|
if isinstance(retry_ctx, dict):
|
|
retry_handoff_id = retry_ctx.get("handoff_id")
|
|
if retry_handoff_id:
|
|
record["retry_handoff_id"] = retry_handoff_id
|
|
write_json(locator_path, record)
|
|
if isinstance(store, StateStore):
|
|
if retry_handoff_id:
|
|
# One-save transition: update active_locator, clear pending flag,
|
|
# and clear context together. Restore pre-state on save failure.
|
|
# A mismatch (False) or a save fault (raises) must stop before
|
|
# the provider process seam so we never launch a duplicate
|
|
# invocation against a retry intent we failed to commit.
|
|
if not store.commit_retry_handoff_locator(task, retry_handoff_id, str(locator_path)):
|
|
raise ExecutionDecisionError("retry handoff commit mismatch")
|
|
else:
|
|
store.update_task(task, active_locator=str(locator_path))
|
|
prefix = f"[{task.directory.name}][{role}][a{attempt:02d}]"
|
|
|
|
def persist_locator_record() -> None:
|
|
"""Do not abort a live model solely because a locator refresh failed."""
|
|
try:
|
|
write_json(locator_path, record)
|
|
except OSError as exc:
|
|
record["locator_write_error"] = str(exc)
|
|
attempt_event(
|
|
prefix,
|
|
f"locator 기록 경고: locator={locator_path} error={exc}",
|
|
)
|
|
|
|
attempt_event(prefix, f"locator={locator_path}")
|
|
try:
|
|
append_milestone_event(
|
|
task,
|
|
event="START",
|
|
execution_id=identity,
|
|
role=role,
|
|
attempt=attempt,
|
|
model=spec.display,
|
|
result="running",
|
|
locator=locator_path,
|
|
)
|
|
except OSError as exc:
|
|
line = f"milestone work log setup failed: {exc}"
|
|
heartbeat_path.write_text(line + "\n", encoding="utf-8")
|
|
record.update(
|
|
status="failed",
|
|
finished_at=now_iso(),
|
|
exit_code=1,
|
|
failure_class="work-log-setup",
|
|
failure_source="work-log",
|
|
provider_transport_failure_confirmed=False,
|
|
work_log_error=str(exc),
|
|
)
|
|
persist_locator_record()
|
|
attempt_event(prefix, line)
|
|
return 1, "work-log-setup", locator_path
|
|
command = build_command(
|
|
spec,
|
|
prompt,
|
|
workspace,
|
|
session_id,
|
|
attempt_dir,
|
|
pi_resume_session=pi_resume_session,
|
|
)
|
|
before_agy = agy_conversations() if spec.cli == "agy" else {}
|
|
diagnostics: list[str] = []
|
|
diagnostic_origins: list[str] = []
|
|
control_violation: str | None = None
|
|
try:
|
|
process = await asyncio.create_subprocess_exec(
|
|
*command,
|
|
cwd=workspace,
|
|
env={
|
|
**os.environ,
|
|
AGENT_PROCESS_MARKER_ENV: process_marker,
|
|
},
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
limit=10 * 1024 * 1024,
|
|
start_new_session=True,
|
|
)
|
|
# Keep the child PID in the locator before monitoring output. If this
|
|
# dispatcher is interrupted, a later dispatcher can distinguish a
|
|
# genuinely live, silent model from a stale locator and must not launch
|
|
# a duplicate continuation.
|
|
record["agent_pid"] = process.pid
|
|
record["agent_process_start_token"] = process_start_token(process.pid)
|
|
persist_locator_record()
|
|
except FileNotFoundError:
|
|
line = f"command not found: {command[0]}"
|
|
heartbeat_path.write_text(line + "\n", encoding="utf-8")
|
|
failure_class = "generic-error"
|
|
try:
|
|
append_milestone_event(
|
|
task,
|
|
event="FINISH",
|
|
execution_id=identity,
|
|
role=role,
|
|
attempt=attempt,
|
|
model=spec.display,
|
|
result=f"failed:{failure_class}:127",
|
|
locator=locator_path,
|
|
)
|
|
except OSError as exc:
|
|
record["work_log_runtime_error"] = str(exc)
|
|
failure_class = "work-log-runtime-write"
|
|
record.update(
|
|
status="failed",
|
|
finished_at=now_iso(),
|
|
exit_code=127,
|
|
failure_class=failure_class,
|
|
failure_source=(
|
|
"work-log" if failure_class == "work-log-runtime-write" else "cli-launch"
|
|
),
|
|
provider_transport_failure_confirmed=False,
|
|
)
|
|
persist_locator_record()
|
|
attempt_event(prefix, line)
|
|
return 127, failure_class, locator_path
|
|
|
|
readers: list[asyncio.Task[None]] = []
|
|
try:
|
|
assert process.stdout is not None and process.stderr is not None
|
|
queue: asyncio.Queue[tuple[str, bytes | None]] = asyncio.Queue()
|
|
|
|
async def pump(channel: str, stream: asyncio.StreamReader) -> None:
|
|
try:
|
|
while True:
|
|
raw = await stream.readline()
|
|
if not raw:
|
|
break
|
|
await queue.put((channel, raw))
|
|
finally:
|
|
await queue.put((channel, None))
|
|
|
|
readers = [
|
|
asyncio.create_task(pump("stdout", process.stdout)),
|
|
asyncio.create_task(pump("stderr", process.stderr)),
|
|
]
|
|
finished_streams = 0
|
|
loop = asyncio.get_running_loop()
|
|
last_native_mtime: int | None = None
|
|
last_stream_mtime: int | None = None
|
|
last_native_progress_at = loop.time()
|
|
last_stream_progress_at = loop.time()
|
|
with (
|
|
stream_path.open("w", encoding="utf-8") as stream_log,
|
|
normalized_output_path.open("w", encoding="utf-8") as normalized_output_log,
|
|
heartbeat_path.open("a", encoding="utf-8") as heartbeat_log,
|
|
):
|
|
while finished_streams < len(readers):
|
|
try:
|
|
channel, raw = await asyncio.wait_for(
|
|
queue.get(), timeout=STREAM_HEARTBEAT_SECONDS
|
|
)
|
|
except asyncio.TimeoutError:
|
|
try:
|
|
stream_mtime = stream_path.stat().st_mtime_ns
|
|
except OSError:
|
|
stream_mtime = None
|
|
if stream_mtime is not None:
|
|
record["stream_log_mtime_ns"] = stream_mtime
|
|
if stream_mtime != last_stream_mtime:
|
|
last_stream_mtime = stream_mtime
|
|
last_stream_progress_at = loop.time()
|
|
record.pop("pi_silence_inspection", None)
|
|
native_path = (
|
|
str(pi_resume_session)
|
|
if pi_resume_session is not None
|
|
else native_session_path(
|
|
spec.cli,
|
|
workspace,
|
|
record.get("session_id"),
|
|
attempt_dir,
|
|
)
|
|
)
|
|
if native_path:
|
|
record["native_session_path"] = native_path
|
|
native_mtime = native_session_mtime_ns(
|
|
record.get("native_session_path")
|
|
)
|
|
if native_mtime is not None:
|
|
record["native_session_mtime_ns"] = native_mtime
|
|
if native_mtime != last_native_mtime:
|
|
last_native_mtime = native_mtime
|
|
last_native_progress_at = loop.time()
|
|
# Native events and the separately flushed stream log
|
|
# are peer progress signals. A trailing toolResult only
|
|
# selects the timeout budget; it never overrides later
|
|
# reasoning/text output.
|
|
record["pi_activity_state"] = "working"
|
|
pi_session_state = pi_native_session_state(
|
|
record.get("native_session_path")
|
|
)
|
|
pi_phase = pi_session_state.phase
|
|
is_pi_tool_execution = pi_phase == "tool-running"
|
|
# Outside a toolCall->toolResult interval, model stdout/stderr
|
|
# is the liveness signal. A completed tool result changes phase
|
|
# but must not reset the model-response silence clock.
|
|
pi_inactive_seconds = loop.time() - (
|
|
max(last_native_progress_at, last_stream_progress_at)
|
|
if is_pi_tool_execution
|
|
else last_stream_progress_at
|
|
)
|
|
if spec.local_pi:
|
|
record["pi_session_phase"] = pi_phase
|
|
record["pi_session_phase_reason"] = (
|
|
pi_session_state.reason
|
|
)
|
|
record["pi_expected_tool_call_ids"] = list(
|
|
pi_session_state.expected_tool_call_ids
|
|
)
|
|
record["pi_completed_tool_call_ids"] = list(
|
|
pi_session_state.completed_tool_call_ids
|
|
)
|
|
record["pi_pending_tool_call_ids"] = list(
|
|
pi_session_state.pending_tool_call_ids
|
|
)
|
|
record["pi_stall_timeout_seconds"] = None
|
|
record.setdefault("pi_activity_state", "starting")
|
|
if (
|
|
spec.local_pi
|
|
and not is_pi_tool_execution
|
|
and pi_inactive_seconds >= PI_MODEL_RESPONSE_STALL_SECONDS
|
|
and "pi_silence_inspection" not in record
|
|
):
|
|
inspection = {
|
|
"at": now_iso(),
|
|
"silence_seconds": round(pi_inactive_seconds, 3),
|
|
"stream_tail": log_tail_excerpt(stream_path),
|
|
}
|
|
record["pi_silence_inspection"] = inspection
|
|
diagnostic = (
|
|
f"Pi {pi_phase} stream produced no update for "
|
|
f"{pi_inactive_seconds:.1f}s; recorded stream tail for inspection "
|
|
"without terminating the model process"
|
|
)
|
|
heartbeat_log.write(f"[silence-inspection] {diagnostic}\n")
|
|
heartbeat_log.flush()
|
|
persist_locator_record()
|
|
attempt_event(prefix, f"모델응답점검: {diagnostic}")
|
|
non_pi_inactive_seconds = loop.time() - max(
|
|
last_native_progress_at, last_stream_progress_at
|
|
)
|
|
if (
|
|
not spec.local_pi
|
|
and non_pi_inactive_seconds
|
|
>= PI_MODEL_RESPONSE_STALL_SECONDS
|
|
and "stream_silence_inspection" not in record
|
|
):
|
|
inspection = {
|
|
"at": now_iso(),
|
|
"silence_seconds": round(non_pi_inactive_seconds, 3),
|
|
"stream_tail": log_tail_excerpt(stream_path),
|
|
}
|
|
record["stream_silence_inspection"] = inspection
|
|
diagnostic = (
|
|
f"{spec.cli} emitted no stream output or native-session event for "
|
|
f"{non_pi_inactive_seconds:.1f}s; recorded stream tail for inspection "
|
|
"without terminating the model process"
|
|
)
|
|
heartbeat_log.write(f"[silence-inspection] {diagnostic}\n")
|
|
heartbeat_log.flush()
|
|
persist_locator_record()
|
|
attempt_event(prefix, f"모델응답점검: {diagnostic}")
|
|
heartbeat = (
|
|
f"작업중... locator={locator_path} "
|
|
f"native_session={record.get('native_session_path') or 'none'} "
|
|
f"native_mtime_ns={record.get('native_session_mtime_ns', 'none')}"
|
|
)
|
|
if spec.local_pi:
|
|
heartbeat += (
|
|
f" pi_activity={record.get('pi_activity_state')}"
|
|
f" pi_phase={pi_phase}"
|
|
)
|
|
heartbeat_log.write(f"[heartbeat] {heartbeat}\n")
|
|
heartbeat_log.flush()
|
|
persist_locator_record()
|
|
# Heartbeat is recovery state, not a user-visible lifecycle
|
|
# event. Keep it out of the caller-facing event stream.
|
|
continue
|
|
if raw is None:
|
|
finished_streams += 1
|
|
continue
|
|
record.pop("pi_silence_inspection", None)
|
|
record.pop("stream_silence_inspection", None)
|
|
if spec.local_pi and channel == "stdout":
|
|
record["pi_activity_state"] = "streaming"
|
|
line = raw.decode("utf-8", errors="replace").rstrip("\n")
|
|
stream_log.write(f"[{channel}] {line}\n")
|
|
stream_log.flush()
|
|
diagnostic = terminal_diagnostic(spec.cli, channel, line)
|
|
if diagnostic:
|
|
diagnostics.append(diagnostic)
|
|
diagnostic_origins.append(f"{spec.cli}:{channel}")
|
|
if spec.cli == "codex" and role == "review" and channel == "stdout":
|
|
collaboration_tool = codex_collaboration_tool(line)
|
|
if collaboration_tool and control_violation is None:
|
|
control_violation = collaboration_tool
|
|
diagnostics.append(
|
|
f"official review invoked forbidden collaboration tool: "
|
|
f"{collaboration_tool}"
|
|
)
|
|
diagnostic_origins.append("dispatcher:review-control")
|
|
attempt_event(
|
|
prefix,
|
|
f"리뷰 제어 계약 위반: collaboration-tool="
|
|
f"{collaboration_tool}",
|
|
)
|
|
await terminate_process_group(process)
|
|
rendered, discovered = (
|
|
render_json_line(spec.cli, line) if channel == "stdout" else ([line], None)
|
|
)
|
|
if discovered and record.get("session_id") != discovered:
|
|
record["session_id"] = discovered
|
|
if pi_resume_session is None:
|
|
record["native_session_path"] = native_session_path(
|
|
spec.cli, workspace, discovered, attempt_dir
|
|
)
|
|
persist_locator_record()
|
|
for display_line in rendered:
|
|
if display_line:
|
|
normalized_output_log.write(display_line + "\n")
|
|
normalized_output_log.flush()
|
|
# Child output is retained for recovery and review but is
|
|
# not itself a dispatcher lifecycle event.
|
|
await asyncio.gather(*readers)
|
|
return_code = await process.wait()
|
|
except asyncio.CancelledError:
|
|
for reader in readers:
|
|
reader.cancel()
|
|
if readers:
|
|
await asyncio.gather(*readers, return_exceptions=True)
|
|
await terminate_process_group(process)
|
|
runtime_error: OSError | None = None
|
|
try:
|
|
append_milestone_event(
|
|
task,
|
|
event="FINISH",
|
|
execution_id=identity,
|
|
role=role,
|
|
attempt=attempt,
|
|
model=spec.display,
|
|
result="failed:cancelled",
|
|
locator=locator_path,
|
|
)
|
|
except OSError as exc:
|
|
runtime_error = exc
|
|
record.update(
|
|
status="failed",
|
|
finished_at=now_iso(),
|
|
exit_code="cancelled",
|
|
failure_class="cancelled",
|
|
failure_source="caller-cancel",
|
|
provider_transport_failure_confirmed=False,
|
|
)
|
|
if runtime_error is not None:
|
|
record["work_log_runtime_error"] = str(runtime_error)
|
|
persist_locator_record()
|
|
raise
|
|
|
|
if spec.cli == "agy":
|
|
after_agy = agy_conversations()
|
|
changed = [
|
|
path for path, mtime in after_agy.items()
|
|
if path not in before_agy or before_agy[path] != mtime
|
|
]
|
|
if changed:
|
|
selected = max(changed, key=lambda path: after_agy[path])
|
|
record["session_id"] = selected.stem
|
|
record["native_session_path"] = str(selected)
|
|
agy_diagnostics = agy_log_diagnostics(attempt_dir / "agy-cli.log")
|
|
diagnostics.extend(agy_diagnostics)
|
|
diagnostic_origins.extend("agy:cli-log" for _ in agy_diagnostics)
|
|
native_path = (
|
|
str(pi_resume_session)
|
|
if pi_resume_session is not None
|
|
else native_session_path(
|
|
spec.cli, workspace, record.get("session_id"), attempt_dir
|
|
)
|
|
)
|
|
if native_path:
|
|
record["native_session_path"] = native_path
|
|
native_mtime = native_session_mtime_ns(record.get("native_session_path"))
|
|
if native_mtime is not None:
|
|
record["native_session_mtime_ns"] = native_mtime
|
|
failure_source: str | None = None
|
|
failure_evidence: str | None = None
|
|
failure_evidence_source: str | None = None
|
|
provider_transport_failure_confirmed = False
|
|
termination = termination_signal(return_code)
|
|
if termination is not None:
|
|
record["termination_signal"] = termination[0]
|
|
record["termination_signal_inferred"] = termination[1]
|
|
if control_violation:
|
|
failure_class = "review-control-violation"
|
|
failure_source = "dispatcher-control"
|
|
failure_evidence_source = "dispatcher:review-control"
|
|
for index in range(len(diagnostics) - 1, -1, -1):
|
|
if diagnostic_origins[index] == failure_evidence_source:
|
|
failure_evidence = diagnostics[index]
|
|
break
|
|
elif return_code != 0 and termination is not None:
|
|
failure_class = "process-terminated"
|
|
failure_source = "process-termination"
|
|
record["termination_initiator"] = "unknown"
|
|
else:
|
|
classified_failure, classified_evidence = classify_failure_with_evidence(
|
|
"\n".join(diagnostics[-50:])
|
|
)
|
|
if return_code != 0 or classified_evidence is not None:
|
|
failure_class = classified_failure
|
|
failure_evidence = classified_evidence
|
|
else:
|
|
failure_class = None
|
|
if failure_class is not None and failure_evidence is not None:
|
|
for index in range(len(diagnostics) - 1, -1, -1):
|
|
if diagnostics[index] == failure_evidence:
|
|
failure_evidence_source = diagnostic_origins[index]
|
|
break
|
|
if failure_class in PROVIDER_TRANSPORT_FAILURES:
|
|
failure_source = "provider-terminal-diagnostic"
|
|
provider_transport_failure_confirmed = failure_evidence is not None
|
|
elif failure_evidence is not None:
|
|
failure_source = "cli-terminal-diagnostic"
|
|
elif return_code != 0:
|
|
failure_source = "cli-exit"
|
|
try:
|
|
append_milestone_event(
|
|
task,
|
|
event="FINISH",
|
|
execution_id=identity,
|
|
role=role,
|
|
attempt=attempt,
|
|
model=spec.display,
|
|
result=(
|
|
f"succeeded:0"
|
|
if return_code == 0 and failure_class is None
|
|
else f"failed:{failure_class or 'generic-error'}:{return_code}"
|
|
),
|
|
locator=locator_path,
|
|
)
|
|
except OSError as exc:
|
|
if failure_class is not None:
|
|
record["prior_failure_class"] = failure_class
|
|
record["work_log_runtime_error"] = str(exc)
|
|
failure_class = "work-log-runtime-write"
|
|
failure_source = "work-log"
|
|
failure_evidence = None
|
|
failure_evidence_source = None
|
|
provider_transport_failure_confirmed = False
|
|
if failure_evidence is not None:
|
|
record["failure_evidence_excerpt"] = failure_evidence[:FAILURE_EVIDENCE_LIMIT]
|
|
record["failure_evidence_truncated"] = (
|
|
len(failure_evidence) > FAILURE_EVIDENCE_LIMIT
|
|
)
|
|
if failure_evidence_source is not None:
|
|
record["failure_evidence_source"] = failure_evidence_source
|
|
record.update(
|
|
status="succeeded" if return_code == 0 and failure_class is None else "failed",
|
|
finished_at=now_iso(),
|
|
exit_code=return_code,
|
|
failure_class=failure_class,
|
|
failure_source=failure_source,
|
|
provider_transport_failure_confirmed=provider_transport_failure_confirmed,
|
|
)
|
|
persist_locator_record()
|
|
return return_code, failure_class, locator_path
|
|
|
|
|
|
def base_prompt(task: Task, role: str, spec: AgentSpec) -> str:
|
|
if role == "review":
|
|
target = task.review or task.directory
|
|
if task.review:
|
|
return f"Read {target.resolve()} and start the review. Keep artifact content in English. Final in Korean."
|
|
return f"Continue the review for {target.resolve()}. Keep artifact content in English. Final in Korean."
|
|
if task.plan is None:
|
|
raise RuntimeError("worker PLAN이 없다")
|
|
target = task.plan.resolve()
|
|
if role == "selfcheck":
|
|
if task.review is None:
|
|
raise RuntimeError("selfcheck CODE_REVIEW 파일이 없다")
|
|
return (
|
|
f"Think in English. Keep artifact content in English. Final in Korean. Read {task.review.resolve()} and fill "
|
|
"every missing implementation field. Do not finish until all implementation "
|
|
"fields are complete. This is a self-check of completed work, not a review. "
|
|
f"Read {target} and finish any missing work. Recheck and fix your work."
|
|
)
|
|
if spec.local_pi:
|
|
return f"Think in English. Keep artifact content in English. Final in Korean. Read {target} and complete the task."
|
|
return f"Read {target} and complete the task. Keep artifact content in English. Final in Korean."
|
|
|
|
|
|
|
|
def build_context_package(
|
|
workspace: Path, task: Task, locator: Path, *, previous_spec: AgentSpec, next_spec: AgentSpec
|
|
) -> dict[str, Any]:
|
|
"""Build the fail-closed continuation context for a target transition."""
|
|
|
|
if not locator.is_file():
|
|
raise ExecutionDecisionError("logical context locator가 없다")
|
|
try:
|
|
record = json.loads(locator.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
raise ExecutionDecisionError("logical context locator를 읽을 수 없다") from exc
|
|
if not isinstance(record, dict):
|
|
raise ExecutionDecisionError("logical context locator 형식이 잘못됐다")
|
|
workspace_value = record.get("workspace")
|
|
if not isinstance(workspace_value, str) or not workspace_value:
|
|
raise ExecutionDecisionError("logical context workspace가 없다")
|
|
recorded_workspace = Path(workspace_value)
|
|
if not recorded_workspace.is_absolute() or recorded_workspace.resolve() != workspace.resolve():
|
|
raise ExecutionDecisionError("logical context workspace가 일치하지 않는다")
|
|
if record.get("task") != task.name:
|
|
raise ExecutionDecisionError("logical context task가 일치하지 않는다")
|
|
if task.plan is None or not task.plan.is_file():
|
|
raise ExecutionDecisionError("logical context PLAN이 없다")
|
|
plan_value = record.get("plan_path")
|
|
if not isinstance(plan_value, str) or not plan_value:
|
|
raise ExecutionDecisionError("logical context PLAN 경로가 없다")
|
|
plan_path = Path(plan_value)
|
|
if not plan_path.is_absolute() or plan_path.resolve() != task.plan.resolve():
|
|
raise ExecutionDecisionError("logical context PLAN 경로가 일치하지 않는다")
|
|
required = {
|
|
"normalized_output": ("normalized_output_log", "normalized-output.log"),
|
|
"raw_log": ("stream_log", "stream.log"),
|
|
}
|
|
paths: dict[str, str] = {}
|
|
attempt_dir = locator.resolve().parent
|
|
for field, (record_field, filename) in required.items():
|
|
value = record.get(record_field)
|
|
if not isinstance(value, str) or not value:
|
|
raise ExecutionDecisionError(f"logical context {field} artifact가 없다")
|
|
path = Path(value)
|
|
expected = attempt_dir / filename
|
|
if not path.is_absolute() or path.resolve() != expected or not expected.is_file():
|
|
raise ExecutionDecisionError(f"logical context {field} artifact가 locator attempt와 일치하지 않는다")
|
|
paths[field] = str(expected)
|
|
same_pi = previous_spec.local_pi and next_spec.local_pi
|
|
package = {
|
|
"plan": str(task.plan.resolve()), "locator": str(locator.resolve()),
|
|
"workspace": str(workspace.resolve()), **paths,
|
|
"resume_mode": "native" if same_pi else "logical",
|
|
}
|
|
if same_pi:
|
|
native = Path(str(record.get("native_session_path", "")))
|
|
if not native.is_file():
|
|
raise ExecutionDecisionError("same-Pi logical context native session이 없다")
|
|
package["native_session_path"] = str(native.resolve())
|
|
return package
|
|
|
|
|
|
def canonical_selector_failover_route(decision: dict[str, Any] | None) -> bool:
|
|
if not isinstance(decision, dict):
|
|
return False
|
|
candidates = decision.get("candidates")
|
|
return isinstance(candidates, list) and len(candidates) > 1
|
|
|
|
|
|
def logical_context_prompt(context: dict[str, Any]) -> str:
|
|
plan = context["plan"]
|
|
locator = context["locator"]
|
|
workspace = context["workspace"]
|
|
raw_log = context["raw_log"]
|
|
normalized_output = context["normalized_output"]
|
|
return (
|
|
f"Think in English. Keep artifact content in English. Final in Korean. "
|
|
f"Read plan={plan}, locator={locator}, workspace={workspace}, "
|
|
f"raw_log={raw_log}, normalized_output={normalized_output} and complete the task."
|
|
)
|
|
|
|
|
|
def continuation_prompt_from_package(
|
|
context_package: dict[str, Any],
|
|
*,
|
|
target: dict[str, Any] | None = None,
|
|
native_resume: bool = False,
|
|
) -> str:
|
|
if native_resume or context_package.get("resume_mode") == "native":
|
|
return (
|
|
"Think in English. Keep artifact content in English. Final in Korean. Continue this session and complete "
|
|
"the current task."
|
|
)
|
|
plan = context_package["plan"]
|
|
locator = context_package["locator"]
|
|
workspace = context_package["workspace"]
|
|
raw_log = context_package["raw_log"]
|
|
normalized_output = context_package["normalized_output"]
|
|
return (
|
|
f"Think in English. Keep artifact content in English. Final in Korean. "
|
|
f"Read plan={plan}, locator={locator}, workspace={workspace}, "
|
|
f"raw_log={raw_log}, normalized_output={normalized_output} and complete the task."
|
|
)
|
|
|
|
|
|
def continuation_prompt(
|
|
task: Task,
|
|
role: str,
|
|
locator: Path | None = None,
|
|
*,
|
|
local_pi: bool = False,
|
|
resume_same_pi_session: bool = False,
|
|
context: dict[str, Any] | None = None,
|
|
) -> str:
|
|
if context is not None:
|
|
return continuation_prompt_from_package(
|
|
context,
|
|
native_resume=resume_same_pi_session or context.get("resume_mode") == "native",
|
|
)
|
|
if local_pi:
|
|
if resume_same_pi_session:
|
|
return (
|
|
"Think in English. Keep artifact content in English. Final in Korean. Continue this session and complete "
|
|
"the current task."
|
|
)
|
|
if role == "selfcheck" and task.plan and task.review:
|
|
return (
|
|
f"Think in English. Keep artifact content in English. Final in Korean. Read {task.review.resolve()} and fill "
|
|
"every missing implementation field. Do not finish until all implementation "
|
|
"fields are complete. This is a self-check of completed work, not a review. "
|
|
f"Read {task.plan.resolve()} and finish any missing work. Recheck and fix "
|
|
"your work."
|
|
)
|
|
target = task.plan or task.directory
|
|
return f"Think in English. Keep artifact content in English. Final in Korean. Read {target.resolve()} and complete the task."
|
|
if role == "review":
|
|
return f"Continue the review for {task.directory.resolve()}. Keep artifact content in English. Final in Korean."
|
|
return (
|
|
f"Continue from {locator.resolve() if locator else task.directory.resolve()}. Check the saved context and current "
|
|
"workspace. Keep artifact content in English. Final in Korean."
|
|
)
|
|
|
|
|
|
async def run_escalating(
|
|
workspace: Path,
|
|
store: StateStore,
|
|
task: Task,
|
|
role: str,
|
|
initial: AgentSpec,
|
|
initial_resume_locator: Path | None = None,
|
|
) -> tuple[bool, Path | None]:
|
|
spec = initial
|
|
previous_locator = initial_resume_locator
|
|
codex_recovery_count = 0
|
|
codex_session_stall_retries = 0
|
|
review_control_retries = 0
|
|
pi_recovery_retries = 0
|
|
generic_retries = 0
|
|
terminal_recovery_retries = 0
|
|
pi_resume_locator = initial_resume_locator
|
|
recovery_failures = 0
|
|
stage_budget: StageFailureBudget | None = None
|
|
if isinstance(store, StateStore):
|
|
state = store.task_state(task)
|
|
persisted = state.get("recovery_failures", {})
|
|
if isinstance(persisted, dict):
|
|
recovery_failures = int(persisted.get(role, 0))
|
|
decisions = state.get("execution_decisions", {})
|
|
decision = decisions.get(role) if isinstance(decisions, dict) and role in {"worker", "review"} else None
|
|
if isinstance(decision, dict):
|
|
stage_budget = StageFailureBudget.from_decision(store, task, decision)
|
|
recovery_failures = stage_budget.count()
|
|
legacy_recovery: LegacyPromotionRecovery | None = None
|
|
if initial_resume_locator is not None and isinstance(store, StateStore):
|
|
state = store.task_state(task)
|
|
legacy_recovery = legacy_promotion_recovery(
|
|
store.runs,
|
|
task,
|
|
state,
|
|
)
|
|
if legacy_recovery is not None:
|
|
recovery_failures = 1
|
|
persisted_failures = dict(state.get("recovery_failures", {}))
|
|
persisted_failures[legacy_recovery.role] = recovery_failures
|
|
store.update_task(
|
|
task,
|
|
blocked=None,
|
|
recovery_failures=persisted_failures,
|
|
legacy_terminal_reclassification={
|
|
"role": legacy_recovery.role,
|
|
"failure_class": legacy_recovery.failure_class,
|
|
"evidence_source": legacy_recovery.evidence_source,
|
|
"prior_dispatcher_sha256":
|
|
legacy_recovery.prior_dispatcher_sha256,
|
|
"current_dispatcher_sha256":
|
|
DISPATCHER_SOURCE_SHA256,
|
|
"locator": str(legacy_recovery.locator),
|
|
"failed_cli": legacy_recovery.failed_cli,
|
|
"failed_model": legacy_recovery.failed_model,
|
|
"failed_reasoning_effort":
|
|
legacy_recovery.failed_reasoning_effort,
|
|
},
|
|
)
|
|
else:
|
|
legacy_recovery = persisted_legacy_promotion_recovery(
|
|
task,
|
|
state,
|
|
initial_resume_locator,
|
|
role,
|
|
)
|
|
if legacy_recovery is not None and legacy_recovery.role == role:
|
|
failed_spec = failed_spec_from_recovery(legacy_recovery)
|
|
next_spec = promoted_spec(failed_spec, codex_recovery_count)
|
|
if next_spec is not None:
|
|
banner(
|
|
"모델승격",
|
|
task.name,
|
|
[
|
|
f"from={failed_spec.display}",
|
|
f"to={next_spec.display}",
|
|
f"failure_class={legacy_recovery.failure_class}",
|
|
"failure_source=legacy-terminal-reclassification",
|
|
f"failure_evidence_source={legacy_recovery.evidence_source}",
|
|
"dispatcher_source_sha256="
|
|
f"{legacy_recovery.prior_dispatcher_sha256}",
|
|
f"dispatcher_source_current_sha256={DISPATCHER_SOURCE_SHA256}",
|
|
f"locator={legacy_recovery.locator}",
|
|
],
|
|
)
|
|
spec = next_spec
|
|
if recovery_failures >= RECOVERY_FAILURE_LIMIT:
|
|
locator = initial_resume_locator
|
|
reason = (
|
|
f"{role} recovery failure limit already exhausted: "
|
|
f"{recovery_failures}/{RECOVERY_FAILURE_LIMIT}"
|
|
)
|
|
if isinstance(store, StateStore):
|
|
decision = store.task_state(task).get("execution_decisions", {}).get(role, {})
|
|
store.update_task(
|
|
task,
|
|
blocked=f"{reason} locator={locator}",
|
|
blocker_evidence={
|
|
"role": role,
|
|
"failure_class": None,
|
|
"locator": str(locator) if locator else None,
|
|
"selected": decision.get("selected") if isinstance(decision, dict) else None,
|
|
"work_unit_id": decision.get("work_unit_id") if isinstance(decision, dict) else None,
|
|
},
|
|
)
|
|
banner(
|
|
"작업차단",
|
|
task.name,
|
|
[
|
|
"reason=recovery-failure-limit",
|
|
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
|
|
f"locator={locator}",
|
|
],
|
|
)
|
|
return False, locator
|
|
context: dict[str, Any] | None = None
|
|
while True:
|
|
prompt = (
|
|
base_prompt(task, role, spec)
|
|
if previous_locator is None
|
|
else continuation_prompt(
|
|
task,
|
|
role,
|
|
previous_locator,
|
|
local_pi=spec.local_pi,
|
|
resume_same_pi_session=pi_resume_locator is not None,
|
|
context=context,
|
|
)
|
|
)
|
|
context = None
|
|
rc, failure, locator = await invoke(
|
|
workspace,
|
|
store,
|
|
task,
|
|
role,
|
|
spec,
|
|
prompt,
|
|
resume_locator=pi_resume_locator,
|
|
)
|
|
pi_resume_locator = None
|
|
if rc == 0 and failure is None:
|
|
if isinstance(store, StateStore):
|
|
state = store.task_state(task)
|
|
persisted = dict(state.get("recovery_failures", {}))
|
|
persisted.pop(role, None)
|
|
store.update_task(task, recovery_failures=persisted)
|
|
if stage_budget is not None:
|
|
stage_budget.reset_on_success()
|
|
return True, locator
|
|
failure = failure or "generic-error"
|
|
if failure in {
|
|
"work-log-blocked",
|
|
"work-log-incomplete",
|
|
"work-log-setup",
|
|
"work-log-runtime-write",
|
|
}:
|
|
banner("작업차단", task.name, failure_report_lines(failure, locator))
|
|
return False, locator
|
|
recovery_failures += 1
|
|
if stage_budget is not None:
|
|
selected = stage_budget.store.task_state(task)["execution_decisions"][role]["selected"]
|
|
transition = stage_budget.store.task_state(task)["execution_decisions"][role]["transition"]["trigger"]
|
|
recovery_failures = stage_budget.record_failure(target=selected, transition=transition)
|
|
if isinstance(store, StateStore):
|
|
state = store.task_state(task)
|
|
persisted = dict(state.get("recovery_failures", {}))
|
|
persisted[role] = recovery_failures
|
|
store.update_task(task, recovery_failures=persisted)
|
|
if recovery_failures >= RECOVERY_FAILURE_LIMIT:
|
|
reason = (
|
|
f"{role} recovery failure limit exhausted: "
|
|
f"{recovery_failures}/{RECOVERY_FAILURE_LIMIT}"
|
|
)
|
|
if isinstance(store, StateStore):
|
|
decision = store.task_state(task).get("execution_decisions", {}).get(role, {})
|
|
store.update_task(
|
|
task,
|
|
blocked=f"{reason} locator={locator}",
|
|
blocker_evidence={
|
|
"role": role,
|
|
"failure_class": failure,
|
|
"locator": str(locator) if locator else None,
|
|
"selected": decision.get("selected") if isinstance(decision, dict) else None,
|
|
"work_unit_id": decision.get("work_unit_id") if isinstance(decision, dict) else None,
|
|
},
|
|
)
|
|
banner(
|
|
"작업차단",
|
|
task.name,
|
|
[
|
|
"reason=recovery-failure-limit",
|
|
*failure_report_lines(failure, locator),
|
|
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
|
|
],
|
|
)
|
|
return False, locator
|
|
if role == "review" and failure == "review-control-violation":
|
|
review_control_retries += 1
|
|
banner(
|
|
"리뷰재시도",
|
|
task.name,
|
|
[
|
|
*failure_report_lines(failure, locator),
|
|
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
|
|
],
|
|
)
|
|
previous_locator = None
|
|
await asyncio.sleep(min(30, 2 ** min(review_control_retries, 5)))
|
|
continue
|
|
current_decision = None
|
|
quota_snapshot = None
|
|
if isinstance(store, StateStore):
|
|
task_state = store.task_state(task)
|
|
decisions = task_state.get("execution_decisions", {})
|
|
if isinstance(decisions, dict):
|
|
current_decision = decisions.get(role)
|
|
quota_snapshot = task_state.get("quota_snapshot")
|
|
|
|
if canonical_selector_failover_route(current_decision) and failure in QUALIFIED_FAILOVER_FAILURES:
|
|
try:
|
|
if failure == "provider-quota":
|
|
derived = derive_work_unit_quota_evidence(
|
|
current_decision,
|
|
status="exhausted",
|
|
reason="confirmed_runtime_provider_quota",
|
|
)
|
|
quota_snapshot = derived
|
|
next_decision = select_execution_decision(
|
|
task,
|
|
stage=role,
|
|
prior_decision=current_decision,
|
|
quota_snapshot=quota_snapshot,
|
|
transition="failover",
|
|
failure_class=failure,
|
|
)
|
|
next_spec = agent_spec_from_decision(next_decision)
|
|
if next_spec != spec:
|
|
if locator is None:
|
|
raise ExecutionDecisionError("logical context locator가 없다")
|
|
context = build_context_package(
|
|
workspace, task, locator, previous_spec=spec, next_spec=next_spec
|
|
)
|
|
commit_execution_decision(store, task, role, next_decision)
|
|
banner(
|
|
"모델승격" if role == "worker" else "리뷰승격",
|
|
task.name,
|
|
[
|
|
f"from={spec.display}",
|
|
f"to={next_spec.display}",
|
|
*failure_report_lines(failure, locator),
|
|
],
|
|
)
|
|
spec = next_spec
|
|
previous_locator = locator
|
|
continue
|
|
else:
|
|
commit_execution_decision(store, task, role, next_decision)
|
|
except (ExecutionDecisionError, OSError, ValueError) as exc:
|
|
code = getattr(exc, "code", "")
|
|
if not code:
|
|
if "no_failover_candidate" in str(exc):
|
|
code = "no_failover_candidate"
|
|
else:
|
|
code = exc.__class__.__name__
|
|
store.update_task(
|
|
task, blocked=f"{role} selector decision 실패 [{code}]: {exc}"
|
|
)
|
|
banner(
|
|
"작업차단",
|
|
task.name,
|
|
[f"reason={code}", *failure_report_lines(failure, locator)],
|
|
)
|
|
return False, locator
|
|
if spec.local_pi:
|
|
if (
|
|
spec.model.startswith("laguna-s")
|
|
and failure in {"context-limit", "session-stall"}
|
|
):
|
|
pi_recovery_retries += 1
|
|
banner(
|
|
"Pi세션연속재시작",
|
|
task.name,
|
|
[
|
|
f"model={spec.display}",
|
|
*failure_report_lines(failure, locator),
|
|
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
|
|
],
|
|
)
|
|
previous_locator = locator
|
|
pi_resume_locator = locator
|
|
await asyncio.sleep(min(30, 2 ** min(pi_recovery_retries, 5)))
|
|
continue
|
|
pi_recovery_retries += 1
|
|
if failure == "session-stall":
|
|
event = "세션응답복구재시도"
|
|
elif failure in {
|
|
"provider-connection",
|
|
"provider-stream-disconnect",
|
|
}:
|
|
event = "세션연결재시도"
|
|
else:
|
|
event = "Pi복구재시도"
|
|
banner(
|
|
event,
|
|
task.name,
|
|
[
|
|
f"model={spec.display}",
|
|
*failure_report_lines(failure, locator),
|
|
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
|
|
],
|
|
)
|
|
previous_locator = locator
|
|
await asyncio.sleep(min(30, 2 ** min(pi_recovery_retries, 5)))
|
|
continue
|
|
if spec.cli == "codex" and failure == "session-stall":
|
|
codex_session_stall_retries += 1
|
|
banner(
|
|
"세션응답복구재시도",
|
|
task.name,
|
|
[
|
|
f"model={spec.display}",
|
|
*failure_report_lines(failure, locator),
|
|
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
|
|
],
|
|
)
|
|
previous_locator = locator
|
|
await asyncio.sleep(min(30, 2 ** min(codex_session_stall_retries, 5)))
|
|
continue
|
|
if failure == "generic-error":
|
|
generic_retries += 1
|
|
banner(
|
|
"작업복구재시도",
|
|
task.name,
|
|
[
|
|
f"model={spec.display}",
|
|
*failure_report_lines(failure, locator),
|
|
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
|
|
],
|
|
)
|
|
previous_locator = locator
|
|
await asyncio.sleep(min(30, 2 ** min(generic_retries, 5)))
|
|
continue
|
|
if failure not in CLOUD_PROMOTION_FAILURES:
|
|
terminal_recovery_retries += 1
|
|
banner(
|
|
"모델복구재시도",
|
|
task.name,
|
|
[
|
|
f"model={spec.display}",
|
|
*failure_report_lines(failure, locator),
|
|
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
|
|
],
|
|
)
|
|
previous_locator = locator
|
|
await asyncio.sleep(
|
|
min(30, 2 ** min(terminal_recovery_retries, 5))
|
|
)
|
|
continue
|
|
if role == "review":
|
|
terminal_recovery_retries += 1
|
|
banner(
|
|
"리뷰재시도",
|
|
task.name,
|
|
[
|
|
f"model={spec.display}",
|
|
"reason=official-review-fixed-target",
|
|
*failure_report_lines(failure, locator),
|
|
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
|
|
],
|
|
)
|
|
previous_locator = locator
|
|
await asyncio.sleep(
|
|
min(30, 2 ** min(terminal_recovery_retries, 5))
|
|
)
|
|
continue
|
|
if (
|
|
isinstance(store, StateStore)
|
|
and role == "worker"
|
|
and isinstance(current_decision, dict)
|
|
):
|
|
try:
|
|
next_decision = select_execution_decision(
|
|
task,
|
|
stage=role,
|
|
prior_decision=current_decision,
|
|
quota_snapshot=quota_snapshot,
|
|
transition="promotion",
|
|
failure_class=failure,
|
|
)
|
|
except ExecutionDecisionError as exc:
|
|
if "no_promotion_target" in str(exc):
|
|
# canonical promotion 대상 없음: selector-backed worker는
|
|
# 현재 target recovery/budget exhaustion 또는 block으로만 종결.
|
|
# legacy promoted_spec()으로의 fallthrough 금지.
|
|
banner(
|
|
"모델재시도",
|
|
task.name,
|
|
[
|
|
f"model={spec.display}",
|
|
"reason=no-promotion-target",
|
|
*failure_report_lines(failure, locator),
|
|
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
|
|
],
|
|
)
|
|
previous_locator = locator
|
|
terminal_recovery_retries += 1
|
|
await asyncio.sleep(
|
|
min(30, 2 ** min(terminal_recovery_retries, 5))
|
|
)
|
|
continue
|
|
store.update_task(
|
|
task,
|
|
blocked=f"{role} selector promotion 실패: {exc}",
|
|
)
|
|
banner(
|
|
"작업차단",
|
|
task.name,
|
|
[
|
|
"reason=selector-promotion",
|
|
*failure_report_lines(failure, locator),
|
|
],
|
|
)
|
|
return False, locator
|
|
else:
|
|
next_spec = agent_spec_from_decision(next_decision)
|
|
if next_spec == spec:
|
|
store.update_task(
|
|
task,
|
|
blocked="selector promotion이 현재 target을 다시 선택했다",
|
|
)
|
|
return False, locator
|
|
if locator is None:
|
|
store.update_task(
|
|
task, blocked="logical context locator가 없다"
|
|
)
|
|
return False, locator
|
|
try:
|
|
context = build_context_package(
|
|
workspace,
|
|
task,
|
|
locator,
|
|
previous_spec=spec,
|
|
next_spec=next_spec,
|
|
)
|
|
except ExecutionDecisionError as exc:
|
|
store.update_task(task, blocked=str(exc))
|
|
return False, locator
|
|
commit_execution_decision(store, task, role, next_decision)
|
|
banner(
|
|
"모델승격",
|
|
task.name,
|
|
[
|
|
f"from={spec.display}",
|
|
f"to={next_spec.display}",
|
|
*failure_report_lines(failure, locator),
|
|
],
|
|
)
|
|
spec = next_spec
|
|
previous_locator = locator
|
|
continue
|
|
next_spec = promoted_spec(spec, codex_recovery_count)
|
|
if next_spec is None:
|
|
terminal_recovery_retries += 1
|
|
banner(
|
|
"모델복구재시도",
|
|
task.name,
|
|
[
|
|
f"model={spec.display}",
|
|
*failure_report_lines(failure, locator),
|
|
f"retry={recovery_failures}/{RECOVERY_FAILURE_LIMIT}",
|
|
],
|
|
)
|
|
previous_locator = locator
|
|
await asyncio.sleep(min(30, 2 ** min(terminal_recovery_retries, 5)))
|
|
continue
|
|
if spec.cli == "codex":
|
|
codex_recovery_count += 1
|
|
banner(
|
|
"모델승격",
|
|
task.name,
|
|
[
|
|
f"from={spec.display}",
|
|
f"to={next_spec.display}",
|
|
*failure_report_lines(failure, locator),
|
|
],
|
|
)
|
|
spec = next_spec
|
|
previous_locator = locator
|
|
|
|
|
|
def task_signature(workspace: Path, task: Task) -> str:
|
|
digest = hashlib.sha256()
|
|
if not task.directory.exists():
|
|
return "moved"
|
|
for path in sorted(p for p in task.directory.iterdir() if p.is_file()):
|
|
if (
|
|
PLAN_RE.match(path.name)
|
|
or REVIEW_RE.match(path.name)
|
|
or path.name.endswith(".log")
|
|
):
|
|
digest.update(path.name.encode())
|
|
digest.update(sha256_file(path).encode())
|
|
for raw_path in sorted(task.write_set):
|
|
path = Path(raw_path)
|
|
path = path if path.is_absolute() else workspace / path
|
|
digest.update(raw_path.encode())
|
|
if path.is_file():
|
|
digest.update(str(path.stat().st_mode).encode())
|
|
digest.update(sha256_file(path).encode())
|
|
elif path.exists():
|
|
digest.update(b"non-file")
|
|
else:
|
|
digest.update(b"missing")
|
|
return digest.hexdigest()
|
|
|
|
|
|
def read_verdict(path: Path) -> str | None:
|
|
if not path.exists():
|
|
return None
|
|
return verdict_from_text(path.read_text(encoding="utf-8", errors="replace"))
|
|
|
|
|
|
def verdict_from_text(text: str) -> str | None:
|
|
selected: tuple[re.Match[str], re.Pattern[str], re.Pattern[str]] | None = None
|
|
for heading_re, line_re, block_re in VERDICT_SCHEMA_MATCHERS:
|
|
headings = list(heading_re.finditer(text))
|
|
if not headings:
|
|
continue
|
|
# A duplicated heading, or headings from both schemas, is ambiguous.
|
|
if len(headings) != 1 or selected is not None:
|
|
return None
|
|
selected = (headings[0], line_re, block_re)
|
|
if selected is None:
|
|
return None
|
|
heading, line_re, block_re = selected
|
|
next_heading = re.search(r"^##\s+", text[heading.end():], re.MULTILINE)
|
|
end = heading.end() + next_heading.start() if next_heading else len(text)
|
|
section = text[heading.end():end]
|
|
inline_matches = list(line_re.finditer(section))
|
|
block_matches = list(block_re.finditer(section))
|
|
matches = inline_matches + block_matches
|
|
return matches[0].group(1) if len(matches) == 1 else None
|
|
|
|
|
|
def matching_archive_directories_by_name(
|
|
workspace: Path,
|
|
task_name: str,
|
|
*,
|
|
require_complete: bool = True,
|
|
) -> list[Path]:
|
|
archive = workspace / "agent-task" / "archive"
|
|
parts = task_name.split("/")
|
|
if not archive.is_dir() or len(parts) not in {1, 2}:
|
|
return []
|
|
group = parts[0]
|
|
final_name = parts[-1]
|
|
suffix_re = re.compile(rf"^{re.escape(final_name)}(?:_\d+)?$")
|
|
matches: list[Path] = []
|
|
try:
|
|
years = list(archive.iterdir())
|
|
except FileNotFoundError:
|
|
return []
|
|
for year in years:
|
|
if not year.is_dir():
|
|
continue
|
|
try:
|
|
months = list(year.iterdir())
|
|
except FileNotFoundError:
|
|
continue
|
|
for month in months:
|
|
if not month.is_dir():
|
|
continue
|
|
parent = month if len(parts) == 1 else month / group
|
|
if not parent.is_dir():
|
|
continue
|
|
try:
|
|
candidates = list(parent.iterdir())
|
|
except FileNotFoundError:
|
|
continue
|
|
for candidate in candidates:
|
|
if (
|
|
candidate.is_dir()
|
|
and suffix_re.match(candidate.name)
|
|
and (
|
|
not require_complete
|
|
or (candidate / "complete.log").is_file()
|
|
)
|
|
):
|
|
matches.append(candidate)
|
|
return sorted(matches)
|
|
|
|
|
|
def matching_archive_directories(workspace: Path, task: Task) -> list[Path]:
|
|
return matching_archive_directories_by_name(workspace, task.name)
|
|
|
|
|
|
def task_group_name(task_name: str) -> str:
|
|
return task_name.split("/", 1)[0]
|
|
|
|
|
|
def work_log_event_cells(line: str) -> list[str] | None:
|
|
stripped = line.strip()
|
|
if not stripped.startswith("|") or not stripped.endswith("|"):
|
|
return None
|
|
cells = [
|
|
cell.strip().replace(r"\|", "|")
|
|
for cell in re.split(r"(?<!\\)\|", stripped[1:-1])
|
|
]
|
|
return cells if len(cells) == 9 else None
|
|
|
|
|
|
def unfinished_work_log_attempts(path: Path) -> list[dict[str, Any]]:
|
|
"""Return START rows that have no matching FINISH row."""
|
|
try:
|
|
lines = path.read_text(
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
).splitlines()
|
|
except OSError:
|
|
raise
|
|
open_attempts: dict[str, dict[str, Any]] = {}
|
|
for line in lines:
|
|
cells = work_log_event_cells(line)
|
|
if cells is None or cells[2] not in {"START", "FINISH"}:
|
|
continue
|
|
try:
|
|
sequence = int(cells[0])
|
|
attempt = int(cells[5])
|
|
except ValueError:
|
|
continue
|
|
locator = cells[8]
|
|
key = locator or "\0".join(
|
|
(cells[3], cells[4], cells[5], cells[6])
|
|
)
|
|
if cells[2] == "START":
|
|
open_attempts[key] = {
|
|
"sequence": sequence,
|
|
"task_name": cells[3],
|
|
"role": cells[4],
|
|
"attempt": attempt,
|
|
"model": cells[6],
|
|
"locator": locator,
|
|
}
|
|
else:
|
|
open_attempts.pop(key, None)
|
|
return sorted(
|
|
open_attempts.values(),
|
|
key=lambda record: int(record["sequence"]),
|
|
)
|
|
|
|
|
|
def close_unfinished_work_log_attempts(path: Path) -> int:
|
|
"""Close orphaned START rows after verified group completion."""
|
|
unfinished = unfinished_work_log_attempts(path)
|
|
for record in unfinished:
|
|
locator = Path(str(record["locator"]))
|
|
append_work_log_event(
|
|
path,
|
|
task_name=str(record["task_name"]),
|
|
event="FINISH",
|
|
execution_id=f"reconciled-{record['sequence']}",
|
|
role=str(record["role"]),
|
|
attempt=int(record["attempt"]),
|
|
model=str(record["model"]),
|
|
result="reconciled:verified-complete-archive",
|
|
locator=locator,
|
|
)
|
|
return len(unfinished)
|
|
|
|
|
|
def archived_task_group_directories(
|
|
workspace: Path,
|
|
task_group: str,
|
|
) -> list[Path]:
|
|
"""Return month-local archive directories for one logical task group."""
|
|
archive_root = workspace / "agent-task" / "archive"
|
|
if not archive_root.is_dir():
|
|
return []
|
|
suffix_re = re.compile(rf"^{re.escape(task_group)}(?:_\d+)?$")
|
|
matches: list[Path] = []
|
|
try:
|
|
years = list(archive_root.iterdir())
|
|
except FileNotFoundError:
|
|
return []
|
|
for year in years:
|
|
if not year.is_dir():
|
|
continue
|
|
try:
|
|
months = list(year.iterdir())
|
|
except FileNotFoundError:
|
|
continue
|
|
for month in months:
|
|
if not month.is_dir():
|
|
continue
|
|
try:
|
|
candidates = list(month.iterdir())
|
|
except FileNotFoundError:
|
|
continue
|
|
matches.extend(
|
|
candidate
|
|
for candidate in candidates
|
|
if candidate.is_dir() and suffix_re.fullmatch(candidate.name)
|
|
)
|
|
return sorted(matches)
|
|
|
|
|
|
def next_work_log_archive_number(
|
|
workspace: Path,
|
|
task_group: str,
|
|
) -> int:
|
|
numbers = [
|
|
int(match.group(1))
|
|
for directory in archived_task_group_directories(workspace, task_group)
|
|
for path in directory.glob("work_log_*.log")
|
|
if (match := WORK_LOG_ARCHIVE_RE.fullmatch(path.name))
|
|
]
|
|
return max(numbers, default=-1) + 1
|
|
|
|
|
|
def completed_group_archive_directory(
|
|
task_group: str,
|
|
task_names: set[str],
|
|
completed_tasks: dict[str, str],
|
|
) -> Path | None:
|
|
candidates: list[tuple[int, str, Path]] = []
|
|
for task_name in task_names:
|
|
archive_raw = completed_tasks.get(task_name)
|
|
if not archive_raw:
|
|
continue
|
|
archive = Path(archive_raw)
|
|
if not archive.is_dir():
|
|
continue
|
|
complete_log = archive / "complete.log"
|
|
target = archive if task_name == task_group else archive.parent
|
|
try:
|
|
completed_at = (
|
|
complete_log.stat().st_mtime_ns
|
|
if complete_log.is_file()
|
|
else archive.stat().st_mtime_ns
|
|
)
|
|
except OSError:
|
|
continue
|
|
candidates.append((completed_at, str(target), target))
|
|
return (
|
|
max(candidates, key=lambda item: (item[0], item[1]))[2]
|
|
if candidates
|
|
else None
|
|
)
|
|
|
|
|
|
def archive_completed_group_work_logs(
|
|
workspace: Path,
|
|
observed_tasks: set[str],
|
|
completed_tasks: dict[str, str],
|
|
active_or_running: set[str],
|
|
) -> tuple[dict[str, str], dict[str, str]]:
|
|
"""Archive each completed task-group timeline after its last writer exits."""
|
|
observed_by_group: dict[str, set[str]] = {}
|
|
for task_name in observed_tasks:
|
|
observed_by_group.setdefault(task_group_name(task_name), set()).add(
|
|
task_name
|
|
)
|
|
active_groups = {
|
|
task_group_name(task_name)
|
|
for task_name in active_or_running
|
|
}
|
|
archived: dict[str, str] = {}
|
|
errors: dict[str, str] = {}
|
|
for task_group, task_names in sorted(observed_by_group.items()):
|
|
if task_group in active_groups or not task_names <= set(completed_tasks):
|
|
continue
|
|
active_source = (
|
|
workspace / "agent-task" / task_group / WORK_LOG_NAME
|
|
)
|
|
legacy_sources = {
|
|
Path(completed_tasks[task_name]) / WORK_LOG_NAME
|
|
for task_name in task_names
|
|
if (Path(completed_tasks[task_name]) / WORK_LOG_NAME).is_file()
|
|
}
|
|
sources = {
|
|
*legacy_sources,
|
|
*([active_source] if active_source.is_file() else []),
|
|
}
|
|
if not sources:
|
|
continue
|
|
if len(sources) != 1:
|
|
errors[task_group] = (
|
|
"active/archive WORK_LOG source가 여러 개라 자동 병합할 수 없다: "
|
|
+ ",".join(str(path) for path in sorted(sources))
|
|
)
|
|
continue
|
|
target_directory = completed_group_archive_directory(
|
|
task_group,
|
|
task_names,
|
|
completed_tasks,
|
|
)
|
|
if target_directory is None:
|
|
errors[task_group] = (
|
|
"검증된 task archive에서 WORK_LOG 대상 디렉터리를 정할 수 없다"
|
|
)
|
|
continue
|
|
archive_number = next_work_log_archive_number(
|
|
workspace,
|
|
task_group,
|
|
)
|
|
destination = target_directory / f"work_log_{archive_number}.log"
|
|
source = next(iter(sources))
|
|
if destination.exists():
|
|
errors[task_group] = (
|
|
"WORK_LOG archive destination이 이미 존재한다: "
|
|
f"source={source} destination={destination}"
|
|
)
|
|
continue
|
|
try:
|
|
close_unfinished_work_log_attempts(source)
|
|
source.replace(destination)
|
|
except OSError as exc:
|
|
errors[task_group] = (
|
|
f"WORK_LOG archive 실패: source={source} "
|
|
f"destination={destination} error={exc}"
|
|
)
|
|
continue
|
|
if source == active_source:
|
|
for task_name in sorted(task_names, reverse=True):
|
|
if "/" not in task_name:
|
|
continue
|
|
try:
|
|
(workspace / "agent-task" / task_name).rmdir()
|
|
except OSError:
|
|
pass
|
|
try:
|
|
active_source.parent.rmdir()
|
|
except OSError:
|
|
pass
|
|
archived[task_group] = str(destination.resolve())
|
|
return archived, errors
|
|
|
|
|
|
def task_attempt_log_directories(runs: Path, task_name: str) -> list[Path]:
|
|
"""Return dispatcher-owned attempt directories whose locator names the task."""
|
|
matches: list[Path] = []
|
|
if not runs.is_dir():
|
|
return matches
|
|
for attempt_dir in runs.iterdir():
|
|
if not attempt_dir.is_dir():
|
|
continue
|
|
locator = attempt_dir / "locator.json"
|
|
try:
|
|
record = json.loads(locator.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
continue
|
|
if record.get("task") != task_name:
|
|
continue
|
|
matches.append(attempt_dir)
|
|
return matches
|
|
|
|
|
|
def cleanup_completed_task_attempt_logs(runs: Path, task_name: str) -> int:
|
|
"""Remove only dispatcher-owned logs for a task after its complete archive exists."""
|
|
removed = 0
|
|
for attempt_dir in task_attempt_log_directories(runs, task_name):
|
|
try:
|
|
shutil.rmtree(attempt_dir)
|
|
except OSError as exc:
|
|
attempt_event(
|
|
"[attempt-log-cleanup-warning]",
|
|
f"task={task_name} path={attempt_dir} error={exc}",
|
|
)
|
|
continue
|
|
removed += 1
|
|
return removed
|
|
|
|
|
|
def review_fingerprints(workspace: Path, task: Task) -> set[tuple[str, str]]:
|
|
directories = [task.directory] if task.directory.is_dir() else []
|
|
directories.extend(matching_archive_directories(workspace, task))
|
|
fingerprints: set[tuple[str, str]] = set()
|
|
for directory in directories:
|
|
for path in directory.iterdir():
|
|
if path.is_file() and (path.name.startswith("code_review_") or REVIEW_RE.match(path.name)):
|
|
fingerprints.add((str(path.resolve()), sha256_file(path)))
|
|
return fingerprints
|
|
|
|
|
|
def review_outcome(
|
|
workspace: Path, task: Task, prior_fingerprints: set[tuple[str, str]]
|
|
) -> dict[str, str]:
|
|
if task.directory.is_dir():
|
|
directories = [task.directory]
|
|
archives: list[Path] = []
|
|
else:
|
|
archives = matching_archive_directories(workspace, task)
|
|
directories = list(archives)
|
|
newest_directory: Path | None = None
|
|
newest_log: Path | None = None
|
|
newest_mtime = -1
|
|
for directory in directories:
|
|
logs = list(directory.glob("code_review_*.log"))
|
|
logs.extend(path for path in directory.iterdir() if path.is_file() and REVIEW_RE.match(path.name))
|
|
for log in logs:
|
|
mtime = log.stat().st_mtime_ns
|
|
fingerprint = (str(log.resolve()), sha256_file(log))
|
|
if fingerprint not in prior_fingerprints and mtime > newest_mtime and read_verdict(log):
|
|
newest_directory = directory
|
|
newest_log = log
|
|
newest_mtime = mtime
|
|
verdict = read_verdict(newest_log) if newest_log else "UNKNOWN"
|
|
if newest_directory in archives:
|
|
state = "archived"
|
|
elif newest_directory and (newest_directory / "USER_REVIEW.md").exists():
|
|
state = "user-review"
|
|
elif newest_directory and (newest_directory / "complete.log").exists():
|
|
state = "complete-finalization"
|
|
elif newest_log and REVIEW_RE.match(newest_log.name):
|
|
state = "finalization-pending"
|
|
elif newest_directory and any(REVIEW_RE.match(path.name) for path in newest_directory.iterdir() if path.is_file()):
|
|
state = "follow-up"
|
|
else:
|
|
state = "changed"
|
|
return {
|
|
"verdict": verdict or "UNKNOWN",
|
|
"state": state,
|
|
"path": str(newest_directory or task.directory),
|
|
"review_log": str(newest_log) if newest_log else "unknown",
|
|
}
|
|
|
|
|
|
async def run_worker(
|
|
workspace: Path,
|
|
store: StateStore,
|
|
task: Task,
|
|
resume_locator: Path | None = None,
|
|
quota_snapshot: dict[str, Any] | None = None,
|
|
) -> None:
|
|
retry_context = store.task_state(task).get("retry_quota_refresh_context")
|
|
if resume_locator is None and isinstance(retry_context, dict):
|
|
locator_value = retry_context.get("locator")
|
|
if isinstance(locator_value, str) and locator_value:
|
|
resume_locator = Path(locator_value)
|
|
|
|
# If an active_locator already exists from a prior attempt that wrote its
|
|
# locator but crashed before consuming the pending handoff, consume it now
|
|
# to prevent a duplicate invocation. The locator write is the durable
|
|
# commitment; the pending handoff is the logical intent. Consume the intent
|
|
# when the commitment is already present.
|
|
if isinstance(store, StateStore):
|
|
prior_state = store.task_state(task)
|
|
prior_active = prior_state.get("active_locator")
|
|
prior_pending = prior_state.get("retry_quota_refresh_pending")
|
|
if prior_active and prior_pending:
|
|
consumed = False
|
|
# Prefer handoff_id matching: read the stable identity from the
|
|
# active locator file so we can match across crash boundaries
|
|
# where the locator path changes.
|
|
try:
|
|
prior_locator_data = json.loads(Path(prior_active).read_text(encoding="utf-8"))
|
|
if isinstance(prior_locator_data, dict):
|
|
handoff_id = prior_locator_data.get("retry_handoff_id")
|
|
if handoff_id:
|
|
consumed = store.commit_retry_handoff_locator(
|
|
task, handoff_id, prior_active,
|
|
)
|
|
except (OSError, json.JSONDecodeError):
|
|
pass
|
|
if not consumed:
|
|
# Fallback: match by locator path when handoff_id is
|
|
# unavailable (e.g. crash between state update and locator
|
|
# write, or pre-existing state from a prior dispatcher
|
|
# version).
|
|
consumed = store.consume_matching_retry_handoff(task, prior_active)
|
|
|
|
try:
|
|
decision, spec = persisted_execution_decision(
|
|
store, task, stage="worker", quota_snapshot=quota_snapshot
|
|
)
|
|
except ExecutionDecisionError as exc:
|
|
store.update_task(task, blocked=str(exc))
|
|
banner("작업차단", task.name, [f"reason={exc}"])
|
|
return
|
|
work_log = milestone_work_log_path(task)
|
|
banner(
|
|
"작업시작",
|
|
task.name,
|
|
[
|
|
f"model={spec.display}",
|
|
f"plan={task.plan.resolve()}",
|
|
f"work_log={work_log.resolve()}",
|
|
],
|
|
)
|
|
success, locator = await run_escalating(
|
|
workspace,
|
|
store,
|
|
task,
|
|
"worker",
|
|
spec,
|
|
initial_resume_locator=resume_locator,
|
|
)
|
|
if not success:
|
|
current = store.task_state(task).get("blocked")
|
|
store.update_task(
|
|
task, blocked=current or f"worker failure locator={locator}"
|
|
)
|
|
return
|
|
completed_spec = agent_spec_from_locator(locator) or spec
|
|
try:
|
|
_mark_worker_done(
|
|
store,
|
|
task,
|
|
initial_decision=decision,
|
|
worker_cli=completed_spec.cli,
|
|
worker_model=completed_spec.model,
|
|
)
|
|
except ExecutionDecisionError as exc:
|
|
store.update_task(task, worker_done=False, blocked=f"worker completion validation failed: {exc}")
|
|
banner("작업차단", task.name, [f"reason={exc}"])
|
|
return
|
|
|
|
|
|
def _require_same_runtime_identity(
|
|
expected_spec: AgentSpec,
|
|
worker_cli: str,
|
|
worker_model: str,
|
|
) -> None:
|
|
"""Verify the worker CLI/model matches the validated completing decision spec.
|
|
|
|
Ensures the actual worker that ran is the same runtime identity that the
|
|
completing decision authorizes. Prevents a cloud-completed worker from
|
|
being recorded as a Pi selfcheck target or vice versa.
|
|
"""
|
|
if expected_spec.cli != worker_cli:
|
|
raise ExecutionDecisionError(
|
|
f"worker runtime CLI 불일치: expected={expected_spec.cli} actual={worker_cli}"
|
|
)
|
|
if expected_spec.model != worker_model:
|
|
raise ExecutionDecisionError(
|
|
f"worker runtime model 불일치: expected={expected_spec.model} actual={worker_model}"
|
|
)
|
|
|
|
|
|
def _mark_worker_done(
|
|
store: StateStore,
|
|
task: Task,
|
|
*,
|
|
initial_decision: dict[str, Any],
|
|
worker_cli: str,
|
|
worker_model: str,
|
|
) -> None:
|
|
"""Persist worker completion with the authoritative completing decision.
|
|
|
|
Uses the persisted execution_decisions worker entry as the sole authoritative
|
|
source. Does not fall back to initial_decision even when the persisted
|
|
decision is malformed—malformed persisted state blocks completion rather
|
|
than silently reverting to a speculative initial decision.
|
|
|
|
Validates the completing decision through the strict contract validator
|
|
and verifies the worker CLI/model identity matches the normalized spec.
|
|
On any validation failure, raises ExecutionDecisionError to prevent
|
|
worker_done from being recorded.
|
|
"""
|
|
decisions = store.task_state(task).get("execution_decisions", {})
|
|
if not isinstance(decisions, dict) or "worker" not in decisions:
|
|
raise ExecutionDecisionError(
|
|
"persisted worker decision이 execution_decisions에 없다"
|
|
)
|
|
decision = decisions["worker"]
|
|
if not isinstance(decision, dict):
|
|
raise ExecutionDecisionError(
|
|
"persisted worker decision이 dict가 아니다"
|
|
)
|
|
validated_decision, expected_spec = _validated_completing_decision(
|
|
task, decision
|
|
)
|
|
_require_same_runtime_identity(expected_spec, worker_cli, worker_model)
|
|
execution_class = validated_decision["selected"]["execution_class"]
|
|
store.update_task(
|
|
task,
|
|
worker_done=True,
|
|
worker_cli=worker_cli,
|
|
worker_model=worker_model,
|
|
completing_decision=validated_decision,
|
|
execution_class=execution_class,
|
|
selfcheck_done=(execution_class == "cloud_model"),
|
|
blocked=None,
|
|
)
|
|
|
|
|
|
async def run_selfcheck(
|
|
workspace: Path,
|
|
store: StateStore,
|
|
task: Task,
|
|
resume_locator: Path | None = None,
|
|
) -> None:
|
|
completing = store.task_state(task).get("completing_decision")
|
|
if not isinstance(completing, dict):
|
|
store.update_task(
|
|
task, blocked="completing decision이 없어 selfcheck를 실행할 수 없다"
|
|
)
|
|
banner(
|
|
"작업차단", task.name,
|
|
["reason=missing-completing-decision"],
|
|
)
|
|
return
|
|
try:
|
|
_completed_decision, spec = _validated_completing_decision(task, completing)
|
|
except ExecutionDecisionError as exc:
|
|
store.update_task(task, blocked=str(exc))
|
|
banner("작업차단", task.name, [f"reason={exc}"])
|
|
return
|
|
if not spec.local_pi:
|
|
raise RuntimeError("Pi가 아닌 route에 selfcheck stage가 배정됐다")
|
|
work_log = milestone_work_log_path(task)
|
|
banner(
|
|
"자가검증시작",
|
|
task.name,
|
|
[
|
|
f"model={spec.display}",
|
|
f"plan={task.plan.resolve()}",
|
|
f"work_log={work_log.resolve()}",
|
|
],
|
|
)
|
|
retry = 0
|
|
if isinstance(store, StateStore):
|
|
retry = int(store.task_state(task).get("selfcheck_incomplete", 0))
|
|
if retry >= SELF_CHECK_INCOMPLETE_LIMIT:
|
|
locator = resume_locator
|
|
reason = (
|
|
"selfcheck implementation field limit already exhausted: "
|
|
f"{retry}/{SELF_CHECK_INCOMPLETE_LIMIT}"
|
|
)
|
|
store.update_task(task, blocked=f"{reason} locator={locator}")
|
|
banner(
|
|
"작업차단",
|
|
task.name,
|
|
[
|
|
"reason=selfcheck-incomplete-limit",
|
|
f"retry={retry}/{SELF_CHECK_INCOMPLETE_LIMIT}",
|
|
f"locator={locator}",
|
|
],
|
|
)
|
|
return
|
|
while True:
|
|
success, locator = await run_escalating(
|
|
workspace,
|
|
store,
|
|
task,
|
|
"selfcheck",
|
|
spec,
|
|
initial_resume_locator=resume_locator,
|
|
)
|
|
resume_locator = None
|
|
if not success:
|
|
current = store.task_state(task).get("blocked")
|
|
store.update_task(
|
|
task, blocked=current or f"selfcheck failure locator={locator}"
|
|
)
|
|
return
|
|
errors = implementation_review_errors(task)
|
|
if not errors:
|
|
break
|
|
retry += 1
|
|
if retry >= SELF_CHECK_INCOMPLETE_LIMIT:
|
|
reason = (
|
|
"selfcheck implementation fields remain incomplete: "
|
|
f"{retry}/{SELF_CHECK_INCOMPLETE_LIMIT}"
|
|
)
|
|
store.update_task(
|
|
task,
|
|
blocked=f"{reason} locator={locator}",
|
|
selfcheck_incomplete=retry,
|
|
)
|
|
banner(
|
|
"작업차단",
|
|
task.name,
|
|
[
|
|
"reason=selfcheck-incomplete-limit",
|
|
f"detail={'; '.join(errors)}",
|
|
f"retry={retry}/{SELF_CHECK_INCOMPLETE_LIMIT}",
|
|
f"locator={locator}",
|
|
],
|
|
)
|
|
return
|
|
store.update_task(task, selfcheck_incomplete=retry)
|
|
banner(
|
|
"자가검증재시도",
|
|
task.name,
|
|
[
|
|
f"reason={'; '.join(errors)}",
|
|
f"retry={retry}/{SELF_CHECK_INCOMPLETE_LIMIT}",
|
|
f"locator={locator}",
|
|
],
|
|
)
|
|
store.update_task(
|
|
task,
|
|
selfcheck_done=True,
|
|
selfcheck_incomplete=0,
|
|
blocked=None,
|
|
)
|
|
|
|
|
|
async def run_review(
|
|
workspace: Path,
|
|
store: StateStore,
|
|
task: Task,
|
|
resume_locator: Path | None = None,
|
|
quota_snapshot: dict[str, Any] | None = None,
|
|
) -> str | None:
|
|
try:
|
|
_, spec = persisted_execution_decision(
|
|
store, task, stage="review", quota_snapshot=quota_snapshot
|
|
)
|
|
except ExecutionDecisionError as exc:
|
|
store.update_task(task, blocked=str(exc))
|
|
banner("작업차단", task.name, [f"reason={exc}"])
|
|
return None
|
|
state = store.task_state(task)
|
|
prior_no_progress = int(state.get("review_no_progress", 0))
|
|
if prior_no_progress >= REVIEW_NO_PROGRESS_LIMIT:
|
|
locator = state.get("active_locator")
|
|
reason = (
|
|
"review no-progress limit already exhausted: "
|
|
f"{prior_no_progress}/{REVIEW_NO_PROGRESS_LIMIT}"
|
|
)
|
|
store.update_task(task, blocked=f"{reason} locator={locator}")
|
|
banner(
|
|
"작업차단",
|
|
task.name,
|
|
[
|
|
"reason=review-no-progress-limit",
|
|
f"unchanged_review_attempts={prior_no_progress}/{REVIEW_NO_PROGRESS_LIMIT}",
|
|
f"locator={locator}",
|
|
],
|
|
)
|
|
return None
|
|
before = task_signature(workspace, task)
|
|
prior_review_fingerprints = review_fingerprints(workspace, task)
|
|
target = task.review.resolve() if task.review else task.directory.resolve()
|
|
banner("리뷰시작", task.name, [f"model={spec.display}", f"review={target}"])
|
|
success, locator = await run_escalating(
|
|
workspace,
|
|
store,
|
|
task,
|
|
"review",
|
|
spec,
|
|
initial_resume_locator=resume_locator,
|
|
)
|
|
if not success:
|
|
current = store.task_state(task).get("blocked")
|
|
store.update_task(
|
|
task, blocked=current or f"review failure locator={locator}"
|
|
)
|
|
return None
|
|
after = task_signature(workspace, task)
|
|
if before == after:
|
|
state = store.task_state(task)
|
|
count = int(state.get("review_no_progress", 0)) + 1
|
|
if count >= REVIEW_NO_PROGRESS_LIMIT:
|
|
reason = (
|
|
"review made no progress: "
|
|
f"{count}/{REVIEW_NO_PROGRESS_LIMIT} locator={locator}"
|
|
)
|
|
store.update_task(
|
|
task,
|
|
review_no_progress=count,
|
|
blocked=reason,
|
|
)
|
|
banner(
|
|
"작업차단",
|
|
task.name,
|
|
[
|
|
"reason=review-no-progress-limit",
|
|
f"unchanged_review_attempts={count}/{REVIEW_NO_PROGRESS_LIMIT}",
|
|
f"locator={locator}",
|
|
],
|
|
)
|
|
return None
|
|
store.update_task(task, review_no_progress=count)
|
|
banner(
|
|
"루프정체경고",
|
|
task.name,
|
|
[
|
|
f"unchanged_review_attempts={count}/{REVIEW_NO_PROGRESS_LIMIT}",
|
|
f"locator={locator}",
|
|
],
|
|
)
|
|
await asyncio.sleep(min(30, count * 5))
|
|
else:
|
|
store.update_task(task, review_no_progress=0, blocked=None)
|
|
outcome = review_outcome(workspace, task, prior_review_fingerprints)
|
|
banner(
|
|
"리뷰결과",
|
|
task.name,
|
|
[
|
|
f"verdict={outcome['verdict']}",
|
|
f"state={outcome['state']}",
|
|
f"path={outcome['path']}",
|
|
f"review_log={outcome['review_log']}",
|
|
f"locator={locator}",
|
|
],
|
|
)
|
|
if outcome["verdict"] == "PASS" and outcome["state"] == "archived":
|
|
banner("작업완료", task.name, [f"archive={outcome['path']}", f"locator={locator}"])
|
|
return outcome["path"]
|
|
if outcome["verdict"] == "UNKNOWN" or outcome["state"] == "changed":
|
|
raise RuntimeError(
|
|
"official review가 판정과 다음 파일 상태를 materialize하지 않았다; "
|
|
f"locator={locator}"
|
|
)
|
|
if outcome["state"] == "archived":
|
|
raise RuntimeError(
|
|
f"PASS가 아닌 review가 완료 archive로 이동했다: "
|
|
f"verdict={outcome['verdict']} locator={locator}"
|
|
)
|
|
return None
|
|
|
|
|
|
def status_lines(
|
|
task: Task,
|
|
stage: str,
|
|
dependency: str,
|
|
decision: dict[str, Any] | None = None,
|
|
) -> list[str]:
|
|
route = f"{task.lane}-G{task.grade:02d}" if task.lane and task.grade else "recovery"
|
|
base = [f"stage={stage}", f"route={route}", f"dependency={dependency}"]
|
|
if decision is not None:
|
|
return base + selector_evidence_lines(decision)
|
|
return base
|
|
|
|
|
|
def select_dispatch_candidates(
|
|
ready: list[tuple[Task, str]],
|
|
) -> tuple[list[tuple[Task, str]], list[tuple[Task, str]], str]:
|
|
ready_reviews = [(task, stage) for task, stage in ready if stage == "review"]
|
|
ready_workers = [(task, stage) for task, stage in ready if stage in {"worker", "selfcheck"}]
|
|
# Each scheduler pass admits every dependency-ready task. The running map
|
|
# prevents only duplicate attempts for the same task; phases and write-sets
|
|
# never impose a cross-task barrier.
|
|
return ready_reviews + ready_workers, [], ""
|
|
|
|
|
|
def ensure_review_shared_state(workspace: Path) -> None:
|
|
helper = workspace / "agent-ops" / "bin" / "ai-ignore.sh"
|
|
if not helper.is_file():
|
|
raise RuntimeError(f"review shared-state helper가 없다: {helper}")
|
|
command = [
|
|
"bash",
|
|
"-c",
|
|
'source "$1" && agent_ops_ensure_gitignore_task_artifact_block "$2"',
|
|
"agent-task-review-preflight",
|
|
str(helper),
|
|
str(workspace / ".gitignore"),
|
|
]
|
|
completed = subprocess.run(
|
|
command,
|
|
cwd=workspace,
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
if completed.returncode != 0:
|
|
diagnostic = (completed.stderr or completed.stdout).strip()
|
|
raise RuntimeError(
|
|
f"review shared-state preflight 실패: {diagnostic or completed.returncode}"
|
|
)
|
|
|
|
|
|
async def dispatch(args: argparse.Namespace) -> int:
|
|
workspace = Path(args.workspace).resolve()
|
|
store = StateStore(workspace)
|
|
try:
|
|
try:
|
|
return await dispatch_with_store(args, workspace, store)
|
|
except Exception as exc:
|
|
# A scheduler/control-plane exception must not make asyncio.run()
|
|
# cancel already-running agent attempts. Keep this loop alive until
|
|
# every owned background task finishes naturally; the next
|
|
# dispatcher run reconciles their file/state results.
|
|
current = asyncio.current_task()
|
|
active = [
|
|
task
|
|
for task in asyncio.all_tasks()
|
|
if task is not current and not task.done()
|
|
]
|
|
if active:
|
|
banner(
|
|
"디스패처복구대기",
|
|
args.task_group or "agent-task",
|
|
[
|
|
f"running_async_tasks={len(active)}",
|
|
"scheduler 예외와 무관하게 실행 중 agent를 자연 종료까지 추적",
|
|
],
|
|
)
|
|
await asyncio.gather(*active, return_exceptions=True)
|
|
raise DispatcherInterruptedWithActiveWork(
|
|
f"running agent가 있던 scheduler 예외: {exc}"
|
|
) from exc
|
|
raise
|
|
finally:
|
|
store.close()
|
|
|
|
|
|
async def dispatch_with_store(
|
|
args: argparse.Namespace,
|
|
workspace: Path,
|
|
store: StateStore,
|
|
) -> int:
|
|
orchestration_scope = args.task_group or "__all__"
|
|
if args.retry_blocked and not args.dry_run:
|
|
store.mark_retry_quota_refresh(args.task_group)
|
|
running: dict[str, asyncio.Task[str | None]] = {}
|
|
last_wait: dict[str, str] = {}
|
|
completed_tasks: dict[str, str] = {}
|
|
fatal_errors: dict[str, str] = {}
|
|
control_plane_errors: dict[str, str] = {}
|
|
work_log_archive_errors: dict[str, str] = {}
|
|
review_shared_state_ready = False
|
|
candidate_scope: set[str] | None = None
|
|
task_cache: dict[str, Task] | None = None
|
|
resume_locators: dict[str, Path] = {}
|
|
legacy_recoveries: dict[str, LegacyPromotionRecovery] = {}
|
|
live_external_processes: dict[str, str] = {}
|
|
|
|
while True:
|
|
if task_cache is None:
|
|
tasks = scan_tasks(workspace, args.task_group)
|
|
task_cache = {task.name: task for task in tasks}
|
|
else:
|
|
tasks = sorted(task_cache.values(), key=lambda task: (task.index, task.name))
|
|
if args.dry_run:
|
|
persistent_errors: dict[str, str] = {}
|
|
observed_tasks: set[str] = set()
|
|
live_external_processes = {}
|
|
else:
|
|
store.prepare_orchestration(orchestration_scope, tasks, workspace)
|
|
live_external_processes = orchestration_live_agent_processes(
|
|
store,
|
|
orchestration_scope,
|
|
)
|
|
active_or_running = (
|
|
{task.name for task in tasks}
|
|
| set(running)
|
|
| set(live_external_processes)
|
|
)
|
|
reconciled_completed, persistent_errors = store.reconcile_orchestration(
|
|
orchestration_scope,
|
|
workspace,
|
|
active_or_running,
|
|
)
|
|
completed_tasks.update(reconciled_completed)
|
|
for task_name in persistent_errors:
|
|
completed_tasks.pop(task_name, None)
|
|
observed_tasks = store.orchestration_tasks(orchestration_scope)
|
|
work_log_archives, work_log_archive_errors = (
|
|
archive_completed_group_work_logs(
|
|
workspace,
|
|
observed_tasks,
|
|
completed_tasks,
|
|
active_or_running,
|
|
)
|
|
)
|
|
for task_group, archive in sorted(work_log_archives.items()):
|
|
banner(
|
|
"작업로그아카이브",
|
|
task_group,
|
|
[f"archive={archive}"],
|
|
)
|
|
if not tasks and not running:
|
|
if live_external_processes:
|
|
for task_name, detail in sorted(
|
|
live_external_processes.items()
|
|
):
|
|
banner(
|
|
"작업수행중",
|
|
task_name,
|
|
[
|
|
"이전 dispatcher의 model process를 종료시키지 않고 추적",
|
|
detail,
|
|
],
|
|
)
|
|
await asyncio.sleep(STREAM_HEARTBEAT_SECONDS)
|
|
continue
|
|
if control_plane_errors:
|
|
banner(
|
|
"디스패치추적대기",
|
|
args.task_group or "agent-task",
|
|
[
|
|
"예상하지 못한 dispatcher 중단 결과를 재조정해야 함",
|
|
*(
|
|
f"interrupted[{name}]={reason}"
|
|
for name, reason in sorted(control_plane_errors.items())
|
|
),
|
|
],
|
|
)
|
|
return 3
|
|
if work_log_archive_errors:
|
|
banner(
|
|
"디스패치추적대기",
|
|
args.task_group or "agent-task",
|
|
[
|
|
"완료 task group의 WORK_LOG archive를 재시도해야 함",
|
|
*(
|
|
f"work-log-archive[{group}]={reason}"
|
|
for group, reason in sorted(
|
|
work_log_archive_errors.items()
|
|
)
|
|
),
|
|
],
|
|
)
|
|
return 3
|
|
if args.task_group and not observed_tasks and not completed_tasks:
|
|
reason = (
|
|
"명시한 task group에서 관찰된 active task나 "
|
|
"검증된 complete.log 이력이 없다"
|
|
)
|
|
if not args.dry_run:
|
|
store.mark_orchestration_blocked(orchestration_scope, {})
|
|
banner(
|
|
"디스패치차단",
|
|
args.task_group,
|
|
[f"reason=unobserved-task-group", reason],
|
|
)
|
|
return 2
|
|
pending_attempt_logs = {
|
|
name: paths
|
|
for name in completed_tasks
|
|
if (
|
|
paths := task_attempt_log_directories(
|
|
store.runs,
|
|
name,
|
|
)
|
|
)
|
|
}
|
|
if pending_attempt_logs:
|
|
banner(
|
|
"디스패치추적대기",
|
|
args.task_group or "agent-task",
|
|
[
|
|
"완료 task의 attempt 로그 정리가 아직 끝나지 않음",
|
|
*(
|
|
f"attempt-log-cleanup-pending[{name}]="
|
|
+ ",".join(str(path) for path in paths)
|
|
for name, paths in sorted(pending_attempt_logs.items())
|
|
),
|
|
],
|
|
)
|
|
return 3
|
|
incomplete = sorted(observed_tasks - completed_tasks.keys())
|
|
if incomplete or fatal_errors or persistent_errors:
|
|
details = [
|
|
*(f"incomplete={name}" for name in incomplete),
|
|
*(
|
|
f"persistent[{name}]={reason}"
|
|
for name, reason in sorted(persistent_errors.items())
|
|
),
|
|
*(f"error[{name}]={reason}" for name, reason in sorted(fatal_errors.items())),
|
|
]
|
|
if not args.dry_run:
|
|
store.mark_orchestration_blocked(
|
|
orchestration_scope,
|
|
{
|
|
name: (
|
|
"blocked",
|
|
persistent_errors.get(name)
|
|
or fatal_errors.get(name)
|
|
or "관찰된 task가 완료되지 않았다",
|
|
)
|
|
for name in (
|
|
set(incomplete)
|
|
| set(fatal_errors)
|
|
| set(persistent_errors)
|
|
)
|
|
},
|
|
)
|
|
banner("디스패치차단", args.task_group or "agent-task", details)
|
|
return 2
|
|
if not args.dry_run:
|
|
store.mark_orchestration_complete(orchestration_scope)
|
|
banner(
|
|
"작업완료",
|
|
args.task_group or "agent-task",
|
|
[
|
|
"active task 없음",
|
|
f"verified_complete_tasks={len(completed_tasks)}",
|
|
*(f"complete[{name}]={path}" for name, path in sorted(completed_tasks.items())),
|
|
],
|
|
)
|
|
return 0
|
|
|
|
task_by_name = {task.name: task for task in tasks}
|
|
finished_names: set[str] = set()
|
|
complete_log_created = False
|
|
for name, future in list(running.items()):
|
|
if not future.done():
|
|
continue
|
|
finished_names.add(name)
|
|
completed_archive: str | None = None
|
|
try:
|
|
completed_archive = future.result()
|
|
if completed_archive:
|
|
complete_log_created = True
|
|
store.mark_orchestration_task_complete(
|
|
orchestration_scope, name, completed_archive
|
|
)
|
|
completed_tasks[name] = completed_archive
|
|
except Exception as exc: # keep other independent tasks alive
|
|
banner(
|
|
"디스패치추적대기",
|
|
name,
|
|
[f"agent coroutine exception={exc}"],
|
|
)
|
|
control_plane_errors[name] = str(exc)
|
|
task = task_by_name.get(name)
|
|
if task:
|
|
store.clear_active(task)
|
|
if not completed_archive:
|
|
refreshed = read_task_directory(workspace, task.directory)
|
|
if refreshed is None:
|
|
task_cache.pop(name, None)
|
|
else:
|
|
task_cache[name] = refreshed
|
|
if completed_archive:
|
|
task_cache.pop(name, None)
|
|
del running[name]
|
|
if finished_names:
|
|
if complete_log_created:
|
|
# Running reviewers may be archiving their own active directory
|
|
# while this completion-triggered scan runs. Preserve their
|
|
# already-loaded Task snapshots and do not reread those mutable
|
|
# directories until their futures finish.
|
|
running_snapshots = {
|
|
name: task_cache[name]
|
|
for name in running
|
|
if name in task_cache
|
|
}
|
|
tasks = scan_tasks(
|
|
workspace,
|
|
args.task_group,
|
|
exclude_names=set(running),
|
|
)
|
|
task_cache = {
|
|
**{task.name: task for task in tasks},
|
|
**running_snapshots,
|
|
}
|
|
tasks = sorted(
|
|
task_cache.values(),
|
|
key=lambda task: (task.index, task.name),
|
|
)
|
|
store.prepare_orchestration(orchestration_scope, tasks, workspace)
|
|
live_external_processes = orchestration_live_agent_processes(
|
|
store,
|
|
orchestration_scope,
|
|
)
|
|
active_or_running = (
|
|
{task.name for task in tasks}
|
|
| set(running)
|
|
| set(live_external_processes)
|
|
)
|
|
reconciled_completed, persistent_errors = store.reconcile_orchestration(
|
|
orchestration_scope,
|
|
workspace,
|
|
active_or_running,
|
|
)
|
|
completed_tasks.update(reconciled_completed)
|
|
for task_name in persistent_errors:
|
|
completed_tasks.pop(task_name, None)
|
|
observed_tasks = store.orchestration_tasks(orchestration_scope)
|
|
work_log_archives, work_log_archive_errors = (
|
|
archive_completed_group_work_logs(
|
|
workspace,
|
|
observed_tasks,
|
|
completed_tasks,
|
|
active_or_running,
|
|
)
|
|
)
|
|
for task_group, archive in sorted(
|
|
work_log_archives.items()
|
|
):
|
|
banner(
|
|
"작업로그아카이브",
|
|
task_group,
|
|
[f"archive={archive}"],
|
|
)
|
|
candidate_scope = None
|
|
else:
|
|
tasks = sorted(task_cache.values(), key=lambda task: (task.index, task.name))
|
|
candidate_scope = finished_names
|
|
|
|
if not tasks and not running:
|
|
# A completion-triggered full scan may have removed the last active task.
|
|
continue
|
|
|
|
ready: list[tuple[Task, str]] = []
|
|
waiting_tasks: list[str] = []
|
|
externally_active: list[tuple[Task, str]] = []
|
|
blocked_details: dict[str, tuple[str, str, str]] = {}
|
|
for task in tasks:
|
|
# A running future owns this task directory. Re-reading its review
|
|
# or dependency files can race with review finalization/archive and
|
|
# must never interrupt unrelated tasks.
|
|
if task.name in running:
|
|
continue
|
|
if task.name in control_plane_errors:
|
|
reason = (
|
|
"예상하지 못한 agent coroutine 중단 결과를 "
|
|
"다음 dispatcher가 재조정해야 함: "
|
|
f"{control_plane_errors[task.name]}"
|
|
)
|
|
blocked_details[task.name] = (
|
|
"디스패치추적대기",
|
|
"interrupted",
|
|
reason,
|
|
)
|
|
waiting_tasks.append(task.name)
|
|
continue
|
|
state = store.peek_task_state(task) if args.dry_run else store.task_state(task)
|
|
legacy_recovery: LegacyPromotionRecovery | None = None
|
|
legacy_blocker_reclassified = False
|
|
if state.get("blocked"):
|
|
legacy_recovery = legacy_promotion_recovery(
|
|
store.runs,
|
|
task,
|
|
state,
|
|
)
|
|
legacy_blocker_reclassified = legacy_recovery is not None
|
|
else:
|
|
legacy_recovery = (
|
|
pending_persisted_legacy_promotion_recovery(task, state)
|
|
)
|
|
if legacy_recovery is not None:
|
|
legacy_recoveries[task.name] = legacy_recovery
|
|
resume_locators[task.name] = legacy_recovery.locator
|
|
if legacy_blocker_reclassified:
|
|
state = dict(state)
|
|
state["blocked"] = None
|
|
if not args.dry_run:
|
|
recovery_failures = dict(
|
|
state.get("recovery_failures", {})
|
|
)
|
|
# Ten identical generic retries represent one terminal
|
|
# quota/context/model failure after reclassification.
|
|
recovery_failures[legacy_recovery.role] = 1
|
|
store.update_task(
|
|
task,
|
|
blocked=None,
|
|
recovery_failures=recovery_failures,
|
|
legacy_terminal_reclassification={
|
|
"role": legacy_recovery.role,
|
|
"failure_class": legacy_recovery.failure_class,
|
|
"evidence_source": legacy_recovery.evidence_source,
|
|
"prior_dispatcher_sha256":
|
|
legacy_recovery.prior_dispatcher_sha256,
|
|
"current_dispatcher_sha256":
|
|
DISPATCHER_SOURCE_SHA256,
|
|
"locator": str(legacy_recovery.locator),
|
|
"failed_cli": legacy_recovery.failed_cli,
|
|
"failed_model": legacy_recovery.failed_model,
|
|
"failed_reasoning_effort":
|
|
legacy_recovery.failed_reasoning_effort,
|
|
},
|
|
)
|
|
state = store.task_state(task)
|
|
active_predecessors = live_predecessors(
|
|
task,
|
|
set(running) | set(live_external_processes),
|
|
)
|
|
if active_predecessors:
|
|
dependency_ready = False
|
|
dependency = (
|
|
"predecessor FINISH 대기: "
|
|
+ ",".join(active_predecessors)
|
|
)
|
|
else:
|
|
dependency_ready, dependency = dependency_state(
|
|
workspace,
|
|
task,
|
|
)
|
|
stage = task_stage(task, state)
|
|
if state.get("active_stage"):
|
|
active_stage = str(state["active_stage"])
|
|
active_live, active_detail = external_active_is_live(state)
|
|
if active_live:
|
|
reason = f"외부 실행중: stage={active_stage}; {active_detail}"
|
|
active_key = (
|
|
f"active|{active_stage}|{state.get('active_locator') or 'unknown'}"
|
|
)
|
|
externally_active.append((task, active_stage))
|
|
blocked_details[task.name] = ("작업중", stage, reason)
|
|
if not args.dry_run and last_wait.get(task.name) != active_key:
|
|
banner("작업중", task.name, status_lines(task, stage, reason))
|
|
last_wait[task.name] = active_key
|
|
continue
|
|
if not args.dry_run:
|
|
resume_locator = laguna_resume_locator(state)
|
|
if resume_locator is not None:
|
|
resume_locators[task.name] = resume_locator
|
|
banner(
|
|
"작업복구",
|
|
task.name,
|
|
status_lines(task, stage, f"stale active 제외: {active_detail}"),
|
|
)
|
|
store.clear_active(task)
|
|
state = store.task_state(task)
|
|
reason = ""
|
|
if task.errors:
|
|
reason = "; ".join(task.errors)
|
|
elif task.user_review:
|
|
blocking, detail = user_review_blocker_state(task.user_review)
|
|
if task.plan is not None or task.review is not None:
|
|
blocking = False
|
|
detail = "active PLAN/CODE_REVIEW와 공존한다"
|
|
reason = (
|
|
f"USER_REVIEW 대기: {task.user_review}; {detail}"
|
|
if blocking
|
|
else (
|
|
"USER_REVIEW stop 계약 불충족: "
|
|
f"{task.user_review}; {detail}"
|
|
)
|
|
)
|
|
elif state.get("blocked"):
|
|
reason = str(state["blocked"])
|
|
elif not dependency_ready:
|
|
reason = dependency
|
|
if reason:
|
|
wait_key = f"{stage}|{reason}"
|
|
event = (
|
|
"작업차단"
|
|
if stage in {"blocked", "user-review"} or state.get("blocked")
|
|
else "작업대기"
|
|
)
|
|
blocked_details[task.name] = (event, stage, reason)
|
|
if not args.dry_run and last_wait.get(task.name) != wait_key:
|
|
banner(event, task.name, status_lines(task, stage, reason))
|
|
last_wait[task.name] = wait_key
|
|
waiting_tasks.append(task.name)
|
|
continue
|
|
if task.name not in running and (
|
|
candidate_scope is None or task.name in candidate_scope
|
|
):
|
|
ready.append((task, stage))
|
|
|
|
admission_time = datetime.now(KST)
|
|
if args.dry_run:
|
|
ready_by_name = {task.name: stage for task, stage in ready}
|
|
batch_snapshot = build_admission_batch_snapshot(store, ready, admission_time)
|
|
for task in tasks:
|
|
if task.name in ready_by_name:
|
|
stage = ready_by_name[task.name]
|
|
selector_stage = "review" if stage == "review" else "worker"
|
|
preview_state = store.peek_task_state(task)
|
|
decisions = preview_state.get("execution_decisions", {})
|
|
prior_decision = (
|
|
decisions.get(selector_stage)
|
|
if isinstance(decisions, dict)
|
|
else None
|
|
)
|
|
quota_snapshot = (
|
|
batch_snapshot
|
|
if batch_snapshot is not None
|
|
else (
|
|
preview_state.get("quota_snapshot")
|
|
if isinstance(preview_state.get("quota_snapshot"), dict)
|
|
else None
|
|
)
|
|
)
|
|
decision = read_or_preview_stage_decision(
|
|
task,
|
|
preview_state,
|
|
stage=selector_stage,
|
|
dry_run=args.dry_run,
|
|
quota_snapshot=quota_snapshot,
|
|
)
|
|
spec = agent_spec_from_decision(decision)
|
|
legacy_recovery = legacy_recoveries.get(task.name)
|
|
if legacy_recovery is not None:
|
|
failed_spec = failed_spec_from_recovery(
|
|
legacy_recovery
|
|
)
|
|
spec = promoted_spec(failed_spec, 0) or failed_spec
|
|
lines = status_lines(task, stage, "ready", decision=decision)
|
|
if legacy_recovery is not None:
|
|
lines.extend(
|
|
[
|
|
"recovery=legacy-terminal-reclassification",
|
|
f"failure_class={legacy_recovery.failure_class}",
|
|
f"locator={legacy_recovery.locator}",
|
|
]
|
|
)
|
|
banner(
|
|
"작업대기",
|
|
task.name,
|
|
lines + [f"model={spec.display}"],
|
|
)
|
|
else:
|
|
event, stage, reason = blocked_details[task.name]
|
|
banner(event, task.name, status_lines(task, stage, reason))
|
|
return 2 if waiting_tasks and not ready else 0
|
|
|
|
candidates, deferred, phase_wait_reason = select_dispatch_candidates(ready)
|
|
batch_snapshot = build_admission_batch_snapshot(store, candidates, admission_time)
|
|
for task, stage in deferred:
|
|
wait_key = f"{stage}|{phase_wait_reason}"
|
|
if last_wait.get(task.name) != wait_key:
|
|
banner("작업대기", task.name, status_lines(task, stage, phase_wait_reason))
|
|
last_wait[task.name] = wait_key
|
|
|
|
if (
|
|
not review_shared_state_ready
|
|
and any(stage == "review" for _, stage in candidates)
|
|
):
|
|
try:
|
|
ensure_review_shared_state(workspace)
|
|
except (OSError, RuntimeError) as exc:
|
|
# Shared review setup is a blocker only for reviews. It must
|
|
# not prevent dependency-independent workers/selfchecks from
|
|
# starting and draining in the same scheduler pass.
|
|
remaining_candidates: list[tuple[Task, str]] = []
|
|
for task, stage in candidates:
|
|
if stage != "review":
|
|
remaining_candidates.append((task, stage))
|
|
continue
|
|
reason = f"review shared-state preflight failed: {exc}"
|
|
store.update_task(task, blocked=reason)
|
|
fatal_errors[task.name] = reason
|
|
waiting_tasks.append(task.name)
|
|
blocked_details[task.name] = (
|
|
"작업차단",
|
|
stage,
|
|
reason,
|
|
)
|
|
banner(
|
|
"작업차단",
|
|
task.name,
|
|
status_lines(task, stage, reason),
|
|
)
|
|
candidates = remaining_candidates
|
|
else:
|
|
review_shared_state_ready = True
|
|
|
|
scheduled = False
|
|
for task, stage in candidates:
|
|
store.mark_active(task, stage)
|
|
resume_locator = resume_locators.pop(task.name, None)
|
|
state = store.task_state(task)
|
|
task_snapshot = batch_snapshot
|
|
if stage == "worker" and has_persisted_worker_decision(state, task) and not retry_quota_refresh_pending(state):
|
|
task_snapshot = None
|
|
|
|
if stage == "review":
|
|
future = asyncio.create_task(
|
|
run_review(
|
|
workspace,
|
|
store,
|
|
task,
|
|
**(
|
|
{"quota_snapshot": task_snapshot}
|
|
if task_snapshot is not None
|
|
else {}
|
|
),
|
|
**(
|
|
{"resume_locator": resume_locator}
|
|
if resume_locator is not None
|
|
else {}
|
|
),
|
|
)
|
|
)
|
|
elif stage == "selfcheck":
|
|
future = asyncio.create_task(
|
|
run_selfcheck(
|
|
workspace,
|
|
store,
|
|
task,
|
|
**(
|
|
{"resume_locator": resume_locator}
|
|
if resume_locator is not None
|
|
else {}
|
|
),
|
|
)
|
|
)
|
|
else:
|
|
future = asyncio.create_task(
|
|
run_worker(
|
|
workspace,
|
|
store,
|
|
task,
|
|
**(
|
|
{"quota_snapshot": task_snapshot}
|
|
if task_snapshot is not None
|
|
else {}
|
|
),
|
|
**(
|
|
{"resume_locator": resume_locator}
|
|
if resume_locator is not None
|
|
else {}
|
|
),
|
|
)
|
|
)
|
|
running[task.name] = future
|
|
last_wait.pop(task.name, None)
|
|
scheduled = True
|
|
|
|
if running:
|
|
await asyncio.wait(running.values(), return_when=asyncio.FIRST_COMPLETED)
|
|
continue
|
|
if work_log_archive_errors:
|
|
banner(
|
|
"디스패치추적대기",
|
|
args.task_group or "agent-task",
|
|
[
|
|
"완료 task group의 WORK_LOG archive를 재시도해야 함",
|
|
*(
|
|
f"work-log-archive[{group}]={reason}"
|
|
for group, reason in sorted(
|
|
work_log_archive_errors.items()
|
|
)
|
|
),
|
|
],
|
|
)
|
|
return 3
|
|
if externally_active:
|
|
banner(
|
|
"디스패치추적대기",
|
|
args.task_group or "agent-task",
|
|
["새 실행 후보 없음", "active task는 caller가 계속 추적"],
|
|
)
|
|
return 3
|
|
if control_plane_errors:
|
|
banner(
|
|
"디스패치추적대기",
|
|
args.task_group or "agent-task",
|
|
[
|
|
"실행 중이던 독립 작업을 모두 소진했고 재조정이 필요함",
|
|
*(
|
|
f"interrupted[{name}]={reason}"
|
|
for name, reason in sorted(control_plane_errors.items())
|
|
),
|
|
],
|
|
)
|
|
return 3
|
|
if not scheduled:
|
|
store.mark_orchestration_blocked(
|
|
orchestration_scope,
|
|
{
|
|
name: (
|
|
"blocked" if event == "작업차단" else "waiting",
|
|
detail,
|
|
)
|
|
for name, (event, _, detail) in blocked_details.items()
|
|
},
|
|
)
|
|
reason = f"waiting={','.join(sorted(waiting_tasks))}"
|
|
banner(
|
|
"디스패치차단",
|
|
args.task_group or "agent-task",
|
|
[
|
|
"실행 가능한 독립 작업을 모두 소진함",
|
|
reason,
|
|
f"verified_complete_tasks={len(completed_tasks)}",
|
|
*(
|
|
f"complete[{name}]={path}"
|
|
for name, path in sorted(completed_tasks.items())
|
|
),
|
|
*(
|
|
f"{name}: stage={stage}; reason={detail}"
|
|
for name, (_, stage, detail) in sorted(
|
|
blocked_details.items()
|
|
)
|
|
),
|
|
],
|
|
)
|
|
return 2
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--workspace", default=".", help="repository root (default: current directory)")
|
|
parser.add_argument("--task-group", help="run only agent-task/<task_group>")
|
|
parser.add_argument("--dry-run", action="store_true", help="classify and print without launching CLIs")
|
|
parser.add_argument("--retry-blocked", action="store_true", help="clear dispatcher-local blocked state")
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
try:
|
|
return asyncio.run(dispatch(args))
|
|
except KeyboardInterrupt:
|
|
print("\n중단됨", file=sys.stderr)
|
|
return 130
|
|
except DispatcherAlreadyRunning as exc:
|
|
print(f"dispatcher active: {exc}", file=sys.stderr)
|
|
return 3
|
|
except DispatcherTerminalStateError as exc:
|
|
print(f"dispatcher error: {exc}", file=sys.stderr)
|
|
return 2
|
|
except Exception as exc:
|
|
# An unexpected dispatcher failure is not proof that the task group is
|
|
# drained or terminal. The caller must inspect active PIDs/locators and
|
|
# recover instead of treating it like exit 2.
|
|
print(f"dispatcher interrupted: {exc}", file=sys.stderr)
|
|
return 3
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|